diff --git a/.env.example b/.env.example index 036d8f4..c233a7b 100644 --- a/.env.example +++ b/.env.example @@ -26,6 +26,9 @@ CEREBRAS_API_KEY= # OpenRouter — https://openrouter.ai/keys OPENROUTER_API_KEY= +# DeepSeek — https://platform.deepseek.com/api_keys +DEEPSEEK_API_KEY= + # Clerk OAuth (phase 9) — https://clerk.com/keys # Server: CLERK_SECRET_KEY + CLERK_PUBLISHABLE_KEY validate Bearer tokens on /sessions and /chat. # CLI: CLERK_FRONTEND_API + CLERK_OAUTH_CLIENT_ID drive the browser /login PKCE flow. diff --git a/DESIGN.md b/DESIGN.md index 3563904..bbc4e74 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -103,7 +103,7 @@ components: reasoning-block: { ref: "BotMessage.reasoning", role: "quiet-bar block, 'Thinking:' label, dim body" } tool-block: { ref: "BotMessage.tool", role: "quiet-bar block, info-colored tool label, args, pending/error suffix" } composer: { ref: InputBar, role: "Multiline textarea inside the mode-tinted accent bar; hosts overlays" } - status-bar: { ref: StatusBar, role: "Single dim row: mode › model · submit · newline hints" } + status-bar: { ref: StatusBar, role: "Single dim row: mode › model · live phase · fallback notice · loop budget · submit · newline hints" } session-footer: { ref: "SessionShell.footer", role: "Spinner + interrupt hint (left) / tab-agent hint (right)" } dialog-shell: { ref: "DialogProvider.Dialog", role: "Centered modal overlay card; dim backdrop; title + esc-affordance" } search-list: { ref: DialogSearchList, role: "Filter input + scrolling single-select list with selection highlight" } @@ -171,7 +171,14 @@ Read **bottom-up** — the way the user experiences it. This mirrors Claude Code multiline `textarea`. Slash-command and @-mention palettes float **above** it (`position:absolute; bottom:100%`), never pushing the transcript. - **Status line** (`{components.status-bar}`): exactly one dim row, Codex-style — - `mode {glyph.sep-chevron} model {glyph.sep-dot} submit hint {glyph.sep-dot} newline hint`. + `mode {glyph.sep-chevron} model {glyph.sep-dot} live phase {glyph.sep-dot} loop budget {glyph.sep-dot} submit hint {glyph.sep-dot} newline hint`. + The live-phase segment (`等待首字` / `思考中` / `执行中` / `生成中`, each with an + elapsed timer) answers "what is it doing right now" and is derived from the + streaming message, so it can never disagree with the transcript. It disappears + when the agent is idle. + The loop-budget segment (`turn 3/51`, plus `4 tools` when the current turn has + run tools) is `DIM` meta and renders only once a run has started — it answers + "how much of the agent budget has this message spent" without adding a second row. - **Footer** (`{components.session-footer}`): spinner + `esc to interrupt` on the left while streaming; `tab agent` hint pinned right. diff --git a/README.md b/README.md index 1d549c8..396915c 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ MoCode is a Bun monorepo that pairs a rich **TUI client** with a **Hono API serv |---|---| | **Dual agent modes** | **Plan** — read-only exploration and architecture. **Build** — full read/write/bash toolset for implementation. | | **Local tool execution** | The server defines tools; the CLI runs them against `process.cwd()` with path sandboxing. | -| **Multi-provider models** | Anthropic, OpenAI, Google, Groq, Cerebras, and OpenRouter — switch models mid-session with `/models`. | +| **Multi-provider models** | Anthropic, OpenAI, Google, Groq, Cerebras, OpenRouter, and DeepSeek — switch models mid-session with `/models`. | | **Session persistence & recovery** | PostgreSQL (SaaS) or local JSON (`--local`). Esc keeps partial text; tool Esc marks `Interrupted by user` and stops immediately; auto-resume on user-only tail; partial assistant uses `/resume` (regenerate). See [Session recovery](#session-recovery). | | **Rich TUI** | Built with [OpenTUI](https://github.com/opentui/opentui) + React 19 — themes, dialogs, keyboard layers, and slash commands. | | **OAuth sign-in** | Browser-based PKCE flow via Clerk; tokens stored at `~/.mocode/auth.json`. | @@ -148,6 +148,7 @@ mocode | `GROQ_API_KEY` | Server | Groq API key | | `CEREBRAS_API_KEY` | Server | Cerebras API key | | `OPENROUTER_API_KEY` | Server | OpenRouter API key | +| `DEEPSEEK_API_KEY` | Server | DeepSeek API key | | `CLERK_SECRET_KEY` | Server | Clerk secret key (auth) | | `CLERK_PUBLISHABLE_KEY` | Server | Clerk publishable key | | `CLERK_FRONTEND_API` | CLI | Clerk frontend API domain | @@ -232,16 +233,19 @@ Implementation details: [`doc/harness-phase-03-stream-reliability-notes.md`](./d | `llama-3.3-70b-versatile` | Groq (free; weak tool calling) | | `openai/gpt-oss-20b` | Groq (free) | | `openai/gpt-oss-120b` | Groq (free) | -| `gpt-oss-120b` | Cerebras (default for BYOK; free recommended) | +| `gpt-oss-120b` | Cerebras (free recommended) | | `openai/gpt-oss-120b:free` | OpenRouter | +| `deepseek-flash` | DeepSeek (**default**; 1M context; reachable from mainland China) | +| `deepseek-v4-pro` | DeepSeek (stronger, ~4x the price) | The canonical list lives in `packages/shared/src/models.ts`. ### BYOK (`mocode --local`) -- **Default model:** `gpt-oss-120b` (Cerebras). Set provider keys with **`/keys`** → `~/.mocode/keys.json` (local only; never sent to the MoCode server). +- **Default model:** `deepseek-flash` (DeepSeek). Set provider keys with **`/keys`** → `~/.mocode/keys.json` (local only; never sent to the MoCode server). +- **Why not Cerebras/Groq:** those endpoints answer `403 Forbidden` from networks where the provider is region-blocked. DeepSeek is reachable without a proxy; use `/models` to switch. - **`/models` hints:** *Free recommended* / *Free OK* / *Weak tool calling* — avoid weak-tool models for `/explore` and other multi-tool turns. -- **Free-tier picks:** Cerebras `gpt-oss-120b` or Groq `openai/gpt-oss-120b`. Groq Llama ids often malform tool calls in subagent loops. +- **Free-tier picks:** Cerebras `gpt-oss-120b` or Groq `openai/gpt-oss-120b` (only where those APIs are reachable). Groq Llama ids often malform tool calls in subagent loops. Smoke-test server `.env` keys: `bun run packages/server/scripts/test-providers.ts` diff --git a/README.zh-CN.md b/README.zh-CN.md index ed3dde9..72fdc7e 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -18,7 +18,7 @@ MoCode 是一个基于 Bun 的 monorepo,由功能丰富的 **TUI 客户端** |---|---| | **双 Agent 模式** | **Plan** — 只读探索与架构分析。**Build** — 完整读写/bash 工具集,用于实现代码。 | | **本地工具执行** | 服务端定义工具契约,CLI 在 `process.cwd()` 下执行,并做路径沙箱隔离。 | -| **多模型支持** | Anthropic、OpenAI、Google、Groq、Cerebras、OpenRouter — 会话中随时用 `/models` 切换。 | +| **多模型支持** | Anthropic、OpenAI、Google、Groq、Cerebras、OpenRouter、DeepSeek — 会话中随时用 `/models` 切换。 | | **会话持久化与恢复** | PostgreSQL(SaaS)或本地 JSON(`--local`)。Esc 保留 partial 正文;工具执行中 Esc 标记 `Interrupted by user` 并立即停止;末条 user 重开自动续写;partial 助手需 `/resume`(regenerate)。详见 [会话恢复](#会话恢复)。 | | **富 TUI 界面** | 基于 [OpenTUI](https://github.com/opentui/opentui) + React 19 — 主题、对话框、键盘层与斜杠命令。 | | **OAuth 登录** | 通过 Clerk 的浏览器 PKCE 流程;Token 保存在 `~/.mocode/auth.json`。 | @@ -148,6 +148,7 @@ mocode | `GROQ_API_KEY` | 服务端 | Groq API Key | | `CEREBRAS_API_KEY` | 服务端 | Cerebras API Key | | `OPENROUTER_API_KEY` | 服务端 | OpenRouter API Key | +| `DEEPSEEK_API_KEY` | 服务端 | DeepSeek API Key | | `CLERK_SECRET_KEY` | 服务端 | Clerk Secret Key(认证) | | `CLERK_PUBLISHABLE_KEY` | 服务端 | Clerk Publishable Key | | `CLERK_FRONTEND_API` | CLI | Clerk Frontend API 域名 | @@ -232,16 +233,19 @@ macOS Terminal.app 可运行 `mocode --terminal-setup` 一次,启用 Option-as | `llama-3.3-70b-versatile` | Groq(免费;工具调用不稳定) | | `openai/gpt-oss-20b` | Groq(免费) | | `openai/gpt-oss-120b` | Groq(免费) | -| `gpt-oss-120b` | Cerebras(BYOK 默认;免费推荐) | +| `gpt-oss-120b` | Cerebras(免费推荐) | | `openai/gpt-oss-120b:free` | OpenRouter | +| `deepseek-flash` | DeepSeek(**默认**;1M 上下文;国内可直连) | +| `deepseek-v4-pro` | DeepSeek(更强,价格约为 4 倍) | 权威列表见 `packages/shared/src/models.ts`。 ### BYOK(`mocode --local`) -- **默认模型:** `gpt-oss-120b`(Cerebras)。用 **`/keys`** 写入 `~/.mocode/keys.json`(仅存本机,不上传服务端)。 +- **默认模型:** `deepseek-flash`(DeepSeek)。用 **`/keys`** 写入 `~/.mocode/keys.json`(仅存本机,不上传服务端)。 +- **为什么不用 Cerebras/Groq:** 在厂商按地区封锁的网络里,这两个端点会返回 `403 Forbidden`。DeepSeek 可直连;用 `/models` 切换。 - **`/models` 标注:** *免费推荐* / *免费可用* / *工具调用不稳定* — `/explore` 等多工具场景请避开不稳定项。 -- **免费档推荐:** Cerebras `gpt-oss-120b` 或 Groq `openai/gpt-oss-120b`。Groq Llama 系列在子代理多步 tool loop 中易格式错误。 +- **免费档推荐:** Cerebras `gpt-oss-120b` 或 Groq `openai/gpt-oss-120b`(仅在这两家可达的网络下)。Groq Llama 系列在子代理多步 tool loop 中易格式错误。 冒烟测试服务端 `.env` Key:`bun run packages/server/scripts/test-providers.ts` diff --git a/bun.lock b/bun.lock index 02caa27..7121343 100644 --- a/bun.lock +++ b/bun.lock @@ -333,7 +333,7 @@ "@tanstack/query-core": ["@tanstack/query-core@5.101.0", "https://registry.npmmirror.com/@tanstack/query-core/-/query-core-5.101.0.tgz", {}, "sha512-cQetA74EB+seWySv1TTKr828TnP0u39m6LykwDXIo84SNortpDkp30TMEjkqtYCNP9c40uT/iwl6MLiufEt0Ow=="], - "@types/bun": ["@types/bun@1.3.14", "https://registry.npmmirror.com/@types/bun/-/bun-1.3.14.tgz", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + "@types/bun": ["@types/bun@1.4.4", "https://registry.npmmirror.com/@types/bun/-/bun-1.4.4.tgz", { "dependencies": { "bun-types": "1.4.4" } }, "sha512-9wkRDXUXK124pEnAutE6n/l18j+i9GC5IAHK8TmX/2nlySETmky0OU6yL5U8pW1AoKh++AN9auDCr647BfurjQ=="], "@types/node": ["@types/node@25.9.3", "https://registry.npmmirror.com/@types/node/-/node-25.9.3.tgz", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg=="], @@ -393,7 +393,7 @@ "bun-ffi-structs": ["bun-ffi-structs@0.2.3", "https://registry.npmmirror.com/bun-ffi-structs/-/bun-ffi-structs-0.2.3.tgz", { "peerDependencies": { "typescript": "^5" } }, "sha512-pgJiXP+hEgFo9qG51J6ItfY4ocs3vniwNzJ9WhoakB3QB2GdzQxX2EXssentPYlB2hOfJrTjO6iIQkWYzUodpg=="], - "bun-types": ["bun-types@1.3.14", "https://registry.npmmirror.com/bun-types/-/bun-types-1.3.14.tgz", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + "bun-types": ["bun-types@1.4.4", "https://registry.npmmirror.com/bun-types/-/bun-types-1.4.4.tgz", { "dependencies": { "@types/node": "*", "undici-types": "*" } }, "sha512-yjrJh5oGX7mMPKGd19YXyIjmEuj4G4zNC30bs7PM9Az7NQ3ndF3j6hbeRdvJTRQEhJ/FYQwW0a2moweEi2hMxQ=="], "bundle-name": ["bundle-name@4.1.0", "https://registry.npmmirror.com/bundle-name/-/bundle-name-4.1.0.tgz", { "dependencies": { "run-applescript": "^7.0.0" } }, "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q=="], diff --git a/docs/agent-permissions.md b/docs/agent-permissions.md index 63cbb7c..2b332d0 100644 --- a/docs/agent-permissions.md +++ b/docs/agent-permissions.md @@ -24,4 +24,258 @@ In Build mode the model should: ## Plan mode Plan mode has no bash tool. Git inspection uses read-only `gitStatus` and `gitDiff` tools instead. + +## Enforcement layers + +Permission is not one check — it is four, applied in order. Each layer assumes the +previous one may have been bypassed, which is why Plan mode is enforced twice. + +| Layer | Where | What it stops | +|---|---|---| +| 1. Tool contracts | `packages/shared/src/schemas.ts` (`getToolContracts`) | Plan mode never *receives* write/bash tool definitions | +| 2. Executor mode gate | `packages/cli/src/lib/local-tools.ts` (`READ_ONLY_TOOLS`) | A write tool that leaked through is still rejected before touching disk | +| 3. Path containment | `packages/cli/src/lib/sandbox-path.ts` | `../../etc/passwd`, absolute paths, and **symlinks pointing outside the project** — **file tools only** | +| 4. Hard denies | `packages/cli/src/lib/permission-rules.ts` (`findHardDeny`) | Host-destroying commands — refused **even after the user clicks approve** | + +### Layer 0: the OS sandbox (the only thing that contains a shell) + +Layers 1–3 are path checks, and **a path check cannot contain a shell**. That was +not a theory: manual acceptance caught the agent running + +``` +Bash: head -3 secret.txt # secret.txt -> /etc/passwd +## +# User Database +``` + +which bypassed `readFile` entirely and printed `/etc/passwd`. The file tools were +fenced; the shell was not. + +So the outermost layer is an OS sandbox, applied at spawn: + +| host | shell process | +|---|---| +| seatbelt applies (macOS) | `sandbox-exec -p bash -c …`, `(deny default)` | +| bwrap applies (Linux) | `bwrap --ro-bind / / --bind … bash -c …` | +| neither | `bash -c …`, **unconfined** — every command needs approval | + +Verified on macOS with seatbelt active — the original exploit, replayed: + +``` +$ ln -s /etc/passwd secret.txt +$ head -3 secret.txt +head: secret.txt: Operation not permitted (exit 1) + +readFile("secret.txt") → Path is outside the project directory +``` + +Both independent guards refuse, and neither asks for approval first: the command is +not blocklisted, the shell is confined, so it simply runs and the **kernel** denies +the read. That distinction matters — a denial that arrives as an approval prompt is +a policy, a denial that arrives as `EPERM` is a boundary. + +**Without a working OS sandbox there is no containment, only gating.** The approval +policy says so explicitly: when `isShellConfined()` is false, every bash command +asks, because a blocklist in front of an unconfined shell is theatre. + +### Capability tracking: `available` vs `enforced` + +`sandbox-capability.ts` probes the host at first use (seatbelt on macOS, bwrap on +Linux) and reports two separate flags: + +| flag | meaning | +|---|---| +| `available` | the host *can* apply a sandbox — a trivial profile actually ran | +| `enforced` | the spawn site *does* apply it to shell commands | + +The spawn site now wraps the shell when both flags hold: + +| host | shell command | approval | +|---|---|---| +| seatbelt/bwrap applies | `sandbox-exec -p bash -c …` / `bwrap … bash -c …` | blocklist only | +| nothing applies | `bash -c …` | **every** command asks | + +When the shell is confined, `head -3 secret.txt` fails with a seatbelt denial — +the content never reaches the model. When it is not confined, the run still +happens but each command needs a human, because a blocklist alone is theatre. + +### Profile policy + +Seatbelt denies by default and allows only: + +- **read**: `/` in general — a narrow allowlist made dyld/cryptex hosts abort at + startup, so the rule is a broad allow followed by an explicit deny list +- **denied read**: `/etc/passwd`, `/etc/shadow`, `/etc/sudoers`, `/root`, + `/var/root`, plus `~/.ssh`, `~/.aws`, `~/.gnupg`, `~/.kube`, `~/.docker`, + `~/.config/gh`, `~/.config/gcloud`, `~/.npmrc`, `~/.yarnrc`, `~/.pypirc`, + `~/.netrc` +- **write**: project tree, `$TMPDIR`, `/dev/null`, `/dev/tty`, plus + `MOCODE_SANDBOX_EXTRA_WRITE` + +`$HOME` is **not** unreadable as a whole — only the listed credential trees are. +If you add a tool that stores tokens somewhere new, add that path to +`deniedReadPaths`; the deny list is the boundary, not the absence of an allow rule. + +bwrap cannot express "deny by default" without rebuilding the root, so it mounts a +read-only `/` with a writable project and masks `~/.ssh`, `~/.aws`, `~/.gnupg`, +`/etc/shadow` and friends onto `/dev/null`. + +### Verifying confinement + +```bash +# 1. What does this host report? +bun run packages/cli/scripts/acceptance.ts # prints the capability line + +# 2. Does the fence actually hold? +# Ask the agent to run: bash: head -1 /etc/passwd +# Confined → "Operation not permitted" (seatbelt), no content +# Unconfined → content returned, and the command required approval first +``` + +`MOCODE_DISABLE_SANDBOX=1` turns confinement off for tools that break under +seatbelt; the approval policy notices and goes back to asking for everything. +`MOCODE_ALLOW_UNCONFINED_SHELL=1` is the separate, explicit "I accept an +unconfined shell, stop asking" switch. + +### Layer 3: containment is checked twice + +The old check was lexical only, so a symlink inside the project defeated it: + +```bash +ln -s /etc ./docs # docs lives inside the project +readFile("docs/passwd") # lexically inside, actually /etc/passwd +``` + +`resolveInsideCwd` now also compares **real paths** of the deepest existing +ancestor, which works for files that do not exist yet (`writeFile` creating a new +directory tree). + +### Layer 4: approval is not a licence + +Approval exists for *destructive but legitimate* work (`rm -rf ./dist`). It does +not extend to `mkfs`, `dd of=/dev/…`, `rm -rf /`, or a fork bomb — the executor +refuses those before spawning, so a mistaken "approve" cannot format a disk. + +### Shell environment is an allowlist + +The spawned shell receives only the keys in `SANDBOX_ENV_KEYS` (`PATH`, `HOME`, +`LANG`, `SHELL`, `TMPDIR`, …) with `TERM=dumb`. Every secret the CLI was started +with — `DEEPSEEK_API_KEY`, `CLERK_SECRET_KEY`, `DATABASE_URL` — is absent from the +child process. Previously bash inherited all of `process.env`. + +### Refusal codes and audit trail + +A refusal the model merely *reasons about* is not a control — the model can be +argued out of it. The executor is the authority, and every decision it makes is +recorded. + +Refusals raise `ToolPermissionError` with a machine-readable code: + +| code | meaning | +|---|---| +| `PATH_ESCAPE` | resolved path left the project (lexically or via symlink) | +| `HARD_DENY` | host-destroying command, not approvable | +| `MODE_DENIED` | tool not available in the current mode | +| `UNKNOWN_TOOL` | tool name is not in the executor's switch | + +The message stays human-readable for the model; the code is what dashboards and +audits group by. Prose drifts, codes do not. + +Every call — allowed, refused, or failed — appends one JSONL line to +`~/.mocode/audit.jsonl` (override with `MOCODE_AUDIT_PATH`): + +```json +{"timestamp":"2026-11-11T13:40:02.114Z","tool":"readFile","mode":"BUILD","outcome":"denied","code":"PATH_ESCAPE","detail":"path=/etc/passwd"} +``` + +Three rules make the log safe to keep: + +1. **No file contents.** Only short single-line scalars are summarised; anything + longer than 120 chars or containing a newline is dropped, not truncated. +2. **Never breaks a call.** A full disk or read-only home returns `false` instead + of throwing — losing a log line must not fail a working tool. +3. **One line per decision**, append-only, `jq`-friendly. + +### Rotation + +An unbounded audit file is its own outage. The log rotates before each append: + +| | default | override | +|---|---|---| +| rotate at | 5 MB | `MOCODE_AUDIT_MAX_BYTES` | +| archives kept | 5 | `MOCODE_AUDIT_MAX_FILES` | + +Archives are numbered newest-first — `audit.jsonl.1` holds the most recent rotated +segment and `.N` the oldest, which is dropped on the next rotation. Disk cost is +therefore bounded: `maxBytes × (maxFiles + 1)`. + +Rotation is size-triggered rather than time-triggered on purpose: a CLI process may +live for minutes or for a day, and time-based cutoffs either lose the burst or +create dozens of empty files. + +**Known limit:** rotation is a rename done synchronously in-process. Two CLI +processes sharing one `~/.mocode` could race; the loser appends to a file that was +just rotated away. Acceptable for a single-user CLI, not for a shared agent host — +that needs an append-only sink (file → log shipper), not a local file. + +### User rules + +The built-in policy is a floor, not the whole story: a team needs to say "never +`git push` here" or "always allow `npm test`" without patching source. + +Rules live in JSON, so they are reviewable and diffable: + +```jsonc +// /.mocode/permissions.json — committed with the repo +// ~/.mocode/permissions.json — applies to every project +{ + "rules": [ + { "tool": "bash", "pattern": "^git push", "behavior": "deny" }, + { "tool": "bash", "pattern": "^npm (test|run lint)", "behavior": "allow" }, + { "tool": "writeFile", "behavior": "ask" } + ] +} +``` + +| field | meaning | +|---|---| +| `tool` | exact tool name, or `*` | +| `pattern` | optional JS regex tested against the command (bash) or path (file tools) | +| `behavior` | `allow` / `deny` / `ask` | + +**Precedence is deny > allow > ask**, and a `deny` in either file wins over an +`allow` in the other — a safety rule must not be defeatable by a broader allow. +Within one behaviour the first matching rule wins, so the file reads top-down. + +Enforcement happens in two places on purpose: + +1. the **approval gate** — `deny` returns immediately without showing a dialog, + `allow` skips the built-in dialog, `ask` forces one; +2. the **executor** — `deny` throws `RULE_DENIED` and is audited, so a caller that + never goes through the UI (evals, subagents) is still bound by the policy. + +A malformed rule is **dropped and reported**, never coerced: turning +`"behavior": "denied"` into an allow would be the worst possible failure. Rules are +re-read on every call, so an edit applies to the next tool call without a restart. + +**Not implemented yet:** writing rules back from the TUI ("always allow for this +project"). The approval dialogs still record session-only allows; persisting them +would need a new dialog action. + +### Regression suite + +`packages/cli/src/lib/local-tools.sandbox.test.ts` runs the attack cases against the +real `executeLocalTool` entry point: + +- absolute path, parent traversal, write-outside, list-outside → all refused +- `dd of=/dev/…` and `mkfs` → refused at execution time +- a canary `DEEPSEEK_API_KEY` is invisible to the shell +- `PATH` still works, `TERM` is `dumb` +- Plan mode still rejects write/bash even if the contract leaked +- every refusal is logged with its code; a 500-char file body never reaches the log + +Manual check: ask the agent to read `/etc/passwd` — it must report a refusal, not +content. Then approve a `mkdir` (should run) and approve `mkfs.ext4 /dev/null` +(should be refused with "cannot be approved"). + \ No newline at end of file diff --git a/docs/context-budget.md b/docs/context-budget.md new file mode 100644 index 0000000..f64c20e --- /dev/null +++ b/docs/context-budget.md @@ -0,0 +1,90 @@ +# Context budget & compaction + +The server replays the stored session on every turn, so the prompt grows linearly +with session length. Left alone, a long session either overflows the model window +or pays for history nobody re-reads. + +Compaction puts a **ceiling** on that growth: past the budget, old turns collapse +into one digest message. + +## Policy + +```ts +{ maxTokens: 60_000, keepRecent: 10, minMessages: 16 } +``` + +- `maxTokens` is a **cost** budget, not a context-window budget. DeepSeek Flash + accepts 1M tokens; 60k is where continued growth stops being worth paying for. +- Below `minMessages` compaction is skipped — a short conversation is mostly + intent, and folding it would destroy the task. + +## What the digest contains + +``` +[compacted] 90 earlier messages folded away (45 user turns). +Tools already used: readFile×23, bash×22. +Files touched: src/module-0.ts, src/module-1.ts, … +Those turns are no longer in context; re-read a file if you need its exact contents. +``` + +It is **deterministic, not an LLM summary**. That is a deliberate trade: + +| | deterministic digest | LLM summary | +|---|---|---| +| cost | 0 | one extra model call per compaction | +| latency | 0 | +1 round trip | +| failure mode | none | the summarizer can itself fail or hallucinate | +| fidelity | what tools ran, which files were touched | prose semantics | + +What the agent actually needs from old turns is *what it already did* — tool names, +file paths — not the prose around it. The last line matters more than it looks: it +tells the model the content is **gone**, so it re-reads instead of inventing. + +## Measured + +```bash +bun run packages/server/scripts/context-compaction-bench.ts +``` + +Synthetic but realistically sized: one tool call per turn, 10k-char output (the +`readFile` truncation limit). + +| turns | prompt tokens (no compaction) | with compaction | +|---|---|---| +| 20 | 55,580 | 55,580 (not yet triggered) | +| 25 | 69,485 | 14,011 (−80%) | +| 50 | **139,010** | **14,011 (−90%)** | + +The number that matters is not the −90% at turn 50, it is that the prompt is +**flat** after turn ~24. Without it, cost per turn grows linearly forever. + +## Design rules + +1. **The cut lands on a user message.** Splitting a tool call from its result would + produce an invalid conversation, so `findCutIndex` snaps forward to the next + `user` role. +2. **Compaction shapes model input only.** The persisted message list and the + messages echoed back to the client stay complete — otherwise the user's own + transcript would lose history. +3. **It is bounded.** Running compaction twice does not compound; a covered test + asserts the second pass never increases tokens. +4. **It is observable.** When a turn is compacted, the finish metadata carries + `compaction: { tokensBefore, tokensAfter }`. + +## Manual acceptance + +1. Start a session, let it run past ~25 tool calls with large files. +2. The finish metadata of that turn should carry `compaction` (visible in the SSE + frame or server logs). +3. Continue the session — the prompt estimate stays flat instead of climbing. +4. Ask the agent about something from the compacted range. Expected: it re-reads + the file rather than fabricating details. That is the digest working. + +## Follow-ups + +- ~~BYOK has no compaction~~ — done: the module moved to `@mocode/shared` and both + the server route and `LocalChatTransport` apply the same policy. +- **Tool-output truncation is still flat** (10k/20k chars per tool). Per-tool, + budget-aware limits would squeeze more out of a turn. +- **No eviction of stale file reads.** Re-reading the same file repeatedly still + costs full tokens each time. diff --git a/docs/decisions.md b/docs/decisions.md new file mode 100644 index 0000000..a1048a1 --- /dev/null +++ b/docs/decisions.md @@ -0,0 +1,103 @@ +# Decisions — the numbers that look arbitrary + +Every constant in this repo that a reader could reasonably call "made up" gets an +entry: the value, where it came from, and the condition that should make you change +it. Anything without a source is marked as a judgment call — pretending otherwise +is how a codebase accumulates cargo cult. + +Sources below were read from first-party code, not blog posts. + +## Turn budget + +| Product | Limit | First-party source | +|---|---|---| +| Claude Agent SDK | **unbounded** — `max_turns: int \| None = None` | `anthropics/claude-agent-sdk-python`, `src/claude_agent_sdk/types.py` | +| LangGraph 1.2.4 | **10007** — `DEFAULT_RECURSION_LIMIT`, env `LANGGRAPH_DEFAULT_RECURSION_LIMIT` | `langgraph/_internal/_config.py:31` | +| OpenAI Agents SDK | **10** — `DEFAULT_MAX_TURNS` | `openai/openai-agents-python`, `src/agents/run_config.py:45` | +| **MoCode, main loop** | **50** continuations = 51 responses | `agent-loop.ts` | +| **MoCode, subagent** | **25** steps | `subagent/runner.ts` | + +**Why 50.** The previous value was 24, copied from the subagent cap. That cap is +itself legacy: the widely repeated "LangGraph defaults to 25" is obsolete — +1.2.4 ships 10007, raised because 25 interrupted legitimate long tasks. The +published range today is 10 (OpenAI) to unbounded (Anthropic). A coding agent sits +near the long end, but "unbounded" turns a bug into an unbounded bill, so this stays +a budget. Measured on 20 eval tasks: max 13 turns, so 50 is ~4x headroom. + +Override: `MOCODE_MAX_TURNS`. + +**Why the subagent stays 25.** It runs synchronously — the parent blocks until it +returns a summary, so a generous budget spends the parent's wall clock. The tight +cap is the feature. (This is a better reason than "keep the two numbers equal", +which is what the previous comment said.) + +## Tool calls per turn: 16 + +No published equivalent. OpenAI's `ToolExecutionConfig.max_function_tool_concurrency` +bounds *concurrency*, not the number of calls in one turn. + +This is a **circuit breaker, not a budget**: measured turns use a mean of 1.26 tools +and a max of 2.33. 16 is roughly 7x the observed max — high enough that batch work +("change the imports in these 12 files") never trips it, low enough that a +pathological 50-way fan-out does. + +Override: `MOCODE_MAX_TOOL_CALLS`. + +## Consecutive identical calls: 3 + +The 4th consecutive call with the same tool and same input stops the run with +`no-progress`. + +**"Consecutive" is the whole design**: `read -> edit -> read` resets the counter, +so normal edit loops are not false positives. 3 is a judgment call — a legitimate +retry rarely repeats an identical call more than twice. + +Override: `MOCODE_MAX_REPEATS`. + +## Context compaction: 60k tokens, keep 10 messages, min 16 + +Anthropic and OpenAI both publish guidance to compact as a conversation approaches +the model's window. **This is not that.** `deepseek-flash` accepts 1M tokens, so the +60k budget is a **cost** ceiling, not a window limit. + +Measured on a 50-turn synthetic session (one tool call per turn, 10k-char output — +the `readFile` truncation limit): 139,010 -> 14,011 tokens, flat after turn ~24. + +`minMessages: 16` exists so a short conversation is never compacted: early turns are +mostly intent, and folding them destroys the task. + +## Provider retries: 3 attempts, 500ms base, 8s cap + +- Retried on the same provider: 408 / 429 / 5xx / timeouts / connection errors. +- **Not** retried on the same provider: 401 / 403 / 404 — they answer identically in + 500ms, so a retry only delays the fallback. +- Delay is exponential (500ms, 1s, 2s, …) plus up to 500ms of jitter, so parallel + retries do not synchronise on the provider. + +Override: `MOCODE_PROVIDER_RETRIES`. + +## Timeouts: connect 20s, stall 120s + +Two different questions, deliberately separate: + +- **connect** bounds DNS + TCP + TLS + response headers, and is cleared the moment + headers arrive. It never bounds generation. +- **stall** bounds *silence*: it resets on every chunk. A five-minute stream that + keeps emitting bytes is healthy and must not be killed. + +Getting this wrong is what produced the "The operation timed out" mid-reply bug. + +Overrides: `MOCODE_CONNECT_TIMEOUT_MS`, `MOCODE_STALL_TIMEOUT_MS`. + +## Audit rotation: 5MB × 5 archives + +Bounded disk: at most ~30MB. Size-triggered rather than time-triggered because a CLI +process may live for minutes or a day, and clock-based cutoffs either lose the burst +or create dozens of empty files. + +## Not verified + +CrewAI's `max_iter` default could not be located in first-party source (its repo moved +to a `lib/crewai-core` layout and the module path guesses failed), so it is **not** +used as evidence for any value here. If you need it, check +`lib/crewai-core/src/crewai/` directly. diff --git a/docs/evals.md b/docs/evals.md new file mode 100644 index 0000000..12684bd --- /dev/null +++ b/docs/evals.md @@ -0,0 +1,112 @@ +# Task evals + +The only measurement in this repo that scores **end-to-end task success** rather +than unit-level correctness. Everything else (340+ unit tests) proves the pieces +work; this proves the agent can do a job. + +## Run + +```bash +bun run packages/cli/evals/run.ts # all cases +bun run packages/cli/evals/run.ts --only fix-off-by-one +``` + +Needs `DEEPSEEK_API_KEY` in `.env`. Each case gets a throwaway temp directory; the +agent's cwd is switched into it so the sandbox still applies. Results land in +`evals/results/latest.json`. + +## What a case is + +```ts +{ + id, category, + prompt, // what the user asks + files, // starting state, written to a temp dir + check, // shell command that must exit 0 +} +``` + +The check is **behavioural** — it runs code and asserts on results, not on the +presence of a string. `rename-across-files` would otherwise pass on a no-op. + +## The meta-test is the important one + +`evals/cases.test.ts` asserts, for every case, that **the check fails in the initial +state**. A case that already passes measures nothing but a false success rate. + +This caught a real defect while the suite was being written: `fix-async-ordering` +initially passed because the microtasks settled before the assertion ran, so the +"broken" code produced the right answer. The fix was to snapshot the array before +the promises resolve. + +## Latest run + +20 cases: the ten single-file ones below, plus ten harder ones (multi-file +propagation, a regression on a neighbouring test, an O(n²)→budget rewrite, a test +the agent is forbidden to edit, immutability, error contract, state cycle, +multi-constraint, cascading order-of-operations bug). + +**20/20 (100%)**, average 6 turns · 8 tool calls · ~15.5k tokens. +Heaviest case: `dedupe-into-helper`, 12 turns / 49k tokens / 26 s. + +A second independent run on a different machine landed at the same 20/20 with +7 turns · 9 tools · ~18.4k tokens per case. The spread (15.5k → 18.4k) is normal +run-to-run variance for the same model — which is why the report stores per-case +numbers rather than a single average. + +The first ten (model `deepseek-flash`, max 15 turns per case): + +| case | ok | turns | tools | tokens | time | +|---|---|---|---|---|---| +| fix-off-by-one | ✅ | 4 | 3 | 6,269 | 6.1 s | +| fix-missing-await | ✅ | 7 | 9 | 18,346 | 15.8 s | +| implement-stub | ✅ | 5 | 6 | 9,910 | 6.3 s | +| add-validation | ✅ | 5 | 6 | 11,792 | 10.4 s | +| fix-json-crash | ✅ | 5 | 6 | 9,332 | 5.9 s | +| rename-across-files | ✅ | 6 | 10 | 12,599 | 7.4 s | +| dedupe-into-helper | ✅ | 12 | 16 | 49,128 | 26.4 s | +| add-unit-test | ✅ | 5 | 7 | 9,723 | 5.1 s | +| fix-async-ordering | ✅ | 11 | 12 | 33,271 | 18.9 s | +| add-cli-flag | ✅ | 5 | 5 | 9,152 | 5.8 s | + +## Being honest about 100% + +Adding ten harder cases did not move the score. That is the finding, not a +failure to report: **this class of micro-task does not discriminate** for a +current-generation coding model. A perfect score here is a floor, not a ceiling. + +What the twenty cases still do not reproduce: + +- **repo-scale context** — a file the agent must find among hundreds, not one it + is handed by name +- **vague reports** — "checkout is sometimes wrong" instead of a failing assert +- **hidden coupling** — the fix passes the visible test but breaks a caller the + agent never read +- **long horizons** — tasks that legitimately need 40+ turns +- **the ask-don't-guess path** — where the correct action is to stop and ask + +The next lever is **realistic fixtures and ambiguity**, not more micro-cases. Ten +more single-file tasks would keep the score at 100% and teach nothing. + +One thing the harder set did buy: `fix-async-ordering` and `dedupe-into-helper` +now take 11–12 turns, so the suite exercises multi-step recovery rather than +one-shot edits. + +## Manual acceptance + +1. `bun run packages/cli/evals/run.ts` — expect 10/10 and a JSON report. +2. Deliberately break something cheap: set `MAX_TURNS=1` in `run.ts`. + Expected: cases now fail with check exit ≠ 0, proving the score tracks the + agent rather than the fixture. +3. Remove `clamp.ts` from the `add-unit-test` fixture in `EVAL_CASES` (not from a + temporary run directory) and re-run `--only add-unit-test`. Expected: the agent + reports that the module is missing instead of inventing an import — that is the + tool-error path working. Note that the prompt says "add tests for clamp()", so + the agent may instead *recreate* `clamp.ts`; treat that as an acceptable + outcome and judge the tool-error path on the cases where it does not. + +## What it costs + +The full suite is ~1 minute and a few cents. Cheap enough to run before a demo and +after any change to the loop, tools, or context policy — which is the point: an +eval you don't run is a dead document. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..810cc5d --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,319 @@ +# Troubleshooting & incident log + +Real failures hit while running MoCode locally, with the evidence that identified +them and what changed as a result. Append new entries at the bottom; keep the +symptom text exactly as the user sees it so search finds it. + +## Reproducing the provider experiment + +```bash +# from the repo root so .env is loaded +bun run packages/server/scripts/provider-failover-bench.ts +``` + +Every failure in that bench is a **real upstream failure** — no mocks. On a +mainland-China network the blocked providers are a free, reproducible fault +injector, which is why the numbers below are trustworthy. + +--- + +## INC-001 — every message fails with `Forbidden` + +**Symptom.** After `bun run dev`, sending any message in the TUI immediately fails +with `Forbidden`. Nothing in the transcript looks like a local error. + +**Root cause.** Not MoCode. The default model was `gpt-oss-120b`, which is bound to +the **Cerebras** provider. Cerebras answers `403` with Cloudflare `error code 1009` +("banned your access based on your country"). MoCode's server passes upstream +errors through verbatim: + +```ts +// packages/server/src/index.ts +app.onError((err, c) => { + if (err instanceof HTTPException) { + return c.json({ error: err.message || "Request failed" }, err.status); + } +``` + +So the provider's `403`/`Forbidden` reaches the CLI unchanged. + +**Evidence** (status codes only; keys never printed): + +| provider | status | body | +|---|---|---| +| cerebras | 403 | `error code: 1009` | +| groq | 403 | `{"error":{"message":"Forbidden"}}` | +| anthropic | 403 | `{"type":"forbidden","message":"Request not allowed"}` | +| openai | — | connect timeout | +| google | — | connect timeout | +| openrouter | 401 | `User not found` | +| **deepseek** | **200** | reachable | + +`grep -rn "403\|forbidden" packages/*/src` finds no `403` in MoCode itself — that +grep is the first thing to run, because it separates "our bug" from "upstream". + +**Fix.** Default model switched to `deepseek-flash` (see INC-004). + +**Lesson.** An error message that is a verbatim upstream string is a pass-through, +not a local failure. Check whether the string exists in the codebase before +debugging the codebase. + +--- + +## INC-002 — DeepSeek model ids no longer exist + +**Symptom.** After wiring DeepSeek, requests fail even though the key is valid. + +**Root cause.** The catalog used the V3-era ids `deepseek-chat` and +`deepseek-reasoner`. The live `/models` endpoint returns only: + +``` +deepseek-flash DeepSeek-V4.1-Flash 1M context, tool calls + vision +deepseek-v4-pro DeepSeek-V4-Pro +``` + +**Fix.** Renamed in `packages/shared/src/models.ts`, pricing re-read from +`api-docs.deepseek.com/quick_start/pricing`, and both ids added to +`packages/server/scripts/test-providers.ts`. + +**Pricing caveat.** DeepSeek bills four rates (cache hit/miss × peak/off-peak). +`ModelPricing` only has two fields, so entries use **peak cache-miss input + peak +output** — deliberately over- rather than under-charging. Modelling the four-rate +case is still open work. + +**Lesson.** Model ids are not stable API surface. `test-providers.ts` exists to +catch exactly this; it only helps if it covers the provider you just added. + +--- + +## INC-003 — OpenRouter key is dead + +**Symptom.** `openai/gpt-oss-120b:free` fails with `401 User not found`. + +**Root cause.** Key revoked or expired. (Not a code issue.) + +**Fix.** Recreate the key in the OpenRouter dashboard. Until then that catalog +entry is unusable and `/models` lists a broken option. + +**Lesson.** A catalog entry that can never succeed is worse than a missing one — +users blame the tool. Provider keys should be validated at startup, not first use. + +--- + +## INC-004 — default model pinned to a single provider + +**Symptom.** The same `403` for every user regardless of which other models exist. + +**Root cause.** `DEFAULT_CHAT_MODEL_ID` was a compile-time constant bound to one +provider, and `resolveChatModel` has no fallback: one unreachable provider means +100% of turns fail. + +**Fix.** Default moved to `deepseek-flash`; README (both languages) documents why. + +**Measured blast radius** — 15 catalog models, one tool-calling request each: + +| policy | reachable | +|---|---| +| no fallback (single provider, hard-bound) | **2/15 (13%)** | +| fallback to `deepseek-flash` → `deepseek-v4-pro` | **15/15 (100%)** | + +Failure latency is bimodal, which drives the failover design: + +- **8 fast failures** (`403`/`401`, 0.2–0.9 s) — cheap to fall through +- **5 slow failures** (connect hang, full 12 s timeout) — expensive + +**Lesson.** Changing the default is a patch, not a fix. A per-attempt timeout is +mandatory before a fallback chain is worth having: without it one hung provider +delays failover by the full timeout budget. And a silent downgrade +(`claude-opus` → `deepseek-flash`) changes capability, so the transcript must +record the downgrade rather than pretend the request succeeded as asked. + +--- + +## INC-005 — `.env` breaks under shell sourcing + +**Symptom.** `source .env` prints `JWT_SECRET: command not found`. + +**Root cause.** `JWT_SECRET =value` — whitespace before `=`. Bun's loader tolerates +it; POSIX shells parse the line as a command. + +**Fix.** Write `KEY=value` with no spaces. + +**Lesson.** An env file that Bun accepts but a shell rejects hides failures in +every script that reads it the other way. Pick one format and enforce it. + +--- + +## INC-006 — failover was measured but not implemented + +**Symptom.** After INC-004 the default model changed, but every non-DeepSeek model +still failed permanently. The bench could *describe* the fix without the product +having it. + +**Root cause.** Two separate gaps: + +1. `chatFetch` rethrew a bare `Error(message)`, **dropping the HTTP status**, so no + caller could tell a retryable 403 from a permanent 400. +2. There was no chain at all — one model, one provider, one attempt. + +**Fix.** + +- `packages/cli/src/lib/provider-chain.ts` — chain building, failure classification, + retry orchestration. Pure, no I/O. +- `packages/cli/src/lib/chat-fetch.ts` — SaaS transport retries the next model by + rewriting only the `model` field of the request body. +- `packages/cli/src/lib/local-chat-transport.ts` — BYOK retries **before the first + UI chunk arrives**; once a chunk is emitted the turn is visible and is never replayed. +- Downgrade is explicit: `{color.error}` segment in the status line plus a toast. + +**Opt-in by design.** Failover only runs when `MOCODE_ALLOW_MODEL_FAILOVER=1`: retrying on +another provider ships the user's code to a vendor they did not choose, and that is a +decision, not a default. The chain is also `deepseek-flash` only — `deepseek-v4-pro` +costs roughly 4× and is never auto-escalated. + +**Measured end-to-end** (real requests, wall clock includes the failed attempt): + +| primary asked | answered by | total | +|---|---|---| +| `claude-sonnet-4-6` | `deepseek-flash` (downgraded) | 1.3 s | +| `gpt-oss-120b` | `deepseek-flash` (downgraded) | 1.1 s | +| `deepseek-flash` | `deepseek-flash` (no downgrade) | 1.0 s | + +Failover overhead on a fast-failing provider is ~0.1–0.3 s, because a 403 returns +before any token is generated. + +**What is deliberately NOT retried:** + +| case | retried | why | +|---|---|---| +| 401 / 403 / 429 / 5xx | yes | another provider may serve the request | +| 400 | no | the request body is wrong — retrying hides our own bug | +| user abort (Esc) | no | a cancellation must not be "helpfully" undone | +| timeout | yes | a hung provider should not end the turn | + +**Lesson.** Retrying is only safe if you can tell *when the work started*. Here the +boundary is the first UI chunk: before it, nothing has been rendered or billed; after +it, a retry would duplicate both. + +--- + +## INC-007 — failover never fired: the error was hiding inside a 200 + +**Symptom.** With everything from INC-006 in place, selecting a blocked model still +produced a bare `Forbidden` — no retry, no downgrade toast, no `fallback` segment. + +**Root cause.** The failover trigger was `!response.ok`, and the server answers +**200** for every chat request: it opens an SSE stream, then the upstream provider +failure arrives as an `error` event *inside that stream*. A status-code check +therefore saw success and returned the broken stream untouched. + +Unit tests passed because they asserted on HTTP statuses; the real transport never +produces a non-2xx status for a provider failure. The test was measuring the wrong +thing. + +**Fix.** `chat-fetch.ts` now peeks at the first events of a 200 stream before +returning it: + +- an `error` event within the first `PEEK_MAX_CHUNKS` events → treat as a failed + attempt and fail over +- a healthy event (`text-delta`, `tool-input-start`, `finish`, …) → stop peeking + and replay every buffered byte into a reconstructed `Response` + +Nothing is dropped: the buffered chunks are re-emitted first, and the remaining +reader is pumped into a new `ReadableStream` with the original headers (minus +`content-length` / `content-encoding`). + +**Lesson.** "The request succeeded" and "the provider worked" are different +questions once you stream. Any retry policy keyed on HTTP status is wrong for an +SSE endpoint whose errors travel in-band. + +--- + +## INC-008 — fix landed, then the agent timed out mid-reply + +**Symptom.** After INC-007 the failover toast and the `fallback → deepseek-flash` +segment appeared correctly, the loop ran (status line showed `turn 12/24`), and then +the reply was replaced with `The operation timed out.` + +**Root cause.** The per-attempt deadline was passed straight to `fetch` as an abort +signal, so it kept ticking for the **whole stream**, not just the connection. Any +turn that streamed longer than `attemptTimeoutMs` (20 s) was killed mid-sentence — +including healthy turns that had already emitted dozens of tokens. + +The failover logic itself was fine; the deadline was scoped wrong. + +**Fix.** The deadline now covers *connect + first useful event* only: + +- a cancellable `AbortController` + `setTimeout` replaces `AbortSignal.timeout` +- once a healthy event is seen, `clearTimeout` releases the stream to run as long + as it needs +- if the deadline fires before that, it is re-thrown as a `TimeoutError` so + `classifyFailover` still treats it as retryable +- a genuine user abort (`init.signal`) is checked first and never retried + +**Regression test.** A stream that emits its first event immediately and its last +chunk 150 ms later, with `attemptTimeoutMs: 50`, must still deliver the late chunk. + +**Lesson.** A timeout has to name *what it bounds*. "Request timed out" is +meaningless for a streaming API; the only useful deadlines are "time to first +byte" and "time between tokens". + +--- + +## INC-009 — every sandboxed shell died with SIGABRT and no stderr + +**Symptom.** On the first machine where seatbelt actually worked, *every* bash call +returned empty output with `exitCode: null`, `signalCode: SIGABRT`, and an empty +stderr. The capability probe itself passed, so the sandbox looked available. + +From the agent's side this was the worst possible failure: a tool that returns +`{stdout: "", stderr: ""}` and no error teaches the model to invent output. + +**Diagnosis.** `SIGABRT` with no stderr is not a policy denial — a denial exits +non-zero and says `Operation not permitted`. The bisect isolated the variables: + +| # | profile | env | result | +|---|---|---|---| +| 1 | trivial | inherited | ok | +| 2 | trivial | allowlisted | ok | +| 3 | full | inherited | ok | +| 4 | full | allowlisted | ok | + +Both inputs were innocent, which pointed at the *spawn*, not the profile. + +**Root cause.** `spawnPlan` returned the bare name `"sandbox-exec"`. +`Bun.spawn` resolves a bare name against the env it is handed — and we hand it a +deliberately minimal allowlist. Switching to `/usr/bin/sandbox-exec` fixed every +variant. The exact reason a resolution miss surfaces as `SIGABRT` rather than +`ENOENT` is **not explained**; the empirical evidence is that the path was the only +changed variable, and the failure disappeared. + +**Fix.** Absolute paths for sandbox binaries, plus two guards: + +- a spawn failure now throws `Shell could not start ( sandbox): …` instead + of returning an empty result +- a shell killed by a signal now throws `Shell terminated by — the + sandbox failed to apply` instead of returning empty output + +**Verified on macOS** (seatbelt active): + +``` +head -1 /etc/passwd → exit 1, "head: /etc/passwd: Operation not permitted" +echo $MOCODE_DOCTOR_CANARY → "absent" +``` + +**Lesson.** "No output" is not a diagnosis. Two independent failure modes produced +identical empty results — a broken sandbox and a working sandbox with a missing +variable — and only exit code, signal, and stderr told them apart. Any tool that can +return empty should be able to say *why* it was empty. + +--- + +## Open follow-ups + +1. **Per-attempt timeout** is wired in the SaaS path only; BYOK still relies on the + provider SDK's own deadline. +2. **Four-rate DeepSeek pricing** (cache × peak) instead of the two-field model. +3. **Downgrade is per-turn state**, not persisted into session history — reopening a + session loses the record that a fallback answered. + diff --git a/packages/cli/evals/cases.test.ts b/packages/cli/evals/cases.test.ts new file mode 100644 index 0000000..2beaa60 --- /dev/null +++ b/packages/cli/evals/cases.test.ts @@ -0,0 +1,75 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { EVAL_CASES, type EvalCase } from "./cases"; + +const created: string[] = []; + +/** Write a case's starting state into a throwaway directory. */ +function materialize(evalCase: EvalCase): string { + const dir = mkdtempSync(join(tmpdir(), `mocode-eval-${evalCase.id}-`)); + created.push(dir); + + for (const [path, content] of Object.entries(evalCase.files)) { + const target = join(dir, path); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, content); + } + + return dir; +} + +function runCheck(dir: string, command: string) { + return Bun.spawnSync(["bash", "-c", command], { cwd: dir, stdout: "pipe", stderr: "pipe" }); +} + +afterAll(() => { + for (const dir of created) rmSync(dir, { recursive: true, force: true }); +}); + +describe("eval case registry", () => { + test("ids are unique", () => { + const ids = EVAL_CASES.map((c) => c.id); + expect(new Set(ids).size).toBe(ids.length); + }); + + test("every case declares a prompt, files and a check", () => { + for (const evalCase of EVAL_CASES) { + expect(evalCase.prompt.length).toBeGreaterThan(20); + expect(Object.keys(evalCase.files).length).toBeGreaterThan(0); + expect(evalCase.check.length).toBeGreaterThan(0); + } + }); + + test("each case ships a verifier, not just a source file", () => { + for (const evalCase of EVAL_CASES) { + expect(Object.keys(evalCase.files)).toContain("verify.ts"); + } + }); +}); + +describe("every case actually measures something", () => { + // This is the most important test in the suite: a case whose check already + // passes before the agent runs contributes nothing but a false success rate. + for (const evalCase of EVAL_CASES) { + test(`${evalCase.id} fails in its initial state`, () => { + const dir = materialize(evalCase); + const result = runCheck(dir, evalCase.check); + + expect(result.exitCode).not.toBe(0); + }); + } + + // A check that passes on a syntactically broken project would be scoring + // "did not crash" rather than "did the work". + for (const evalCase of EVAL_CASES) { + test(`${evalCase.id} check is runnable (not a missing binary)`, () => { + const dir = materialize(evalCase); + const result = runCheck(dir, evalCase.check); + const output = `${result.stdout.toString()}${result.stderr.toString()}`; + + expect(output).not.toContain("command not found"); + }); + } +}); diff --git a/packages/cli/evals/cases.ts b/packages/cli/evals/cases.ts new file mode 100644 index 0000000..1e84835 --- /dev/null +++ b/packages/cli/evals/cases.ts @@ -0,0 +1,588 @@ +/** + * Task eval cases. + * + * Each case is a tiny, self-contained project plus a `check` command that must + * exit 0 once the task is done. Two properties matter more than the count: + * + * 1. **The check must FAIL before the agent touches anything.** A case whose + * check already passes measures nothing. `cases.test.ts` enforces this. + * 2. **The check must be behavioural.** It runs code and asserts on results, not + * on the presence of a string in a file — otherwise "rename the variable" + * would pass. + * + * Cases are deliberately small: a task that needs 40 turns tests the harness as + * much as the agent, and hides which capability actually failed. + */ + +export type EvalCase = { + id: string; + /** What kind of work this exercises — used in the report. */ + category: "fix" | "feature" | "refactor" | "test"; + prompt: string; + /** Starting state written into a temp dir. */ + files: Record; + /** Shell command that must exit 0 when the task is complete. */ + check: string; + /** + * Files restored from `files` before the check runs. + * + * Defaults to `["verify.ts"]`: a case that passes by rewriting its own verifier + * measures nothing. Files the agent is *supposed* to edit must not be listed. + */ + protectedFiles?: string[]; +}; + +const verify = `import assert from "node:assert";`; + +export const EVAL_CASES: EvalCase[] = [ + { + id: "fix-off-by-one", + category: "fix", + prompt: "sumTo(5) should return 15 but returns 10. Fix the loop in sum.ts.", + files: { + "sum.ts": `export function sumTo(n: number): number { + let total = 0; + for (let i = 1; i < n; i++) { + total += i; + } + return total; +} +`, + "verify.ts": `${verify} +import { sumTo } from "./sum"; +assert.equal(sumTo(5), 15); +assert.equal(sumTo(1), 1); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "fix-missing-await", + category: "fix", + prompt: "loadUser returns a Promise instead of the user. Find and fix the missing await.", + files: { + "db.ts": `export async function fetchUser(): Promise<{ name: string }> { + return { name: "ada" }; +} +`, + "index.ts": `import { fetchUser } from "./db"; + +export async function loadUser() { + const user = fetchUser(); + return user.name; +} +`, + "verify.ts": `${verify} +import { loadUser } from "./index"; +assert.deepEqual(await loadUser(), "ada"); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "implement-stub", + category: "feature", + prompt: "parseSize('1.5kb') should return 1536 (bytes). The function is a stub — implement it. Supported units: b, kb, mb.", + files: { + "size.ts": `export function parseSize(input: string): number { + throw new Error("not implemented"); +} +`, + "verify.ts": `${verify} +import { parseSize } from "./size"; +assert.equal(parseSize("512b"), 512); +assert.equal(parseSize("1.5kb"), 1536); +assert.equal(parseSize("2mb"), 2097152); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "add-validation", + category: "feature", + prompt: "withdraw() must throw 'amount must be positive' when given a negative amount. It currently accepts it.", + files: { + "account.ts": `export function withdraw(balance: number, amount: number): number { + return balance - amount; +} +`, + "verify.ts": `${verify} +import { withdraw } from "./account"; +assert.equal(withdraw(100, 30), 70); +assert.throws(() => withdraw(100, -5), /amount must be positive/); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "fix-json-crash", + category: "fix", + prompt: "parseConfig throws on invalid JSON and takes the app down. It should return null instead.", + files: { + "config.ts": `export function parseConfig(raw: string): { port: number } | null { + return JSON.parse(raw); +} +`, + "verify.ts": `${verify} +import { parseConfig } from "./config"; +assert.deepEqual(parseConfig('{"port":3000}'), { port: 3000 }); +assert.equal(parseConfig("{not json"), null); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "rename-across-files", + category: "refactor", + prompt: "Rename formatDate to formatIsoDate everywhere (definition and all call sites). Behaviour must not change.", + files: { + "date.ts": `export function formatDate(d: Date): string { + return d.toISOString().slice(0, 10); +} +`, + "report.ts": `import { formatDate } from "./date"; + +export function reportTitle(d: Date): string { + return "Report " + formatDate(d); +} +`, + "verify.ts": `${verify} +import { formatIsoDate } from "./date"; +import { reportTitle } from "./report"; +assert.equal(formatIsoDate(new Date("2026-03-04T00:00:00Z")), "2026-03-04"); +assert.equal(reportTitle(new Date("2026-03-04T00:00:00Z")), "Report 2026-03-04"); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "dedupe-into-helper", + category: "refactor", + prompt: "slug() is duplicated in two files with slightly different behaviour. Extract one shared slug() into slug.ts and make both files use it. Both call sites must keep working.", + files: { + "posts.ts": `export function slug(title: string): string { + return title.toLowerCase().replace(/[^a-z0-9]+/g, "-"); +} +`, + "tags.ts": `export function slug(title: string): string { + return title.toLowerCase().replace(/ /g, "-"); +} +`, + "verify.ts": `${verify} +import { existsSync, readFileSync } from "node:fs"; +import { slug as postSlug } from "./posts"; +import { slug as tagSlug } from "./tags"; + +// Behaviour must not change... +assert.equal(postSlug("Hello World!"), "hello-world"); +assert.equal(tagSlug("Hello World"), "hello-world"); + +// ...and the duplication must actually be gone: an alias re-export leaves two +// implementations behind while still passing the assertions above. +assert.ok(existsSync("slug.ts"), "slug.ts must exist"); +for (const file of ["posts.ts", "tags.ts"]) { + const source = readFileSync(file, "utf8"); + assert.ok(/from\s+["']\.\/slug["']/.test(source), file + " must import the shared slug"); + assert.ok(!/function\s+slug\s*\(/.test(source), file + " must not define its own slug()"); +} +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "add-unit-test", + category: "test", + prompt: "Add tests for clamp() in clamp.test.ts. Cover the lower bound, the upper bound, and the in-range case. `bun test` must pass.", + files: { + "clamp.ts": `export function clamp(value: number, min: number, max: number): number { + return Math.min(Math.max(value, min), max); +} +`, + "clamp.test.ts": `import { test, expect } from "bun:test"; +`, + "verify.ts": `${verify} +import { readFileSync, writeFileSync } from "node:fs"; + +const original = readFileSync("clamp.ts", "utf8"); + +/** Run the submitted suite and return its exit code. */ +function runTests(): number { + return Bun.spawnSync(["bun", "test", "clamp.test.ts"], { stdout: "pipe", stderr: "pipe" }).exitCode ?? 1; +} + +try { + // 1. The submitted tests must pass against the correct implementation. + assert.equal(runTests(), 0, "submitted tests must pass against the correct clamp()"); + + // 2. They must FAIL when clamp() is broken. Counting \`test(\` occurrences is not + // coverage: three empty callbacks pass every check while testing nothing. + const broken = [ + "export function clamp(value: number, min: number, max: number): number {", + " return value; // ignores both bounds", + "}", + "", + ].join("\n"); + + writeFileSync("clamp.ts", broken); + assert.notEqual(runTests(), 0, "submitted tests do not detect a broken clamp()"); +} finally { + writeFileSync("clamp.ts", original); +} + +console.log("ok"); +`, + }, + check: "bun test && bun verify.ts", + }, + { + id: "fix-async-ordering", + category: "fix", + prompt: "collect() returns an empty array because the promises are never awaited. Fix it so the results are collected in order.", + files: { + "collect.ts": `export async function collect(items: string[]): Promise { + const out: string[] = []; + for (const item of items) { + Promise.resolve(item.toUpperCase()).then((value) => out.push(value)); + } + // Snapshot taken before the promises settle, so the caller always sees []. + return [...out]; +} +`, + "verify.ts": `${verify} +import { collect } from "./collect"; +assert.deepEqual(await collect(["a", "b", "c"]), ["A", "B", "C"]); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "add-cli-flag", + category: "feature", + prompt: "cli.ts should support `--upper` to uppercase its output. Without the flag it must behave exactly as before.", + files: { + "cli.ts": `const args = process.argv.slice(2); +const text = args.filter((a) => !a.startsWith("--")).join(" "); +console.log(text); +`, + "verify.ts": `${verify} +import { spawnSync } from "node:child_process"; +const plain = spawnSync("bun", ["cli.ts", "hello"], { encoding: "utf8" }); +const upper = spawnSync("bun", ["cli.ts", "--upper", "hello"], { encoding: "utf8" }); +assert.equal(plain.stdout.trim(), "hello"); +assert.equal(upper.stdout.trim(), "HELLO"); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + + // ------------------------------------------------------------------------- + // Hard cases. The ten above are single-file with a visible failure message; + // these add multi-file propagation, behavioural regressions, a performance + // budget, and a test the agent is forbidden to edit. + // ------------------------------------------------------------------------- + { + id: "hard-barrel-rename", + category: "refactor", + prompt: + "Rename `formatMoney` to `formatCurrency` everywhere: its definition, the barrel re-export in index.ts, and both call sites. Behaviour must not change.", + files: { + "money.ts": `export function formatMoney(cents: number): string { + return "$" + (cents / 100).toFixed(2); +} +`, + "index.ts": `export { formatMoney } from "./money"; +`, + "cart.ts": `import { formatMoney } from "./index"; +export const cartLabel = (c: number) => "Cart: " + formatMoney(c); +`, + "invoice.ts": `import { formatMoney } from "./index"; +export const invoiceLabel = (c: number) => "Invoice: " + formatMoney(c); +`, + "verify.ts": `${verify} +import { formatCurrency } from "./money"; +import { cartLabel } from "./cart"; +import { invoiceLabel } from "./invoice"; +assert.equal(formatCurrency(1234), "$12.34"); +assert.equal(cartLabel(500), "Cart: $5.00"); +assert.equal(invoiceLabel(99), "Invoice: $0.99"); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-schema-migration", + category: "refactor", + prompt: + "The User type renames `userName` to `displayName`. Update the type, the factory, and both consumers so the program still compiles and prints correctly.", + files: { + "types.ts": `export type User = { id: string; userName: string }; +`, + "factory.ts": `import type { User } from "./types"; +export function makeUser(id: string): User { + return { id, userName: "anon-" + id }; +} +`, + "greet.ts": `import type { User } from "./types"; +export function greet(user: User): string { + return "Hello " + user.userName; +} +`, + "verify.ts": `${verify} +import { readFileSync } from "node:fs"; +import { makeUser } from "./factory"; +import { greet } from "./greet"; + +const user = makeUser("7"); +assert.equal(greet(user), "Hello anon-7"); +assert.equal((user as unknown as { displayName: string }).displayName, "anon-7"); + +// The migration must remove the old field, not bolt the new one on beside it. +for (const file of ["types.ts", "factory.ts", "greet.ts"]) { + assert.ok( + !readFileSync(file, "utf8").includes("userName"), + file + " still references the obsolete userName field", + ); +} +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-regression-neighbour", + category: "fix", + prompt: + "`average([])` must return 0 instead of NaN. Do not change `sum` or `average`'s behaviour for non-empty input — there is a neighbouring test that must keep passing.", + files: { + "stats.ts": `export function sum(values: number[]): number { + return values.reduce((a, b) => a + b, 0); +} + +export function average(values: number[]): number { + return sum(values) / values.length; +} +`, + "neighbour.test.ts": `import { test, expect } from "bun:test"; +import { sum, average } from "./stats"; + +test("sum of non-empty", () => expect(sum([1, 2, 3])).toBe(6)); +test("average of non-empty", () => expect(average([2, 4])).toBe(3)); +`, + "verify.ts": `${verify} +import { average } from "./stats"; +assert.equal(average([]), 0); +assert.equal(average([2, 4]), 3); +console.log("ok"); +`, + }, + check: "bun test neighbour.test.ts && bun verify.ts", + }, + { + id: "hard-perf-budget", + category: "fix", + prompt: + "hasDuplicate is O(n^2) and times out on large inputs. Rewrite it so it handles 40,000 items within the budget in verify.ts, without changing its return values.", + files: { + "dedupe.ts": `export function hasDuplicate(values: number[]): boolean { + for (let i = 0; i < values.length; i++) { + for (let j = 0; j < values.length; j++) { + if (i !== j && values[i] === values[j]) return true; + } + } + return false; +} +`, + "verify.ts": `import assert from "node:assert"; +import { hasDuplicate } from "./dedupe"; + +assert.equal(hasDuplicate([1, 2, 3]), false); +assert.equal(hasDuplicate([1, 2, 1]), true); + +const big = Array.from({ length: 50_000 }, (_, i) => i); +const started = Date.now(); +const result = hasDuplicate(big); +const elapsed = Date.now() - started; + +assert.equal(result, false); +assert.ok(elapsed < 800, "took " + elapsed + "ms, budget 800ms"); +console.log("ok in " + elapsed + "ms"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-test-driven", + category: "test", + prompt: + "csv.test.ts currently fails. Make it pass by fixing csv.ts — you may NOT edit the test file; a pristine copy is kept at csv.test.ts.orig and will be compared.", + files: { + "csv.ts": `export function parseCsv(line: string): string[] { + return line.split(","); +} +`, + "csv.test.ts": `import { test, expect } from "bun:test"; +import { parseCsv } from "./csv"; + +test("trims fields", () => expect(parseCsv("a, b ,c")).toEqual(["a", "b", "c"])); +test("drops empty fields", () => expect(parseCsv("a,,b")).toEqual(["a", "b"])); +test("empty input", () => expect(parseCsv("")).toEqual([])); +`, + "csv.test.ts.orig": `import { test, expect } from "bun:test"; +import { parseCsv } from "./csv"; + +test("trims fields", () => expect(parseCsv("a, b ,c")).toEqual(["a", "b", "c"])); +test("drops empty fields", () => expect(parseCsv("a,,b")).toEqual(["a", "b"])); +test("empty input", () => expect(parseCsv("")).toEqual([])); +`, + "verify.ts": `${verify} +import { readFileSync } from "node:fs"; +const current = readFileSync("csv.test.ts", "utf8"); +const pristine = readFileSync("csv.test.ts.orig", "utf8"); +assert.equal(current, pristine, "the test file was modified"); +console.log("ok"); +`, + }, + // The test and its pristine copy are restored before scoring, so an agent edit + // cannot make them agree. + protectedFiles: ["verify.ts", "csv.test.ts", "csv.test.ts.orig"], + check: "bun test csv.test.ts && bun verify.ts", + }, + { + id: "hard-immutability", + category: "fix", + prompt: + "sortByScore mutates the caller's array, which breaks a caller that reuses it. Return a sorted copy instead; the original array must keep its order.", + files: { + "rank.ts": `export function sortByScore(items: { score: number }[]): { score: number }[] { + return items.sort((a, b) => b.score - a.score); +} +`, + "verify.ts": `${verify} +import { sortByScore } from "./rank"; +const original = [{ score: 1 }, { score: 3 }, { score: 2 }]; +const sorted = sortByScore(original); +assert.deepEqual(sorted.map((i) => i.score), [3, 2, 1]); +assert.deepEqual(original.map((i) => i.score), [1, 3, 2], "input was mutated"); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-error-contract", + category: "feature", + prompt: + "fetchPrice must throw an Error whose message is exactly `price unavailable for ` when the symbol is unknown, and re-throw nothing else. Currently it swallows the failure.", + files: { + "price.ts": `const PRICES: Record = { BTC: 100, ETH: 50 }; + +export async function fetchPrice(symbol: string): Promise { + try { + return PRICES[symbol] as number; + } catch { + return 0; + } +} +`, + "verify.ts": `${verify} +import { fetchPrice } from "./price"; +assert.equal(await fetchPrice("BTC"), 100); +await assert.rejects(() => fetchPrice("DOGE"), /price unavailable for DOGE/); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-state-cycle", + category: "fix", + prompt: + "nextState cycles STOP -> GO -> SLOW -> STOP. The current implementation returns STOP forever after the first call. Fix the transition table.", + files: { + "light.ts": `export type Light = "STOP" | "GO" | "SLOW"; + +const TRANSITIONS: Record = { + STOP: "STOP", + GO: "STOP", + SLOW: "STOP", +}; + +export function nextState(state: Light): Light { + return TRANSITIONS[state]; +} +`, + "verify.ts": `${verify} +import { nextState } from "./light"; +assert.equal(nextState("STOP"), "GO"); +assert.equal(nextState("GO"), "SLOW"); +assert.equal(nextState("SLOW"), "STOP"); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-multi-constraint", + category: "feature", + prompt: + "normalizeTag must satisfy all four rules: trim whitespace, lowercase, replace spaces with '-', and return null for an empty result. It currently does none of them.", + files: { + "tag.ts": `export function normalizeTag(input: string): string | null { + return input; +} +`, + "verify.ts": `${verify} +import { normalizeTag } from "./tag"; +assert.equal(normalizeTag(" Hello World "), "hello-world"); +assert.equal(normalizeTag("A B C"), "a-b-c"); +assert.equal(normalizeTag(" "), null); +assert.equal(normalizeTag(""), null); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-cascade-bug", + category: "fix", + prompt: + "checkoutTotal returns the wrong number. The expected total is subtotal(prices) + tax(subtotal(prices)) + shipping(subtotal(prices)). Fix checkoutTotal. The three helper functions are correct — do not change them.", + files: { + "checkout.ts": `export function subtotal(prices: number[]): number { + return prices.reduce((a, b) => a + b, 0); +} +export function tax(amount: number): number { + return Math.round(amount * 0.1); +} +export function shipping(amount: number): number { + return amount >= 100 ? 0 : 5; +} + +export function checkoutTotal(prices: number[]): number { + const base = subtotal(prices); + const discounted = base - tax(base); + return discounted + shipping(base); +} +`, + "verify.ts": `${verify} +import { checkoutTotal } from "./checkout"; +// 120 -> tax 12 -> 132, free shipping (base >= 100) +assert.equal(checkoutTotal([100, 20]), 132); +// 50 -> tax 5 -> 55 + 5 shipping +assert.equal(checkoutTotal([50]), 60); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, +]; diff --git a/packages/cli/evals/check-runner.ts b/packages/cli/evals/check-runner.ts new file mode 100644 index 0000000..90057ae --- /dev/null +++ b/packages/cli/evals/check-runner.ts @@ -0,0 +1,78 @@ +/** + * Bounded, credential-free check runner for evals. + * + * Two problems this solves, both found in review: + * + * 1. **A hanging check hangs the whole run.** `Bun.spawnSync` has no timeout, so + * one stuck command (e.g. a test that binds a port) blocks every remaining case + * and the report never gets written. + * 2. **Checks inherited the full environment**, including provider credentials — + * and `verify.ts` is agent-writable, so a generated check could print them. + * + * The runner bounds each check, reports a timeout as a failure, and runs with the + * same allowlisted environment the sandboxed tools get. + */ +import { buildSandboxEnv } from "../src/lib/permission-rules"; + +export type CheckResult = { + ok: boolean; + exitCode: number | null; + signal: string | number | null; + output: string; + timedOut: boolean; +}; + +/** Per-check wall clock budget. Generous enough for `bun test`, short enough to bound a run. */ +export const DEFAULT_CHECK_TIMEOUT_MS = 120_000; + +export type CheckOptions = { + timeoutMs?: number; + /** Overrides the default allowlisted env (tests pass a minimal set). */ + env?: Record; +}; + +/** + * Run `command` in `cwd`, bounded by a deadline. + * + * On timeout the child is killed and the result is reported as a failure, so the + * caller can record it and continue with the next case. + */ +export async function runCheck( + cwd: string, + command: string, + options: CheckOptions = {}, +): Promise { + const timeoutMs = options.timeoutMs ?? DEFAULT_CHECK_TIMEOUT_MS; + const env = (options.env ?? buildSandboxEnv()) as Record; + + const proc = Bun.spawn(["bash", "-c", command], { + cwd, + env, + stdout: "pipe", + stderr: "pipe", + }); + + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + proc.kill(); + }, timeoutMs); + + try { + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + const exitCode = await proc.exited; + + return { + ok: !timedOut && exitCode === 0, + exitCode, + signal: proc.signalCode ?? null, + output: `${stdout}${stderr}`.trim(), + timedOut, + }; + } finally { + clearTimeout(timer); + } +} diff --git a/packages/cli/evals/results/latest.json b/packages/cli/evals/results/latest.json new file mode 100644 index 0000000..2f8ffd5 --- /dev/null +++ b/packages/cli/evals/results/latest.json @@ -0,0 +1,255 @@ +{ + "ranAt": "2026-11-11T14:16:04.430Z", + "model": "deepseek-flash", + "maxTurns": 15, + "successRate": 1, + "totals": { + "cases": 20, + "passed": 20, + "avgTurns": 7, + "avgToolCalls": 9, + "avgTokens": 18396 + }, + "results": [ + { + "id": "fix-off-by-one", + "category": "fix", + "ok": true, + "turns": 3, + "toolCalls": 2, + "tokens": 4707, + "ms": 3224, + "checkExit": 0, + "checkTail": "ok", + "summary": "Fixed `sum.ts`: changed the loop bound from `i < n` to `i <= n` so `sumTo(5)` now returns 15." + }, + { + "id": "fix-missing-await", + "category": "fix", + "ok": true, + "turns": 6, + "toolCalls": 8, + "tokens": 11677, + "ms": 8540, + "checkExit": 0, + "checkTail": "ok", + "summary": "Added the missing `await` on `fetchUser()` in `index.ts`, so `loadUser()` now returns `user.name` instead of a Promise." + }, + { + "id": "implement-stub", + "category": "feature", + "ok": true, + "turns": 7, + "toolCalls": 8, + "tokens": 16335, + "ms": 10569, + "checkExit": 0, + "checkTail": "ok", + "summary": "Implemented `parseSize` in `size.ts` (binary units, case/whitespace tolerant, rounds to whole bytes); outputs verified: " + }, + { + "id": "add-validation", + "category": "feature", + "ok": true, + "turns": 8, + "toolCalls": 9, + "tokens": 20978, + "ms": 13436, + "checkExit": 0, + "checkTail": "ok", + "summary": "Done — `withdraw()` now throws `amount must be positive` for negative amounts (verified: existing case still returns 70," + }, + { + "id": "fix-json-crash", + "category": "fix", + "ok": true, + "turns": 7, + "toolCalls": 8, + "tokens": 17697, + "ms": 10449, + "checkExit": 0, + "checkTail": "ok", + "summary": "`parseConfig` now wraps `JSON.parse` in a try/catch and returns `null` on invalid JSON instead of throwing." + }, + { + "id": "rename-across-files", + "category": "refactor", + "ok": true, + "turns": 7, + "toolCalls": 11, + "tokens": 15736, + "ms": 9442, + "checkExit": 0, + "checkTail": "ok", + "summary": "Renamed `formatDate` → `formatIsoDate` in its definition (`date.ts`) and all call sites (`report.ts` import + usage); no" + }, + { + "id": "dedupe-into-helper", + "category": "refactor", + "ok": true, + "turns": 13, + "toolCalls": 16, + "tokens": 62506, + "ms": 30263, + "checkExit": 0, + "checkTail": "ok", + "summary": "Done. Extracted a single `slug()` into `slug.ts` and both files now re-export it.\n\nOne thing worth flagging: `verify.ts`" + }, + { + "id": "add-unit-test", + "category": "test", + "ok": true, + "turns": 5, + "toolCalls": 7, + "tokens": 9563, + "ms": 5152, + "checkExit": 0, + "checkTail": " 3 expect() calls Ran 3 tests across 1 file. [10.00ms]", + "summary": "Added three tests to `clamp.test.ts` covering the lower bound, upper bound, and in-range cases; `bun test` passes (3/3)." + }, + { + "id": "fix-async-ordering", + "category": "fix", + "ok": true, + "turns": 7, + "toolCalls": 8, + "tokens": 16026, + "ms": 9730, + "checkExit": 0, + "checkTail": "ok", + "summary": "`collect()` now awaits its promises via `Promise.all`, so results are returned in input order instead of an empty array." + }, + { + "id": "add-cli-flag", + "category": "feature", + "ok": true, + "turns": 5, + "toolCalls": 6, + "tokens": 9231, + "ms": 6141, + "checkExit": 0, + "checkTail": "ok", + "summary": "`cli.ts` now supports `--upper` (uppercases output); default behavior is unchanged and `verify.ts` passes." + }, + { + "id": "hard-barrel-rename", + "category": "refactor", + "ok": true, + "turns": 6, + "toolCalls": 14, + "tokens": 15167, + "ms": 10378, + "checkExit": 0, + "checkTail": "ok", + "summary": "Renamed `formatMoney` → `formatCurrency` in money.ts (definition), index.ts (barrel re-export), and the imports/uses in " + }, + { + "id": "hard-schema-migration", + "category": "refactor", + "ok": true, + "turns": 12, + "toolCalls": 17, + "tokens": 47448, + "ms": 20573, + "checkExit": 0, + "checkTail": "ok", + "summary": "Renamed `userName` → `displayName` in `types.ts`, `factory.ts`, and `greet.ts` (the consumers); `verify.ts` runs success" + }, + { + "id": "hard-regression-neighbour", + "category": "fix", + "ok": true, + "turns": 6, + "toolCalls": 8, + "tokens": 12714, + "ms": 6475, + "checkExit": 0, + "checkTail": " 2 expect() calls Ran 2 tests across 1 file. [9.00ms]", + "summary": "Added an empty-array guard in `average` (`stats.ts`) so `average([])` returns `0`; `sum` is untouched and non-empty aver" + }, + { + "id": "hard-perf-budget", + "category": "fix", + "ok": true, + "turns": 10, + "toolCalls": 11, + "tokens": 33619, + "ms": 18823, + "checkExit": 0, + "checkTail": "ok in 2ms", + "summary": "Rewrote `hasDuplicate` in `dedupe.ts` as a single-pass `Set` lookup (O(n) instead of O(n²)), keeping return values ident" + }, + { + "id": "hard-test-driven", + "category": "test", + "ok": true, + "turns": 5, + "toolCalls": 7, + "tokens": 10201, + "ms": 5807, + "checkExit": 0, + "checkTail": " 3 expect() calls Ran 3 tests across 1 file. [13.00ms]", + "summary": "Fixed `csv.ts` to trim each field and drop empty ones; all 3 tests pass and the pristine test file check is ok." + }, + { + "id": "hard-immutability", + "category": "fix", + "ok": true, + "turns": 5, + "toolCalls": 5, + "tokens": 9018, + "ms": 5954, + "checkExit": 0, + "checkTail": "ok", + "summary": "`sortByScore` now sorts a shallow copy (`[...items]`), leaving the caller's array order untouched." + }, + { + "id": "hard-error-contract", + "category": "feature", + "ok": true, + "turns": 7, + "toolCalls": 8, + "tokens": 14754, + "ms": 9631, + "checkExit": 0, + "checkTail": "ok", + "summary": "Changed `fetchPrice` in `price.ts` to throw `Error(\"price unavailable for \")` on unknown symbols with no try/cat" + }, + { + "id": "hard-state-cycle", + "category": "fix", + "ok": true, + "turns": 6, + "toolCalls": 7, + "tokens": 13826, + "ms": 7961, + "checkExit": 0, + "checkTail": "ok", + "summary": "Fixed the transition table in `light.ts` to `STOP: \"GO\"`, `GO: \"SLOW\"`, `SLOW: \"STOP\"`, so `nextState` now cycles correc" + }, + { + "id": "hard-multi-constraint", + "category": "feature", + "ok": true, + "turns": 6, + "toolCalls": 7, + "tokens": 13718, + "ms": 8861, + "checkExit": 0, + "checkTail": "ok", + "summary": "`normalizeTag` in `tag.ts` now trims, lowercases, replaces whitespace runs with `-`, and returns `null` when the result " + }, + { + "id": "hard-cascade-bug", + "category": "fix", + "ok": true, + "turns": 6, + "toolCalls": 7, + "tokens": 13001, + "ms": 9219, + "checkExit": 0, + "checkTail": "ok", + "summary": "Fixed `checkoutTotal` in `checkout.ts` to add tax to the base before adding shipping (`base + tax(base) + shipping(base)" + } + ] +} \ No newline at end of file diff --git a/packages/cli/evals/run.ts b/packages/cli/evals/run.ts new file mode 100644 index 0000000..3116dc4 --- /dev/null +++ b/packages/cli/evals/run.ts @@ -0,0 +1,231 @@ +/** + * Task eval runner. + * + * Runs each case against a real model with the real tool implementations, then + * executes the case's check. This is the only measurement in the repo that scores + * *end-to-end task success* rather than unit-level correctness. + * + * Run from the repo root (needs DEEPSEEK_API_KEY in `.env`): + * bun run packages/cli/evals/run.ts + * bun run packages/cli/evals/run.ts --only fix-off-by-one + * + * Each case gets a throwaway directory; the agent's cwd is switched into it, so + * `executeLocalTool` resolves paths against the case and the sandbox still applies. + */ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { generateText, stepCountIs, tool } from "ai"; +import { createOpenAI } from "@ai-sdk/openai"; +import { Mode, getToolContracts, toolInputSchemas } from "@mocode/shared"; +import { executeLocalTool } from "../src/lib/local-tools"; +import { EVAL_CASES, type EvalCase } from "./cases"; +import { runCheck } from "./check-runner"; + +const MAX_TURNS = 15; + +const SYSTEM = [ + "You are a coding agent working inside a small project directory.", + "Use the available tools to read files, then edit them.", + "Make the smallest change that satisfies the request.", + "When the task is done, reply with a one-line summary and stop.", +].join(" "); + +function buildModel() { + const apiKey = process.env.DEEPSEEK_API_KEY; + if (!apiKey) throw new Error("DEEPSEEK_API_KEY is not set — run from the repo root"); + + return createOpenAI({ + apiKey, + baseURL: "https://api.deepseek.com/v1", + name: "deepseek", + }).chat("deepseek-flash"); +} + +/** Real tool implementations — same functions the CLI agent calls. */ +function buildTools() { + const contracts = getToolContracts(Mode.BUILD) as Record; + + return Object.fromEntries( + Object.keys(contracts) + // `task` spawns a subagent; out of scope for a single-turn eval. + .filter((name) => name !== "task") + .map((name) => [ + name, + tool({ + description: contracts[name]?.description ?? name, + inputSchema: toolInputSchemas[name as keyof typeof toolInputSchemas], + execute: async (input: unknown) => executeLocalTool(name, input, Mode.BUILD), + }), + ]), + ); +} + +function materialize(evalCase: EvalCase): string { + const dir = mkdtempSync(join(tmpdir(), `mocode-eval-${evalCase.id}-`)); + + for (const [path, content] of Object.entries(evalCase.files)) { + const target = join(dir, path); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, content); + } + + return dir; +} + +type CaseResult = { + id: string; + category: EvalCase["category"]; + ok: boolean; + turns: number; + toolCalls: number; + tokens: number; + ms: number; + checkExit: number | null; + checkTail: string; + summary: string; +}; + +async function runCase(evalCase: EvalCase, model: ReturnType): Promise { + const dir = materialize(evalCase); + const started = Date.now(); + + let turns = 0; + let toolCalls = 0; + let tokens = 0; + let summary = ""; + + const originalCwd = process.cwd(); + + try { + process.chdir(dir); + + const result = await generateText({ + model, + system: SYSTEM, + prompt: evalCase.prompt, + tools: buildTools(), + stopWhen: stepCountIs(MAX_TURNS), + maxOutputTokens: 1_500, + onStepFinish(step) { + turns += 1; + toolCalls += step.toolCalls.length; + tokens += step.usage?.totalTokens ?? 0; + }, + }); + + summary = result.text.trim().slice(0, 120); + } catch (error) { + summary = `agent error: ${error instanceof Error ? error.message.slice(0, 100) : "unknown"}`; + } finally { + process.chdir(originalCwd); + } + + // Restore verifier files from the fixture before scoring: `verify.ts` (and any + // protected test) is agent-writable, and a case that passes by rewriting its own + // verifier measures nothing. + const protectedFiles = new Set(evalCase.protectedFiles ?? ["verify.ts"]); + for (const [path, content] of Object.entries(evalCase.files)) { + if (!protectedFiles.has(path)) continue; + const target = join(dir, path); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, content); + } + + const check = await runCheck(dir, evalCase.check); + const output = check.output; + + rmSync(dir, { recursive: true, force: true }); + + return { + id: evalCase.id, + category: evalCase.category, + ok: check.ok, + turns, + toolCalls, + tokens, + ms: Date.now() - started, + checkExit: check.exitCode, + checkTail: output.split("\n").slice(-2).join(" ").slice(0, 160), + summary, + }; +} + +const onlyIndex = process.argv.indexOf("--only"); +const only = onlyIndex === -1 ? null : process.argv[onlyIndex + 1]; +const selected = only ? EVAL_CASES.filter((c) => c.id === only) : EVAL_CASES; + +if (selected.length === 0) { + console.error(`No eval case matched ${only ?? ""}`); + process.exit(2); +} + +const model = buildModel(); +const results: CaseResult[] = []; + +console.log(`Running ${selected.length} eval case(s), max ${MAX_TURNS} turns each\n`); +console.log("case".padEnd(24), "ok".padEnd(4), "turns".padEnd(7), "tools".padEnd(7), "tokens".padEnd(9), "time"); + +for (const evalCase of selected) { + process.stdout.write(`${evalCase.id.padEnd(24)} `); + const result = await runCase(evalCase, model); + results.push(result); + + console.log( + (result.ok ? "✅" : "❌").padEnd(4), + String(result.turns).padEnd(7), + String(result.toolCalls).padEnd(7), + String(result.tokens).padEnd(9), + `${(result.ms / 1000).toFixed(1)}s`, + ); + + if (!result.ok) console.log("".padEnd(25), `check exit ${result.checkExit}: ${result.checkTail}`); +} + +const passed = results.filter((r) => r.ok).length; +const avg = (pick: (r: CaseResult) => number) => + results.length === 0 ? 0 : Math.round(results.reduce((sum, r) => sum + pick(r), 0) / results.length); + +console.log("\n=== 汇总 ==="); +console.log(`任务成功率: ${passed}/${results.length} (${Math.round((passed / results.length) * 100)}%)`); +console.log(`平均轮次: ${avg((r) => r.turns)} 平均工具数: ${avg((r) => r.toolCalls)} 平均 token: ${avg((r) => r.tokens)}`); + +const byCategory = new Map(); +for (const result of results) { + const entry = byCategory.get(result.category) ?? { ok: 0, total: 0 }; + entry.total += 1; + if (result.ok) entry.ok += 1; + byCategory.set(result.category, entry); +} + +console.log("\n=== 分类 ==="); +for (const [category, entry] of byCategory) { + console.log(`${category.padEnd(10)} ${entry.ok}/${entry.total}`); +} + +const reportPath = new URL("./results/latest.json", import.meta.url); +mkdirSync(dirname(reportPath.pathname), { recursive: true }); +writeFileSync( + reportPath, + JSON.stringify( + { + ranAt: new Date().toISOString(), + model: "deepseek-flash", + maxTurns: MAX_TURNS, + successRate: passed / results.length, + totals: { + cases: results.length, + passed, + avgTurns: avg((r) => r.turns), + avgToolCalls: avg((r) => r.toolCalls), + avgTokens: avg((r) => r.tokens), + }, + results, + }, + null, + 2, + ), +); + +console.log(`\n报告已写入 ${reportPath.pathname}`); +process.exit(passed === results.length ? 0 : 1); diff --git a/packages/cli/scripts/acceptance.ts b/packages/cli/scripts/acceptance.ts new file mode 100644 index 0000000..a6daf63 --- /dev/null +++ b/packages/cli/scripts/acceptance.ts @@ -0,0 +1,365 @@ +/** + * Manual acceptance runner. + * + * One command that exercises the four shipped capabilities the way a user would + * hit them, and prints PASS/FAIL per check. It replaces "trust me, the tests + * pass" with something you can actually watch. + * + * What it CANNOT cover: colours, toasts, and transcript behaviour in the TUI. + * Those need a real terminal — see `--` section at the bottom of the output. + * + * bun run packages/cli/scripts/acceptance.ts + * bun run packages/cli/scripts/acceptance.ts --with-eval # also runs 2 real tasks + * bun run packages/cli/scripts/acceptance.ts --with-network # also probes real providers + */ +import { readFileSync, rmSync } from "node:fs"; +import { isShellConfined, sandboxCapability } from "../src/lib/sandbox-capability"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Mode } from "@mocode/shared"; +import { executeLocalTool } from "../src/lib/local-tools"; +import { createFailoverFetch } from "../src/lib/chat-fetch"; +import { buildModelChain, classifyFailover, runWithFailover } from "../src/lib/provider-chain"; + +const args = new Set(process.argv.slice(2)); +const withEval = args.has("--with-eval"); +const withNetwork = args.has("--with-network"); + +type Check = { section: string; name: string; ok: boolean; detail: string }; +const checks: Check[] = []; + +function record(section: string, name: string, ok: boolean, detail: string) { + checks.push({ section, name, ok, detail }); + console.log(` ${ok ? "✅" : "❌"} ${name}`); + if (detail) console.log(` ${detail}`); +} + +/** Expect the call to reject; report the message for eyeballing. */ +async function expectRefusal( + section: string, + name: string, + run: () => Promise, + needle: string, +) { + try { + await run(); + record(section, name, false, "NOT REFUSED — this is a real hole"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + record(section, name, message.includes(needle), message.slice(0, 90)); + } +} + +function sh(command: string[]): { code: number; out: string } { + const result = Bun.spawnSync(command, { stdout: "pipe", stderr: "pipe" }); + return { + code: result.exitCode ?? 1, + out: `${result.stdout.toString()}${result.stderr.toString()}`.trim(), + }; +} + +// --------------------------------------------------------------------------- +console.log("\n=== P1 · Provider 失败回退 ===\n"); + +{ + // Policy invariants — the part that must never regress, regardless of network. + const chain = buildModelChain("gpt-oss-120b"); + record( + "P1", + "回退链把主模型排在首位、且不重复尝试同一家", + chain[0] === "gpt-oss-120b" && new Set(chain).size === chain.length, + chain.join(" → "), + ); + + const blocked = classifyFailover(Object.assign(new Error("x"), { statusCode: 403 })); + record("P1", "403 判定为可重试(换一家可能就好了)", blocked.retryable, blocked.detail); + + const badRequest = classifyFailover(Object.assign(new Error("x"), { statusCode: 400 })); + record( + "P1", + "400 判定为不可重试(是我们请求体写错了)", + badRequest.retryable === false, + badRequest.detail, + ); + + const aborted = classifyFailover(Object.assign(new Error("x"), { name: "AbortError" })); + record("P1", "用户按 Esc 取消后不重试", aborted.retryable === false, aborted.detail); + + // Real orchestration with a simulated provider (no network, no cost). + const outcome = await runWithFailover({ + primary: "gpt-oss-120b", + consent: true, + attempt: async (modelId) => { + if (modelId === "gpt-oss-120b") throw Object.assign(new Error("blocked"), { statusCode: 403 }); + return `answered by ${modelId}`; + }, + }); + record( + "P1", + "降级被显式上报(downgradedFrom 非空)", + outcome.downgradedFrom === "gpt-oss-120b" && outcome.modelId === "deepseek-flash", + `${outcome.downgradedFrom} → ${outcome.modelId}`, + ); +} + +{ + // The shape that broke in production: server answers 200, the provider error + // arrives as an SSE event. Status-code-only retry logic never fires here. + const sse = (events: Array>) => + new Response(events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join(""), { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + + let calls = 0; + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + fetchImpl: (async () => { + calls += 1; + return calls === 1 + ? sse([{ type: "start" }, { type: "error", errorText: "Forbidden" }]) + : sse([{ type: "start" }, { type: "text-delta", delta: "ok" }, { type: "finish" }]); + }) as unknown as typeof globalThis.fetch, + readError: async (response) => `HTTP ${response.status}`, + }); + + const response = await fetchWithFallback("http://localhost:3000/chat", { + method: "POST", + body: JSON.stringify({ id: "s", model: "gpt-oss-120b", mode: "build", messages: [] }), + }); + const text = await response.text(); + + record( + "P1", + "200 响应体里的 error 事件也会触发回退", + calls === 2 && text.includes("ok"), + `共 ${calls} 次请求,最终流内容包含成功响应`, + ); +} + +if (withNetwork) { console.log("\n [--with-network] 真实探测三家 Provider(约 40 秒)…"); + const result = sh(["bun", "run", "packages/server/scripts/provider-failover-bench.ts"]); + const e2e = result.out.split("端到端回退")[1] ?? ""; + record( + "P1", + "真实请求下回退到 DeepSeek 并成功", + e2e.includes("deepseek-flash") && e2e.includes("(downgraded)"), + "见下方 provider-failover-bench 输出", + ); + console.log(result.out.split("\n").slice(-12).join("\n")); +} else { + console.log("\n (网络真实探测需加 --with-network)"); +} + +// --------------------------------------------------------------------------- +console.log("\n=== P2 · 权限与沙箱 ===\n"); + +{ + // Informational: which sandbox backend this host can actually apply. + const capability = sandboxCapability(); + const verdict = capability.available + ? `✅ 可用(${capability.backend},${capability.enforced ? "已在 spawn 处套用" : "未套用"})` + : `⚠️ 不可用(${capability.backend})→ 每条 bash 命令都需人工批准`; + + console.log(` ${verdict}`); + console.log(` ${capability.detail}`); +} + +await expectRefusal( + "P2", + "读 /etc/passwd 被拒", + () => executeLocalTool("readFile", { path: "/etc/passwd" }, Mode.BUILD), + "outside the project", +); + +await expectRefusal( + "P2", + "父目录穿越 ../../etc/passwd 被拒", + () => executeLocalTool("readFile", { path: "../../../../etc/passwd" }, Mode.BUILD), + "outside the project", +); + +await expectRefusal( + "P2", + "写到项目外被拒", + () => executeLocalTool("writeFile", { path: "/tmp/mocode-acceptance.txt", content: "x" }, Mode.BUILD), + "outside the project", +); + +await expectRefusal( + "P2", + "硬封禁命令(dd 写设备)即使批准也拒绝", + () => executeLocalTool("bash", { command: "dd if=/dev/zero of=/dev/null count=1" }, Mode.BUILD), + "cannot be approved", +); + +await expectRefusal( + "P2", + "Plan 模式写文件被拒", + () => executeLocalTool("writeFile", { path: "x.txt", content: "y" }, Mode.PLAN), + "not available in PLAN", +); + +{ + // Real leak probe: put a canary in the CLI process and ask the shell for it. + const canary = "sk-acceptance-canary"; + const previousKey = process.env.DEEPSEEK_API_KEY; + process.env.DEEPSEEK_API_KEY = canary; + try { + const result = (await executeLocalTool( + "bash", + { command: 'echo -n "${DEEPSEEK_API_KEY:-absent}"' }, + Mode.BUILD, + )) as { stdout: string; stderr: string; exitCode: number }; + + const clean = result.stdout === "absent" && !result.stdout.includes(canary); + record( + "P2", + "子 shell 看不到 CLI 的密钥", + clean, + clean + ? `echo $DEEPSEEK_API_KEY → "${result.stdout}"` + : `stdout=${JSON.stringify(result.stdout)} exit=${result.exitCode} stderr=${JSON.stringify( + (result.stderr ?? "").slice(0, 200), + )} — 若 exit≠0 说明 shell 没跑起来,与密钥无关`, + ); + } finally { + if (previousKey === undefined) delete process.env.DEEPSEEK_API_KEY; + else process.env.DEEPSEEK_API_KEY = previousKey; + } +} + +{ + // Audit trail: a refusal that leaves no record is indistinguishable from a bug. + const auditPath = join(tmpdir(), `mocode-acceptance-audit-${Date.now()}.jsonl`); + process.env.MOCODE_AUDIT_PATH = auditPath; + + try { + await expectRefusal( + "P2", + "越界读取在审计日志里留下 PATH_ESCAPE", + () => executeLocalTool("readFile", { path: "/etc/shadow" }, Mode.BUILD), + "outside the project", + ); + + const logged = readFileSync(auditPath, "utf8"); + record( + "P2", + "审计日志记录了拒绝原因", + logged.includes('"code":"PATH_ESCAPE"') && logged.includes('"outcome":"denied"'), + logged.trim().split("\n").at(-1)?.slice(0, 120) ?? "", + ); + } catch { + record("P2", "审计日志记录了拒绝原因", false, "日志文件未生成"); + } finally { + delete process.env.MOCODE_AUDIT_PATH; + try { + rmSync(auditPath, { force: true }); + } catch { + /* ignore */ + } + } +} + +{ + // The original exploit, replayed end-to-end: the agent read /etc/passwd through + // bash and `head`, bypassing the file tools entirely. + if (!isShellConfined()) { + console.log(" ⏭ 跳过「bash 读 /etc/passwd」:本机无 OS 沙箱,该路径靠审批阻挡"); + } else { + const shellRead = (await executeLocalTool( + "bash", + { command: "head -1 /etc/passwd 2>&1" }, + Mode.BUILD, + )) as { stdout: string; stderr: string; exitCode: number }; + + const output = `${shellRead.stdout}${shellRead.stderr}`; + const leaked = /^[^:\n]+:[^:\n]*:[0-9]+:/m.test(output); + + record( + "P2", + "bash 也读不到 /etc/passwd(原始逃逸路径已封)", + !leaked, + leaked ? `泄漏了内容:${output.slice(0, 80)}` : output.trim().slice(0, 90), + ); + } +} + +// --------------------------------------------------------------------------- +console.log("\n=== P3 · 上下文压缩 ===\n"); + +{ + const result = sh(["bun", "run", "packages/server/scripts/context-compaction-bench.ts"]); + const triggered = result.out.includes("压缩触发: 是"); + const reduced = /→ (\d+) \(-(\d+)%\)/.exec(result.out); + const percent = reduced?.[2] ?? "?"; + + record( + "P3", + "50 轮会话触发压缩", + triggered, + result.out.split("\n").find((line) => line.startsWith("prompt tokens")) ?? "", + ); + record("P3", "压缩后 prompt 显著下降(≥70%)", Number(percent) >= 70, `降幅 ${percent}%`); + record( + "P3", + "折叠摘要说明了内容已不在上下文", + result.out.includes("re-read a file"), + "摘要末句提示模型需要时重新读文件", + ); +} + +// --------------------------------------------------------------------------- +console.log("\n=== P4 · 任务评估 ===\n"); + +if (withEval) { + console.log(" [--with-eval] 跑 2 条真实任务(约 15 秒,产生少量 token 费用)…"); + for (const id of ["fix-off-by-one", "rename-across-files"]) { + const result = sh(["bun", "run", "packages/cli/evals/run.ts", "--only", id]); + record("P4", `真实任务 ${id} 通过`, result.code === 0, result.out.split("\n")[2] ?? ""); + } +} else { + const result = sh(["bun", "test", "packages/cli/evals/cases.test.ts"]); + record( + "P4", + "元测试:每条用例在初始状态必须失败", + result.code === 0, + result.out.split("\n").find((line) => line.includes("pass")) ?? "", + ); + console.log(" (真实任务需加 --with-eval)"); +} + +// --------------------------------------------------------------------------- +const failed = checks.filter((check) => !check.ok); + +console.log("\n=== 汇总 ==="); +console.log(`通过 ${checks.length - failed.length}/${checks.length}`); + +if (failed.length > 0) { + console.log("\n失败项:"); + for (const check of failed) console.log(` - [${check.section}] ${check.name}: ${check.detail}`); +} + +console.log(` +=== 需要你在真实终端里确认的部分(脚本无法覆盖)=== + +1) 状态行预算可视化 + bun run dev → 发一条会连续调工具的任务 + 期望:状态行出现 "turn 2/24 · 3 tools",数字随工具调用递增 + +2) 降级提示(P1 的肉眼验证) + /models 选 gpt-oss-120b → 发消息 + 期望:Toast「模型 gpt-oss-120b 不可用,已回退到 deepseek-flash」 + 状态行出现红色 "fallback → deepseek-flash" + 回复内容正常,不是 Forbidden + +3) 压缩可观测(P3) + 连续读几个大文件,跑过 25 轮 + 期望:该轮 finish metadata 带 compaction 字段(服务端日志) + +4) 任务评估全量 + bun run packages/cli/evals/run.ts + 期望:10/10,报告写入 packages/cli/evals/results/latest.json +`); + +process.exit(failed.length === 0 ? 0 : 1); diff --git a/packages/cli/scripts/sandbox-bisect.ts b/packages/cli/scripts/sandbox-bisect.ts new file mode 100644 index 0000000..b639482 --- /dev/null +++ b/packages/cli/scripts/sandbox-bisect.ts @@ -0,0 +1,105 @@ +/** + * Sandbox profile bisect. + * + * `SIGABRT` from `sandbox-exec` gives no stderr, so guessing which rule is + * malformed is pointless. This walks from the smallest profile to the full + * generated one, adding one operation at a time, and reports the first case that + * aborts — which names the offending rule. + * + * bun run packages/cli/scripts/sandbox-bisect.ts + */ +import { buildSeatbeltProfile, parsePathList, runtimePaths } from "../src/lib/sandbox-profile"; + +const cwd = process.cwd(); +const options = { + cwd, + home: process.env.HOME, + tmpDir: process.env.TMPDIR, + runtimePaths: runtimePaths(), + extraReadPaths: parsePathList(process.env.MOCODE_SANDBOX_EXTRA_READ), + extraWritePaths: parsePathList(process.env.MOCODE_SANDBOX_EXTRA_WRITE), +}; + +const full = buildSeatbeltProfile(options); + +/** Each case adds exactly one rule to the previous one. */ +const cases: Array<{ name: string; profile: string }> = [ + { name: "1 (allow default) 基线", profile: "(version 1)(allow default)" }, + ...(() => { + const rules = [ + "(deny default)", + "(allow file-read*)", + "(allow process*)", + "(allow signal)", + "(allow sysctl-read)", + "(allow mach-lookup)", + "(allow file-read-metadata)", + "(allow file-map-executable)", + "(allow network*)", + ]; + + return rules.map((rule, index) => ({ + name: `2.${index + 1} + ${rule}`, + profile: `(version 1)\n${rules.slice(0, index + 1).join("\n")}`, + })); + })(), + { name: "3 完整生成的 profile(-p)", profile: full }, +]; + +function run(name: string, profile: string) { + const result = Bun.spawnSync(["/usr/bin/sandbox-exec", "-p", profile, "/bin/echo", "ok"], { + stdout: "pipe", + stderr: "pipe", + }); + + const exit = result.exitCode; + const signal = result.signalCode; + const stdout = result.stdout.toString().trim(); + const stderr = result.stderr.toString().trim(); + + const verdict = exit === 0 && stdout === "ok" ? "OK " : "FAIL"; + console.log(`${verdict} ${name.padEnd(34)} exit=${exit} signal=${signal ?? "-"}`); + + if (verdict === "FAIL") { + if (stderr) console.log(` stderr: ${stderr.slice(0, 300)}`); + if (stdout) console.log(` stdout: ${JSON.stringify(stdout.slice(0, 120))}`); + } + + return verdict === "OK "; +} + +console.log(`cwd: ${cwd}`); +console.log(`TMPDIR: ${process.env.TMPDIR ?? "(unset)"}\n`); + +let lastOk = ""; +let firstFail = ""; + +for (const testCase of cases) { + const ok = run(testCase.name, testCase.profile); + if (ok) lastOk = testCase.name; + else if (!firstFail) firstFail = testCase.name; +} + +// File mode prints parse errors that -p swallows. +console.log("\n--- 完整 profile 用 -f(文件模式会打印语法错误)---"); +const profilePath = `${process.env.TMPDIR ?? "/tmp"}/mocode-bisect.sb`; +await Bun.write(profilePath, full); +const fileRun = Bun.spawnSync(["/usr/bin/sandbox-exec", "-f", profilePath, "/bin/echo", "ok"], { + stdout: "pipe", + stderr: "pipe", +}); +console.log( + ` -f 结果: exit=${fileRun.exitCode} signal=${fileRun.signalCode ?? "-"} stderr=${JSON.stringify( + fileRun.stderr.toString().trim().slice(0, 300), + )}`, +); + +console.log(`\n=== 结论 ===`); +console.log(`最后一条通过的规则组合: ${lastOk || "(无)"}`); +console.log(`第一条失败的: ${firstFail || "(无)"}`); +console.log(` +判读: +- 卡在 2.x 的某一步 → 那一步的规则名就是元凶,删掉或换写法。 +- 1 就失败 → 这台机器的 sandbox-exec 对文字 profile 有问题,只认 -f 文件。 +- 3 失败而 2.x 全过 → 差值就是完整 profile 里的路径规则(subpath 列表)。 +`); diff --git a/packages/cli/scripts/sandbox-doctor.ts b/packages/cli/scripts/sandbox-doctor.ts new file mode 100644 index 0000000..b684cb0 --- /dev/null +++ b/packages/cli/scripts/sandbox-doctor.ts @@ -0,0 +1,168 @@ +/** + * Sandbox doctor. + * + * `SIGABRT` with empty stderr has two candidates, and they need different fixes: + * the profile is malformed, or the child environment we hand `sandbox-exec` is too + * bare. The probes below vary exactly one of those at a time. + * + * bun run packages/cli/scripts/sandbox-doctor.ts + */ +import { buildSandboxEnv } from "../src/lib/permission-rules"; +import { sandboxBackend, sandboxCapability } from "../src/lib/sandbox-capability"; +import { buildSeatbeltProfile, parsePathList, runtimePaths, spawnPlan } from "../src/lib/sandbox-profile"; + +const cwd = process.cwd(); +const backend = sandboxBackend(); + +const profileOptions = { + cwd, + home: process.env.HOME, + tmpDir: process.env.TMPDIR, + runtimePaths: runtimePaths(), + extraReadPaths: parsePathList(process.env.MOCODE_SANDBOX_EXTRA_READ), + extraWritePaths: parsePathList(process.env.MOCODE_SANDBOX_EXTRA_WRITE), +}; + +if (process.argv.includes("--dump-profile")) { + console.log(buildSeatbeltProfile(profileOptions)); + process.exit(0); +} + +console.log("=== capability ==="); +console.log(JSON.stringify(sandboxCapability(), null, 2)); +console.log(`backend used at spawn: ${backend}`); +console.log(`platform: ${process.platform}`); +console.log(`cwd: ${cwd}`); +console.log(`TMPDIR: ${process.env.TMPDIR ?? "(unset)"}`); + +// A canary in the parent process proves the env allowlist drops it. +process.env.MOCODE_DOCTOR_CANARY = "should-not-be-visible"; +const sandboxEnv = buildSandboxEnv(); + +const FULL_PROFILE = buildSeatbeltProfile(profileOptions); +const TRIVIAL_PROFILE = "(version 1)(allow default)"; + +type Variant = { + label: string; + argv: string[]; + env: Record; + expect: string; +}; + +function seatbelt(profile: string, command: string): string[] { + return ["/usr/bin/sandbox-exec", "-p", profile, "bash", "-c", command]; +} + +/** Linux has no sandbox-exec: build the bubblewrap argv from the real plan. */ +function bwrap(command: string): string[] { + const plan = spawnPlan("bwrap", command, profileOptions); + return [plan.file, ...plan.args]; +} + +const isMac = process.platform === "darwin"; + +const macVariants: Variant[] = [ + { + label: "1. 极简 profile + 继承 env (= 能力探测的跑法,已知能过)", + argv: seatbelt(TRIVIAL_PROFILE, "echo hi"), + env: process.env, + expect: "hi", + }, + { + label: "2. 极简 profile + 白名单 env (隔离 env 的影响)", + argv: seatbelt(TRIVIAL_PROFILE, "echo hi"), + env: sandboxEnv, + expect: "hi", + }, + { + label: "3. 完整 profile + 继承 env (隔离 profile 的影响)", + argv: seatbelt(FULL_PROFILE, "echo hi"), + env: process.env, + expect: "hi", + }, + { + label: "4. 完整 profile + 白名单 env (= 真实 spawn,当前失败)", + argv: seatbelt(FULL_PROFILE, "echo hi"), + env: sandboxEnv, + expect: "hi", + }, + { + label: "5. 真实 spawnPlan 路径", + argv: (() => { + const plan = spawnPlan(backend, "echo hi", profileOptions); + return [plan.file, ...plan.args]; + })(), + env: sandboxEnv, + expect: "hi", + }, + { + label: "6. 完整 profile:读 /etc/passwd 应被拒", + argv: seatbelt(FULL_PROFILE, "head -1 /etc/passwd"), + env: process.env, + expect: "denied (no content)", + }, + { + label: "7. 完整 profile:canary 变量应不可见", + argv: seatbelt(FULL_PROFILE, 'echo -n "${MOCODE_DOCTOR_CANARY:-absent}"'), + env: sandboxEnv, + expect: "absent", + }, +]; + +const linuxVariants: Variant[] = [ + { + label: "1. 真实 bwrap 计划 + 继承 env", + argv: bwrap("echo hi"), + env: process.env, + expect: "hi", + }, + { + label: "2. 真实 bwrap 计划 + 白名单 env", + argv: bwrap("echo hi"), + env: sandboxEnv, + expect: "hi", + }, + { + label: "3. 读 /etc/shadow 应被 mask", + argv: bwrap("head -1 /etc/shadow"), + env: process.env, + expect: "denied (no content)", + }, + { + label: "4. canary 变量应不可见", + argv: bwrap('echo -n "${MOCODE_DOCTOR_CANARY:-absent}"'), + env: sandboxEnv, + expect: "absent", + }, +]; + +const variants: Variant[] = isMac ? macVariants : linuxVariants; + +for (const variant of variants) { + const result = Bun.spawnSync(variant.argv, { + cwd, + env: variant.env as Record, + stdout: "pipe", + stderr: "pipe", + }); + + const stdout = result.stdout.toString(); + const stderr = result.stderr.toString(); + + console.log(`\n### ${variant.label}`); + console.log(` expect : ${variant.expect}`); + console.log(` exit : ${result.exitCode} signal: ${result.signalCode ?? "-"}`); + console.log(` stdout : ${JSON.stringify(stdout.slice(0, 200))}`); + console.log(` stderr : ${JSON.stringify(stderr.slice(0, 400))}`); +} + +console.log(` +=== 怎么读 === + +2 崩了(而 1 正常) → 是**白名单 env** 的问题。sandbox-exec 需要某个被我们剔除的 + 环境变量;修法是给它一份完整的 env,再单独约束子进程。 +3 崩了(而 2 正常) → 是**profile** 的问题。跑 sandbox-bisect.ts 定位到具体规则。 +2 和 3 都正常,4 崩 → 两者相互作用(一般仍是 env:profile 里引用了仅 parent 才有的变量)。 +6 有内容 → 沙箱没真正生效,按安全 bug 处理。 +7 打印 canary → env 白名单失效。 +`); diff --git a/packages/cli/src/components/dialogs/keys-wizard-dialog.tsx b/packages/cli/src/components/dialogs/keys-wizard-dialog.tsx index c173aaa..00c7562 100644 --- a/packages/cli/src/components/dialogs/keys-wizard-dialog.tsx +++ b/packages/cli/src/components/dialogs/keys-wizard-dialog.tsx @@ -21,6 +21,7 @@ const PROVIDERS = [ "groq", "cerebras", "openrouter", + "deepseek", ] as const; type ProviderId = (typeof PROVIDERS)[number]; diff --git a/packages/cli/src/components/input-bar.tsx b/packages/cli/src/components/input-bar.tsx index 631b2d5..caa1fbf 100644 --- a/packages/cli/src/components/input-bar.tsx +++ b/packages/cli/src/components/input-bar.tsx @@ -8,7 +8,8 @@ import type { KeyBinding } from "@opentui/core"; import { useNavigate } from "react-router"; import { EmptyBorder } from "./border"; import { StatusBar } from "./status-bar"; -import { CommandMenu } from "./command-menu"; +import type { AgentProgress } from "../lib/agent-loop"; +import type { AgentPhase } from "../lib/agent-phase";import { CommandMenu } from "./command-menu"; import type { Command } from "./command-menu/types"; import { useCommandMenu } from "./command-menu/use-command-menu"; import { useToast } from "../providers/toast"; @@ -264,6 +265,12 @@ type Props = { composerRestoreToken?: number; subagentRunning?: boolean; subagentType?: string | null; + /** Loop budget shown in the status line; see `lib/agent-loop.ts`. */ + agentProgress?: AgentProgress | null; + /** Set when a fallback model answered. */ + agentDowngrade?: { from: string; to: string } | null; + agentPhase?: AgentPhase | null; + agentElapsedMs?: number | null; }; export const TEXTAREA_KEY_BINDINGS: KeyBinding[] = [ @@ -280,6 +287,10 @@ export function InputBar({ composerRestoreToken = 0, subagentRunning = false, subagentType = null, + agentProgress = null, + agentDowngrade = null, + agentPhase = null, + agentElapsedMs = null, }: Props) { const { mode, model, toggleMode, setMode, setModel } = usePromptConfig(); const textareaRef = useRef(null); @@ -643,7 +654,14 @@ export function InputBar({ onContentChange={handleTextareaContentChange} placeholder={`Ask anything... "Fix a bug in the database"`} /> - + diff --git a/packages/cli/src/components/messages/bot-message.render.test.tsx b/packages/cli/src/components/messages/bot-message.render.test.tsx new file mode 100644 index 0000000..096884c --- /dev/null +++ b/packages/cli/src/components/messages/bot-message.render.test.tsx @@ -0,0 +1,51 @@ +import { describe, expect, test } from "bun:test"; +import { Mode } from "@mocode/shared"; +import { BotMessage } from "./bot-message"; +import { renderText } from "../../testing/render"; +import { markToolStarted, resetToolTimings } from "../../lib/tool-timing"; + +type Part = Parameters[0]["parts"][number]; + +function pendingTool(id: string): Part { + return { + type: "tool-bash", + toolCallId: id, + state: "input-available", + input: { command: "npm test", description: "run tests" }, + } as unknown as Part; +} + +describe("BotMessage tool timing", () => { + test("a running tool shows how long it has been running", async () => { + resetToolTimings(); + markToolStarted("call-1", Date.now() - 2_300); + + const frame = await renderText( + , + { width: 100, height: 8 }, + ); + + expect(frame).toContain("2.3s"); + expect(frame).toContain("npm test"); + }); + + test("a finished tool shows no timer", async () => { + resetToolTimings(); + markToolStarted("call-1", Date.now() - 2_300); + + const finished = { + type: "tool-bash", + toolCallId: "call-1", + state: "output-available", + input: { command: "npm test" }, + output: { stdout: "ok", stderr: "", exitCode: 0 }, + } as unknown as Part; + + const frame = await renderText( + , + { width: 100, height: 8 }, + ); + + expect(frame).not.toContain("2.3s"); + }); +}); diff --git a/packages/cli/src/components/messages/bot-message.tsx b/packages/cli/src/components/messages/bot-message.tsx index e2eef43..89efad6 100644 --- a/packages/cli/src/components/messages/bot-message.tsx +++ b/packages/cli/src/components/messages/bot-message.tsx @@ -20,9 +20,10 @@ import { TASK_SUMMARY_MAX_VISIBLE_LINES, } from "../../lib/bot-message-task"; import { groupConsecutiveParts, partRenderKey } from "../../lib/bot-message-parts"; +import { formatToolElapsed, toolElapsedMs } from "../../lib/tool-timing"; import type { LanguageModelUsage } from "ai"; import prettyMs from "pretty-ms"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import { EmptyBorder } from "../border"; import { Spinner } from "../spinner"; import { useTheme } from "../../providers/theme"; @@ -193,6 +194,15 @@ export function BotMessage({ toolsPending, }); const showDuration = shouldShowDurationInFooter({ streaming, durationMs }); + + // While a tool is pending, re-render a few times a second so its elapsed label + // counts up instead of freezing at the value captured when the row appeared. + const [, tick] = useState(0); + useEffect(() => { + if (!toolsPending) return; + const timer = setInterval(() => tick((value) => value + 1), 200); + return () => clearInterval(timer); + }, [toolsPending]); const showMessageFooter = shouldShowAssistantMessageFooter(parts); return ( @@ -239,10 +249,14 @@ export function BotMessage({ ? formatBashToolDisplay(part.input) : null; const argsText = bashDisplay?.command ?? formatToolArgs(part); - const statusSuffix = - part.state !== "output-available" && part.state !== "output-error" + const isPending = + part.state !== "output-available" && part.state !== "output-error"; + const elapsed = isPending ? toolElapsedMs(part.toolCallId) : null; + const statusSuffix = isPending + ? elapsed === null ? " …" - : ""; + : ` … ${formatToolElapsed(elapsed)}` + : ""; const errorSuffix = part.state === "output-error" && part.errorText ? ` ${formatToolErrorForDisplay(toolName, part.errorText)}` diff --git a/packages/cli/src/components/session-shell.tsx b/packages/cli/src/components/session-shell.tsx index e856a62..6a1a144 100644 --- a/packages/cli/src/components/session-shell.tsx +++ b/packages/cli/src/components/session-shell.tsx @@ -3,6 +3,8 @@ import type { ReactNode, RefObject } from "react"; import { InputBar } from "./input-bar"; import { Spinner } from "./spinner"; import { usePromptConfig } from "../providers/prompt-config"; +import type { AgentProgress } from "../lib/agent-loop"; +import type { AgentPhase } from "../lib/agent-phase"; /** Session layout: scrollable transcript, input bar, and status footer. */ type Props = { @@ -16,6 +18,12 @@ type Props = { /** While a Task subagent runs — shows type in footer (D-21). */ subagentRunning?: boolean; subagentType?: string | null; + /** Loop budget shown in the status line; see `lib/agent-loop.ts`. */ + agentProgress?: AgentProgress | null; + /** Set when a fallback model answered. */ + agentDowngrade?: { from: string; to: string } | null; + agentPhase?: AgentPhase | null; + agentElapsedMs?: number | null; composerRestoreText?: string | null; composerRestoreToken?: number; transcriptScrollRef?: RefObject; @@ -29,6 +37,10 @@ export function SessionShell({ interruptible = false, subagentRunning = false, subagentType = null, + agentProgress = null, + agentDowngrade = null, + agentPhase = null, + agentElapsedMs = null, composerRestoreText = null, composerRestoreToken = 0, transcriptScrollRef, @@ -60,7 +72,7 @@ export function SessionShell({ - + { + test("idle: shows mode, model and hints, but no loop budget", async () => { + const frame = await renderText(); + + expect(frame).toContain("Build"); + expect(frame).toContain(">"); + expect(frame).toContain("提交"); + expect(frame).toContain("换行"); + expect(frame).not.toContain("turn"); + expect(frame).not.toContain("tools"); + }); + + test("shows the turn budget once a run has continued", async () => { + const frame = await renderText( + , + ); + + expect(frame).toContain("turn 4/51"); + expect(frame).not.toContain("tools"); + }); + + test("appends the tool count for the current turn", async () => { + const frame = await renderText( + , + ); + + expect(frame).toContain("turn 4/51 · 4 tools"); + }); + + test("hides the budget before the first continued turn", async () => { + const frame = await renderText( + , + ); + + expect(frame).not.toContain("turn"); + }); + + test("keeps the whole status line on a single row (DESIGN.md: status-row = 1)", async () => { + const lines = await renderLines( + , + { width: 100, height: 4 }, + ); + + expect(lines).toHaveLength(1); + expect(lines[0]).toContain("turn 13/51 · 9 tools"); + }); + + test("subagent footer replaces the budget while a Task runs", async () => { + const frame = await renderText( + , + ); + + expect(frame).toContain("explore"); + expect(frame).toContain("esc to interrupt"); + expect(frame).not.toContain("turn"); + }); + + test("shows a downgrade notice when a fallback model answered", async () => { + const frame = await renderText( + , + ); + + expect(frame).toContain("fallback → deepseek-flash"); + }); + + test("shows no downgrade notice on a normal turn", async () => { + const frame = await renderText(); + + expect(frame).not.toContain("fallback"); + }); + + test("narrow width with downgrade + budget stays on one row and drops newline first", async () => { + const lines = await renderLines( + , + // Wide enough for mode/model/downgrade/budget; tight enough to drop newline/note. + { width: 72, height: 4 }, + ); + + expect(lines).toHaveLength(1); + expect(lines[0]).toContain("fallback → deepseek-flash"); + expect(lines[0]).toContain("turn 13/51"); + expect(lines[0]).not.toContain("换行"); + }); +}); + +describe("selectStatusHints", () => { + const hint = { submit: "↵", newline: "⇧↵", note: "Ctrl+J" }; + + test("keeps every hint when there is room", () => { + expect(selectStatusHints(120, 20, hint)).toEqual({ + showSubmit: true, + showNewline: true, + showNote: true, + }); + }); + + test("drops newline before submit under pressure", () => { + expect(selectStatusHints(40, 30, hint)).toEqual({ + showSubmit: true, + showNewline: false, + showNote: false, + }); + }); +}); + +describe("live agent phase", () => { + test("shows the running phase with a counting timer", async () => { + const frame = await renderText( + , + ); + + expect(frame).toContain("执行中 2.4s"); + }); + + test("thinking and generating are distinguishable", async () => { + const thinking = await renderText(); + const generating = await renderText( + , + ); + + expect(thinking).toContain("思考中 1.0s"); + expect(generating).toContain("生成中 9.0s"); + }); + + test("idle shows no phase at all", async () => { + const frame = await renderText(); + + expect(frame).not.toContain("s ·"); + expect(frame).not.toContain("执行中"); + }); + + test("phase and budget coexist on one row", async () => { + const lines = await renderLines( + , + { width: 110, height: 4 }, + ); + + expect(lines).toHaveLength(1); + expect(lines[0]).toContain("执行中 2.4s"); + expect(lines[0]).toContain("turn 3/51"); + }); +}); diff --git a/packages/cli/src/components/status-bar.tsx b/packages/cli/src/components/status-bar.tsx index 5adbe15..89d18e3 100644 --- a/packages/cli/src/components/status-bar.tsx +++ b/packages/cli/src/components/status-bar.tsx @@ -1,24 +1,109 @@ import { TextAttributes } from "@opentui/core"; -import { getNewlineHint } from "../terminal-capabilities"; +import { useTerminalDimensions } from "@opentui/react"; +import { getNewlineHint, type NewlineHint } from "../terminal-capabilities"; import { useTheme } from "../providers/theme"; import { usePromptConfig } from "../providers/prompt-config"; import { Mode } from "@mocode/shared"; +import { formatAgentProgress, type AgentProgress } from "../lib/agent-loop"; +import { formatAgentPhase, type AgentPhase } from "../lib/agent-phase"; import { Spinner } from "./spinner"; type Props = { subagentRunning?: boolean; subagentType?: string | null; + /** Loop budget for the current user message; see `lib/agent-loop.ts`. */ + agentProgress?: AgentProgress | null; + /** Set when a fallback model answered — shown in `{color.error}`, never hidden. */ + agentDowngrade?: { from: string; to: string } | null; + /** Live phase of the running turn, e.g. `执行中`. */ + agentPhase?: AgentPhase | null; + /** Live elapsed time for the run, in ms. */ + agentElapsedMs?: number | null; }; -/** Footer hints: agent mode, model label, and submit/newline shortcuts for this terminal. */ -export function StatusBar({ subagentRunning = false, subagentType = null }: Props) { +/** Horizontal space the shell + composer insets take from the status row. */ +const STATUS_ROW_INSET = 4; + +/** Trim a label to `max` cells, marking the cut with an ellipsis. */ +function truncateToWidth(text: string, max: number): string { + if (cellWidth(text) <= max) return text; + if (max <= 1) return "…"; + + let out = ""; + let used = 0; + for (const char of text) { + const w = char.codePointAt(0)! > 0xff ? 2 : 1; + if (used + w > max - 1) break; + out += char; + used += w; + } + + return `${out}…`; +} + +/** Visible width of a status segment in character cells (CJK ≈ 2 cells). */ +function cellWidth(text: string): number { + let width = 0; + for (const char of text) { + width += char.codePointAt(0)! > 0xff ? 2 : 1; + } + return width; +} + +type HintVisibility = { + showSubmit: boolean; + showNewline: boolean; + showNote: boolean; +}; + +/** + * Drop hints right-to-left under width pressure (DESIGN.md): note → newline → submit. + * Mode / model / downgrade / budget are never dropped. + */ +export function selectStatusHints( + availableCols: number, + fixedCols: number, + hint: NewlineHint, +): HintVisibility { + // Each hint is rendered as `·` + label with gap={1} before both nodes. + const hintCols = (label: string) => 1 + cellWidth("·") + 1 + cellWidth(label); + const submitCols = hintCols(`${hint.submit} 提交`); + const newlineCols = hintCols(`${hint.newline} 换行`); + const noteCols = hint.note ? hintCols(hint.note) : 0; + const budget = Math.max(0, availableCols - fixedCols); + + if (submitCols + newlineCols + noteCols <= budget) { + return { showSubmit: true, showNewline: true, showNote: Boolean(hint.note) }; + } + // Drop note first (rightmost), then newline, then submit. + if (submitCols + newlineCols <= budget) { + return { showSubmit: true, showNewline: true, showNote: false }; + } + if (submitCols <= budget) { + return { showSubmit: true, showNewline: false, showNote: false }; + } + return { showSubmit: false, showNewline: false, showNote: false }; +} + +/** Footer hints: agent mode, model label, loop budget, and submit/newline shortcuts for this terminal. */ +export function StatusBar({ + subagentRunning = false, + subagentType = null, + agentProgress = null, + agentDowngrade = null, + agentPhase = null, + agentElapsedMs = null, +}: Props) { const hint = getNewlineHint(); const { colors } = useTheme(); const { mode, model } = usePromptConfig(); + const { width } = useTerminalDimensions(); + const budgetLabel = formatAgentProgress(agentProgress); + const phaseLabel = formatAgentPhase(agentPhase, agentElapsedMs); if (subagentRunning && subagentType) { return ( - + {subagentType} @@ -29,28 +114,95 @@ export function StatusBar({ subagentRunning = false, subagentType = null }: Prop ); } + const modeLabel = mode === Mode.PLAN ? "Plan" : "Build"; + const downgradeLabel = agentDowngrade ? `fallback → ${agentDowngrade.to}` : null; + + // The row lives inside the shell's padding and the composer inset, so the space + // it can actually use is narrower than the terminal. Budgeting against the raw + // terminal width shows hints that then get clipped by `overflow="hidden"`. + const usableWidth = Math.max(0, width - STATUS_ROW_INSET); + let modelLabel: string = model; + + // Approximate fixed cluster width. `gap={1}` inserts one cell between each + // sibling text node in the row below (mode, >, model, and optional pairs). + const fixedNodes = + 3 + // mode, >, model + (downgradeLabel ? 2 : 0) + + (budgetLabel ? 2 : 0) + + (phaseLabel ? 2 : 0); + let fixedCols = cellWidth(modeLabel) + cellWidth(">") + cellWidth(model); + if (downgradeLabel) fixedCols += cellWidth("·") + cellWidth(downgradeLabel); + if (budgetLabel) fixedCols += cellWidth("·") + cellWidth(budgetLabel); + if (phaseLabel) fixedCols += cellWidth("·") + cellWidth(phaseLabel); + fixedCols += Math.max(0, fixedNodes - 1); // gaps between siblings + + // If even the fixed labels overflow, shrink the model id rather than letting the + // rightmost segments (downgrade, budget) be clipped away silently. + if (fixedCols > usableWidth) { + const overflow = fixedCols - usableWidth; + modelLabel = truncateToWidth(modelLabel, Math.max(4, cellWidth(modelLabel) - overflow)); + fixedCols = Math.max( + 0, + cellWidth(modeLabel) + cellWidth(">") + cellWidth(modelLabel) + (downgradeLabel ? cellWidth("·") + cellWidth(downgradeLabel) : 0) + (budgetLabel ? cellWidth("·") + cellWidth(budgetLabel) : 0) + Math.max(0, fixedNodes - 1), + ); + } + + const visibility = selectStatusHints(usableWidth, fixedCols, hint); + return ( - + - {mode === Mode.PLAN ? "Plan" : "Build"} + {modeLabel} {">"} - {model} - - · - - - {hint.submit} 提交 - - - · - - - {hint.newline} 换行 - - {hint.note ? ( + {modelLabel} + {phaseLabel ? ( + <> + + · + + {phaseLabel} + + ) : null} + {agentDowngrade ? ( + <> + + · + + {downgradeLabel} + + ) : null} + {budgetLabel ? ( + <> + + · + + {budgetLabel} + + ) : null} + {visibility.showSubmit ? ( + <> + + · + + + {hint.submit} 提交 + + + ) : null} + {visibility.showNewline ? ( + <> + + · + + + {hint.newline} 换行 + + + ) : null} + {visibility.showNote && hint.note ? ( <> · diff --git a/packages/cli/src/hooks/use-chat.ts b/packages/cli/src/hooks/use-chat.ts index 6d958ec..569c600 100644 --- a/packages/cli/src/hooks/use-chat.ts +++ b/packages/cli/src/hooks/use-chat.ts @@ -29,6 +29,7 @@ import { import { Mode, toolInputSchemas, type ModeType, type SupportedChatModelId, type ToolContracts } from "@mocode/shared"; import { apiClient } from "../lib/api-client"; import { getErrorMessage } from "../lib/http-errors"; +import { createFailoverFetch, type DowngradeInfo } from "../lib/chat-fetch"; import { getAuth } from "../lib/auth"; import { executeLocalTool } from "../lib/local-tools"; import { requiresApproval, rememberSessionAllow } from "../lib/bash-approval"; @@ -58,6 +59,11 @@ import { type ResumeEligibility, } from "../lib/stream-interrupt"; import { formatChatStreamError } from "../lib/stream-error"; +import { AgentLoop, type AgentProgress, type AgentStopReason } from "../lib/agent-loop"; +import { failoverConsentFromEnv } from "../lib/provider-chain"; +import { deriveAgentPhase, isRunActive, type AgentPhase } from "../lib/agent-phase"; +import { markToolStarted, resetToolTimings } from "../lib/tool-timing"; +import { checkRules } from "../lib/permissions"; import { scheduleLocalSessionPersist } from "./use-chat-persist"; import { killTrackedToolProcesses } from "../lib/tool-process-registry"; import { @@ -204,6 +210,22 @@ export function useChat( /** Blocks sendAutomaticallyWhen after Esc finalizes tool output-error parts. */ const turnInterruptedRef = useRef(false); const [turnInterrupted, setTurnInterrupted] = useState(false); + /** + * Explicit loop budget for the current user message: turn cap, per-turn tool + * cap, and the consecutive-repeat guard. See `lib/agent-loop.ts`. + */ + const agentLoopRef = useRef(new AgentLoop()); + const [agentProgress, setAgentProgress] = useState(null); + /** Non-null when a fallback model answered — must stay visible, never silent. */ + const [agentDowngrade, setAgentDowngrade] = useState(null); + /** Live elapsed time for the current run, so the footer can count up. */ + const [agentElapsedMs, setAgentElapsedMs] = useState(null); + const runStartedAtRef = useRef(null); + const reportDowngradeRef = useRef<(info: DowngradeInfo) => void>(() => {}); + const [agentStop, setAgentStop] = useState<{ + reason: AgentStopReason; + detail: string; + } | null>(null); /** Active subagent run — Esc aborts via linked AbortController (D-10). */ const subagentAbortRef = useRef(null); const [subagentRunning, setSubagentRunning] = useState(false); @@ -213,12 +235,43 @@ export function useChat( const onPersistErrorRef = useRef(options?.onPersistError); onPersistErrorRef.current = options?.onPersistError; + // Latest-ref so the memoised transport can report a downgrade without re-creating. + reportDowngradeRef.current = (info: DowngradeInfo) => { + setAgentDowngrade(info); + toast.show({ + variant: "error", + message: `模型 ${info.from} 不可用,已回退到 ${info.to}`, + }); + }; + + /** + * Surface a loop stop. Deferred via microtask because this is called from the + * SDK's `sendAutomaticallyWhen` predicate, which runs during render reconciliation. + * A stop can be observed twice (tool gate + turn gate), so it reports once. + */ + const agentStopReportedRef = useRef(false); + const reportAgentStop = useCallback( + (decision: { reason: AgentStopReason; detail: string }) => { + // Esc already renders its own interrupted state — no toast for it. + if (decision.reason === "aborted" || agentStopReportedRef.current) return; + agentStopReportedRef.current = true; + queueMicrotask(() => { + setAgentStop({ reason: decision.reason, detail: decision.detail }); + toast.show({ variant: "error", message: decision.detail }); + }); + }, + [toast], + ); + const transport = useMemo(() => { // BYOK: inference + tool schema merge run in-process; MCP execution stays in onToolCall below. if (isLocalMode()) { return new LocalChatTransport({ resolveModel: resolveChatModel, getMcpManager, + // BYOK failures happen in-process; report the downgrade the same way as SaaS. + allowFailover: failoverConsentFromEnv(), + onDowngrade: (info) => reportDowngradeRef.current({ ...info, failures: [] }), buildSystemPrompt: (params) => buildSystemPrompt({ ...params, @@ -227,14 +280,12 @@ export function useChat( }) as ChatTransport; } - const chatFetch = (async (input, init) => { - const response = await globalThis.fetch(input, init); - if (!response.ok) { - const message = await getErrorMessage(response); - throw new Error(message); - } - return response; - }) as typeof globalThis.fetch; + // Retries the request on the next model when a provider is blocked (see lib/chat-fetch.ts). + const chatFetch = createFailoverFetch({ + readError: getErrorMessage, + allowFailover: failoverConsentFromEnv(), + onDowngrade: (info) => reportDowngradeRef.current(info), + }); return new DefaultChatTransport({ api: apiClient.chat.$url().toString(), @@ -351,6 +402,9 @@ export function useChat( messages: initialMessages, transport, async onToolCall({ toolCall }) { + // Started clock for the per-tool elapsed label; presentation only. + markToolStarted(toolCall.toolCallId); + const shouldSkipToolOutput = () => shouldSkipInterruptedToolOutput(toolCall.toolCallId, skipToolOutputIdsRef.current); @@ -361,6 +415,31 @@ export function useChat( const isMcpCall = looksLikeMcpToolName(toolCall.toolName); + // Loop budget: account for every tool call before it runs, so a run that is + // out of turns / out of per-turn calls / repeating itself stops with a named + // reason instead of silently continuing. + const loopDecision = agentLoopRef.current.recordToolCall(toolCall.toolName, toolCall.input); + if (loopDecision.action === "stop") { + reportAgentStop(loopDecision); + if (!shouldSkipToolOutput()) { + const addStopOutput = chat.addToolOutput as (params: { + tool?: keyof ChatTools; + toolCallId: string; + state: "output-error"; + errorText: string; + }) => void; + addStopOutput({ + tool: toolCall.toolName as keyof ChatTools, + toolCallId: toolCall.toolCallId, + state: "output-error", + errorText: loopDecision.detail, + }); + } + return; + } + + setAgentProgress(agentLoopRef.current.snapshot()); + // Phase 04 (04-03): Task subagent — synchronous blocking, summary-only (D-03, D-04). if (toolCall.toolName === "task") { if (shouldSkipToolOutput()) return; @@ -474,9 +553,24 @@ export function useChat( return { approved: true }; } - if (toolCall.toolName === "bash" && mode === Mode.BUILD) { + // User rules from ~/.mocode and /.mocode. + const ruleVerdict = checkRules({ + tool: toolCall.toolName, + input: toolCall.input, + }); + + if (ruleVerdict?.behavior === "deny") { + return { approved: false, reason: ruleVerdict.detail }; + } + + // An explicit allow skips the built-in dialogs; an explicit ask forces + // one even where the built-in policy would have run the tool silently. + const ruleAllows = ruleVerdict?.behavior === "allow"; + const ruleAsks = ruleVerdict?.behavior === "ask"; + + if (!ruleAllows && toolCall.toolName === "bash" && mode === Mode.BUILD) { const { command } = toolInputSchemas.bash.parse(toolCall.input); - if (requiresApproval(command, sessionAllowRef.current)) { + if (ruleAsks || requiresApproval(command, sessionAllowRef.current)) { const verdict = await requestBashApproval(dialog, command); if (verdict === "reject") { return { approved: false, reason: BASH_REJECT_ERROR_TEXT }; @@ -487,12 +581,20 @@ export function useChat( } } + // Only write tools can prompt here; a rule asking about a read is not a + // reason to show a write dialog. + const isWriteTool = + toolCall.toolName === "writeFile" || toolCall.toolName === "editFile"; + if ( - requiresLocalWriteApproval( - toolCall.toolName, - mode, - sessionWriteAllowRef.current, - ) + !ruleAllows && + isWriteTool && + (ruleAsks || + requiresLocalWriteApproval( + toolCall.toolName, + mode, + sessionWriteAllowRef.current, + )) ) { const verdict = await requestLocalWriteApproval( dialog, @@ -581,7 +683,22 @@ export function useChat( }, sendAutomaticallyWhen: (params) => { if (turnInterruptedRef.current) return false; - return lastAssistantMessageIsCompleteWithToolCalls(params); + + // A new user message starts a fresh budget; regenerating keeps the current one. + const lastUser = params.messages.findLast((message) => message.role === "user"); + if (lastUser) agentLoopRef.current.beginUserMessage(lastUser.id); + + if (!lastAssistantMessageIsCompleteWithToolCalls(params)) return false; + + const decision = agentLoopRef.current.beginTurn(); + if (decision.action === "stop") { + reportAgentStop(decision); + return false; + } + + // Status-line budget; deferred for the same reason as reportAgentStop. + queueMicrotask(() => setAgentProgress(agentLoopRef.current.snapshot())); + return true; }, }); @@ -618,6 +735,7 @@ export function useChat( setActiveSlashMessages(null); turnInterruptedRef.current = true; setTurnInterrupted(true); + agentLoopRef.current.abort(); killTrackedToolProcesses(); const snapshot = snapshotChatMessages(chat); @@ -652,6 +770,13 @@ export function useChat( turnInterruptedRef.current = false; setTurnInterrupted(false); skipToolOutputIdsRef.current.clear(); + // Explicit retry: hand the model a fresh budget, otherwise a run that hit + // `max-turns` would stop again immediately on resume. + agentLoopRef.current.reset(); + agentStopReportedRef.current = false; + setAgentStop(null); + setAgentProgress(null); + setAgentDowngrade(null); const eligibility = detectResumeEligibility(chat.messages, chat.status); const action = resolveResumeTransport(eligibility); if (action === "none") return; @@ -743,6 +868,11 @@ export function useChat( turnInterruptedRef.current = false; setTurnInterrupted(false); skipToolOutputIdsRef.current.clear(); + resetToolTimings(); + agentStopReportedRef.current = false; + setAgentStop(null); + setAgentProgress(null); + setAgentDowngrade(null); const trimmed = params.userText.trim(); const exploreMatch = /^\/explore(?:\s+(.*))?$/s.exec(trimmed); @@ -782,10 +912,46 @@ export function useChat( [chat.sendMessage, runSlashSubagent, toast.show], ); + // Tick the run timer while the agent is busy. 200ms is fast enough to look live + // and cheap enough to ignore. + // Active means "the loop is doing something", not "the HTTP stream is open": + // tool execution and the gap before auto-resubmit also count. + const runActive = isRunActive({ status: chat.status, message: chat.messages.at(-1) }); + + useEffect(() => { + const busy = runActive; + + if (!busy) { + runStartedAtRef.current = null; + setAgentElapsedMs(null); + return; + } + + runStartedAtRef.current ??= Date.now(); + const timer = setInterval(() => { + const startedAt = runStartedAtRef.current; + if (startedAt !== null) setAgentElapsedMs(Date.now() - startedAt); + }, 200); + + return () => clearInterval(timer); + }, [runActive]); + + // Derived from the live message, so the label can never disagree with what is + // already on screen. + const agentPhase: AgentPhase | null = useMemo( + () => deriveAgentPhase({ status: chat.status, message: chat.messages.at(-1) }), + [chat.status, chat.messages], + ); + return { messages: displayMessages, status: chat.status, + agentPhase, + agentElapsedMs, turnInterrupted, + agentProgress, + agentDowngrade, + agentStop, subagentRunning, subagentType, error: chat.error diff --git a/packages/cli/src/lib/agent-loop.test.ts b/packages/cli/src/lib/agent-loop.test.ts new file mode 100644 index 0000000..1aec17b --- /dev/null +++ b/packages/cli/src/lib/agent-loop.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, test } from "bun:test"; +import { AgentLoop, DEFAULT_AGENT_LIMITS, formatAgentProgress, toolSignature } from "./agent-loop"; + +describe("AgentLoop turn budget", () => { + test("allows maxTurns turns, then stops with a named reason", () => { + const loop = new AgentLoop({ maxTurns: 3 }); + + expect(loop.beginTurn().action).toBe("continue"); + expect(loop.beginTurn().action).toBe("continue"); + expect(loop.beginTurn().action).toBe("continue"); + + const decision = loop.beginTurn(); + expect(decision).toEqual({ + action: "stop", + reason: "max-turns", + detail: "Stopped after 3 turns without a final answer.", + }); + }); + + test("stop is sticky and reports the original reason", () => { + const loop = new AgentLoop({ maxTurns: 1 }); + loop.beginTurn(); + loop.beginTurn(); + + expect(loop.beginTurn()).toEqual(loop.beginTurn()); + expect(loop.stopped).toBe("max-turns"); + }); +}); + +describe("AgentLoop tool budget", () => { + test("caps tool calls inside one turn and resets per turn", () => { + const loop = new AgentLoop({ maxToolCallsPerTurn: 2 }); + + loop.beginTurn(); + expect(loop.recordToolCall("readFile", { path: "a" }).action).toBe("continue"); + expect(loop.recordToolCall("readFile", { path: "b" }).action).toBe("continue"); + + const capped = loop.recordToolCall("readFile", { path: "c" }); + expect(capped).toMatchObject({ action: "stop", reason: "max-tool-calls" }); + }); + + test("a new turn restores the per-turn budget", () => { + const loop = new AgentLoop({ maxToolCallsPerTurn: 1 }); + + loop.beginTurn(); + loop.recordToolCall("readFile", { path: "a" }); + expect(loop.recordToolCall("readFile", { path: "b" }).action).toBe("stop"); + + const fresh = new AgentLoop({ maxToolCallsPerTurn: 1 }); + fresh.beginTurn(); + fresh.recordToolCall("readFile", { path: "a" }); + fresh.beginTurn(); + expect(fresh.recordToolCall("readFile", { path: "b" }).action).toBe("continue"); + }); +}); + +describe("AgentLoop repeat guard", () => { + test("stops when an identical call repeats consecutively beyond the threshold", () => { + const loop = new AgentLoop({ maxConsecutiveRepeats: 3 }); + loop.beginTurn(); + + for (let i = 0; i < 3; i += 1) { + expect(loop.recordToolCall("bash", { command: "ls" }).action).toBe("continue"); + } + + expect(loop.recordToolCall("bash", { command: "ls" })).toMatchObject({ + action: "stop", + reason: "no-progress", + }); + }); + + test("read → edit → read is normal work, not a loop", () => { + const loop = new AgentLoop({ maxConsecutiveRepeats: 1 }); + loop.beginTurn(); + + expect(loop.recordToolCall("readFile", { path: "a" }).action).toBe("continue"); + expect(loop.recordToolCall("editFile", { path: "a" }).action).toBe("continue"); + expect(loop.recordToolCall("readFile", { path: "a" }).action).toBe("continue"); + }); + + test("same tool with different input does not count as a repeat", () => { + const loop = new AgentLoop({ maxConsecutiveRepeats: 1 }); + loop.beginTurn(); + + expect(loop.recordToolCall("readFile", { path: "a" }).action).toBe("continue"); + expect(loop.recordToolCall("readFile", { path: "b" }).action).toBe("continue"); + expect(loop.recordToolCall("readFile", { path: "c" }).action).toBe("continue"); + }); + + test("signature is insensitive to key order", () => { + expect(toolSignature("grep", { pattern: "x", path: "a" })).toBe( + toolSignature("grep", { path: "a", pattern: "x" }), + ); + expect(toolSignature("grep", { pattern: "x" })).not.toBe( + toolSignature("grep", { pattern: "y" }), + ); + }); +}); + +describe("AgentLoop lifecycle", () => { + test("a new user message resets the budget", () => { + const loop = new AgentLoop({ maxTurns: 1 }); + + loop.beginUserMessage("user-1"); + loop.beginTurn(); + expect(loop.beginTurn().action).toBe("stop"); + + loop.beginUserMessage("user-2"); + expect(loop.beginTurn().action).toBe("continue"); + }); + + test("regenerating the same message keeps the budget", () => { + const loop = new AgentLoop({ maxTurns: 1 }); + + loop.beginUserMessage("user-1"); + loop.beginTurn(); + loop.beginTurn(); + loop.beginUserMessage("user-1"); + + expect(loop.stopped).toBe("max-turns"); + }); + + test("reset() clears a stop so an explicit retry can proceed", () => { + const loop = new AgentLoop({ maxTurns: 1 }); + + loop.beginUserMessage("user-1"); + loop.beginTurn(); + loop.beginTurn(); + + loop.reset(); + expect(loop.stopped).toBeNull(); + expect(loop.beginTurn().action).toBe("continue"); + }); + + test("abort() is sticky", () => { + const loop = new AgentLoop(); + loop.beginUserMessage("user-1"); + loop.beginTurn(); + loop.abort(); + + expect(loop.stopped).toBe("aborted"); + expect(loop.beginTurn()).toMatchObject({ action: "stop", reason: "aborted" }); + }); + + test("snapshot reports budget usage", () => { + const loop = new AgentLoop({ maxTurns: 5, maxToolCallsPerTurn: 4 }); + loop.beginUserMessage("user-1"); + loop.beginTurn(); + loop.recordToolCall("readFile", { path: "a" }); + loop.recordToolCall("readFile", { path: "b" }); + + expect(loop.snapshot()).toEqual({ + turn: 1, + maxTurns: 5, + toolCallsThisTurn: 2, + totalToolCalls: 2, + stopReason: null, + }); + }); + + test("defaults are exposed for the TUI status line", () => { + expect(DEFAULT_AGENT_LIMITS.maxTurns).toBeGreaterThan(0); + }); +}); + +describe("formatAgentProgress", () => { + test("renders nothing before a run starts", () => { + expect(formatAgentProgress(null)).toBeNull(); + expect(formatAgentProgress({ turn: 0, maxTurns: 50, toolCallsThisTurn: 0 })).toBeNull(); + }); + + test("renders the turn budget", () => { + expect(formatAgentProgress({ turn: 3, maxTurns: 50, toolCallsThisTurn: 0 })).toBe("turn 4/51"); + }); + + test("appends the tool count once the turn has run tools", () => { + expect(formatAgentProgress({ turn: 3, maxTurns: 50, toolCallsThisTurn: 4 })).toBe( + "turn 4/51 · 4 tools", + ); + }); + + test("uses the singular label for a single tool call", () => { + expect(formatAgentProgress({ turn: 1, maxTurns: 50, toolCallsThisTurn: 1 })).toBe( + "turn 2/51 · 1 tool", + ); + }); +}); diff --git a/packages/cli/src/lib/agent-loop.ts b/packages/cli/src/lib/agent-loop.ts new file mode 100644 index 0000000..234f782 --- /dev/null +++ b/packages/cli/src/lib/agent-loop.ts @@ -0,0 +1,256 @@ +/** + * Explicit agent-loop controller. + * + * Before this module, MoCode's multi-step loop was implicit: the AI SDK's + * `sendAutomaticallyWhen` re-posted whenever the last assistant message had no + * pending tool calls. That produces a working loop but leaves three questions a + * production agent must be able to answer unanswered: + * + * 1. How much work may one user message spend? → turn + per-turn tool budgets + * 2. When is the agent stuck rather than working? → consecutive-repeat guard + * 3. Why did it stop? → named stop reasons + * + * One *turn* is one model response plus the tool calls it requested. The first + * response after a user message is turn 1. {@link AgentLoop.beginTurn} runs when + * the loop decides to continue, so `maxTurns` bounds *continued* turns: with + * `maxTurns: 50` a single user message can produce at most 51 model responses. + * + * Why 50 (see `docs/decisions.md`): published defaults span a wide range — + * OpenAI Agents SDK caps at 10, Claude Agent SDK leaves it unbounded, and + * LangGraph 1.x defaults its recursion limit to 10007 after finding 25 too tight + * for real tasks. A coding agent needs headroom, but "unbounded" turns a bug into + * an unbounded bill, so this stays a budget. + * + * Pure module: no React, no network, no filesystem — unit-testable in isolation. + */ + +/** Why the loop stopped. Every exit is named so the TUI can explain it. */ +export type AgentStopReason = + | "max-turns" + | "max-tool-calls" + | "no-progress" + | "aborted" + | "error"; + +/** Budgets for one user message. */ +export type AgentLoopLimits = { + /** Model responses the loop may continue to. */ + maxTurns: number; + /** Tool calls allowed inside a single turn. */ + maxToolCallsPerTurn: number; + /** Identical consecutive tool calls tolerated before declaring no progress. */ + maxConsecutiveRepeats: number; +}; + +export const DEFAULT_AGENT_LIMITS: AgentLoopLimits = { + maxTurns: 50, + // A circuit breaker, not a budget: measured turns use 2-3 tools (max 2.33). + maxToolCallsPerTurn: 16, + maxConsecutiveRepeats: 3, +}; + +/** + * Operator overrides for the budgets. + * + * Undefined values are dropped rather than spread, so a missing env var cannot + * blank out a default with `undefined`. + */ +export function agentLimitsFromEnv(): Partial { + const read = (name: string): number | undefined => { + const parsed = Number(process.env[name]); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : undefined; + }; + + const limits: Partial = {}; + const maxTurns = read("MOCODE_MAX_TURNS"); + const maxToolCallsPerTurn = read("MOCODE_MAX_TOOL_CALLS"); + const maxConsecutiveRepeats = read("MOCODE_MAX_REPEATS"); + + if (maxTurns !== undefined) limits.maxTurns = maxTurns; + if (maxToolCallsPerTurn !== undefined) limits.maxToolCallsPerTurn = maxToolCallsPerTurn; + if (maxConsecutiveRepeats !== undefined) limits.maxConsecutiveRepeats = maxConsecutiveRepeats; + + return limits; +} + +export type AgentLoopDecision = + | { action: "continue" } + | { action: "stop"; reason: AgentStopReason; detail: string }; + +export type AgentLoopSnapshot = { + turn: number; + maxTurns: number; + toolCallsThisTurn: number; + totalToolCalls: number; + stopReason: AgentStopReason | null; +}; + +/** Reactive subset rendered by the status line ({components.status-bar} in DESIGN.md). */ +export type AgentProgress = Pick< + AgentLoopSnapshot, + "turn" | "maxTurns" | "toolCallsThisTurn" +>; + +/** + * Status-line label for the loop budget, or `null` when no run has started. + * One dim segment: `turn 3/51`, plus `· 4 tools` when the turn has run tools. + */ +export function formatAgentProgress(progress: AgentProgress | null): string | null { + if (!progress || progress.turn <= 0) return null; + + // `turn` counts *continuations*: the first response is already spent by the time + // the loop decides to continue, and `maxTurns` bounds continuations. Showing the + // raw counter labelled "turn" under-reported both the spend and the budget. + const responses = progress.turn + 1; + const budget = progress.maxTurns + 1; + const base = `turn ${responses}/${budget}`; + if (progress.toolCallsThisTurn <= 0) return base; + const label = progress.toolCallsThisTurn === 1 ? "tool" : "tools"; + return `${base} · ${progress.toolCallsThisTurn} ${label}`; +} + +/** + * Stable serialization used for tool-call signatures. Key order in model-emitted + * JSON is not stable, so a plain `JSON.stringify` would miss true repeats. + */ +function stableKey(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null"; + if (Array.isArray(value)) return `[${value.map(stableKey).join(",")}]`; + + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => `${JSON.stringify(k)}:${stableKey(v)}`); + + return `{${entries.join(",")}}`; +} + +/** Signature of one tool call: same tool + same normalized input. */ +export function toolSignature(toolName: string, input: unknown): string { + return `${toolName}:${stableKey(input)}`; +} + +export class AgentLoop { + private readonly limits: AgentLoopLimits; + private turn = 0; + private toolCallsThisTurn = 0; + private totalToolCalls = 0; + private stopReason: AgentStopReason | null = null; + private stopDetail = ""; + private userMessageId: string | null = null; + private lastSignature: string | null = null; + private consecutiveRepeats = 0; + + constructor(limits: Partial = {}) { + this.limits = { ...DEFAULT_AGENT_LIMITS, ...agentLimitsFromEnv(), ...limits }; + } + + /** + * Start (or continue) the loop for a user message. A different id starts a + * fresh budget; the same id keeps the current one, so regenerating a reply does + * not silently hand the model an extra 50 turns. + */ + beginUserMessage(messageId: string): void { + if (this.userMessageId === messageId) return; + this.userMessageId = messageId; + this.reset(); + } + + /** Clear budgets and stop state, keeping the current user message id. */ + reset(): void { + this.turn = 0; + this.toolCallsThisTurn = 0; + this.totalToolCalls = 0; + this.stopReason = null; + this.stopDetail = ""; + this.lastSignature = null; + this.consecutiveRepeats = 0; + } + + /** Ask permission to start the next model turn. */ + beginTurn(): AgentLoopDecision { + const stopped = this.stoppedDecision(); + if (stopped) return stopped; + + if (this.turn >= this.limits.maxTurns) { + return this.fail( + "max-turns", + `Stopped after ${this.limits.maxTurns} turns without a final answer.`, + ); + } + + this.turn += 1; + this.toolCallsThisTurn = 0; + return { action: "continue" }; + } + + /** Account for one tool call before it executes. */ + recordToolCall(toolName: string, input: unknown): AgentLoopDecision { + const stopped = this.stoppedDecision(); + if (stopped) return stopped; + + if (this.toolCallsThisTurn >= this.limits.maxToolCallsPerTurn) { + return this.fail( + "max-tool-calls", + `Stopped after ${this.limits.maxToolCallsPerTurn} tool calls in one turn.`, + ); + } + + this.toolCallsThisTurn += 1; + this.totalToolCalls += 1; + + const signature = toolSignature(toolName, input); + // Only *consecutive* identical calls count. read → edit → read is normal + // work; read → read → read → read with identical input is a loop. + this.consecutiveRepeats = signature === this.lastSignature ? this.consecutiveRepeats + 1 : 1; + this.lastSignature = signature; + + if (this.consecutiveRepeats > this.limits.maxConsecutiveRepeats) { + return this.fail( + "no-progress", + `Stopped: ${toolName} was called ${this.consecutiveRepeats} times in a row with identical input.`, + ); + } + + return { action: "continue" }; + } + + /** Escape hatch for user interruption (Esc). */ + abort(): void { + this.fail("aborted", "Stopped by user."); + } + + /** Escape hatch for a transport or tool failure that ends the run. */ + failWith(reason: AgentStopReason, detail: string): AgentLoopDecision { + return this.fail(reason, detail); + } + + get stopped(): AgentStopReason | null { + return this.stopReason; + } + + snapshot(): AgentLoopSnapshot { + return { + turn: this.turn, + maxTurns: this.limits.maxTurns, + toolCallsThisTurn: this.toolCallsThisTurn, + totalToolCalls: this.totalToolCalls, + stopReason: this.stopReason, + }; + } + + private fail(reason: AgentStopReason, detail: string): AgentLoopDecision { + // Sticky: once stopped, every later decision repeats the same reason. + if (this.stopReason === null) { + this.stopReason = reason; + this.stopDetail = detail; + } + + return { action: "stop", reason: this.stopReason, detail: this.stopDetail }; + } + + private stoppedDecision(): AgentLoopDecision | null { + if (this.stopReason === null) return null; + return { action: "stop", reason: this.stopReason, detail: this.stopDetail }; + } +} diff --git a/packages/cli/src/lib/agent-phase.test.ts b/packages/cli/src/lib/agent-phase.test.ts new file mode 100644 index 0000000..749af79 --- /dev/null +++ b/packages/cli/src/lib/agent-phase.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from "bun:test"; +import { + deriveAgentPhase, + isRunActive, + formatAgentPhase, + formatElapsed, + type PhaseMessage, +} from "./agent-phase"; + +function message(parts: unknown[]): PhaseMessage { + return { parts }; +} + +const pendingTool = { type: "tool-bash", state: "input-available" }; +const doneTool = { type: "tool-bash", state: "output-available" }; +const reasoning = { type: "reasoning" }; +const text = { type: "text" }; + +describe("isRunActive", () => { + test("status alone does not decide it", () => { + // The loop is client-driven: between a model step and the auto-resubmit the + // status is `ready` while tools are still executing. + expect(isRunActive({ status: "ready", message: message([pendingTool]) })).toBe(true); + }); + + test("idle once the last tool call is resolved", () => { + expect(isRunActive({ status: "ready", message: message([doneTool]) })).toBe(false); + }); + + test("streaming is active even with no parts yet", () => { + expect(isRunActive({ status: "streaming" })).toBe(true); + }); +}); + +describe("deriveAgentPhase", () => { + test("stays 'running' in the gap between a step and the resubmit", () => { + expect(deriveAgentPhase({ status: "ready", message: message([pendingTool]) })).toBe("running"); + }); + + test("idle when the chat is not running", () => { + expect(deriveAgentPhase({ status: "ready", message: message([text]) })).toBeNull(); + expect(deriveAgentPhase({ status: "error" })).toBeNull(); + }); + + test("waiting for the first token", () => { + expect(deriveAgentPhase({ status: "submitted" })).toBe("waiting"); + expect(deriveAgentPhase({ status: "streaming", message: message([]) })).toBe("waiting"); + }); + + test("thinking while only reasoning has streamed", () => { + expect(deriveAgentPhase({ status: "streaming", message: message([reasoning]) })).toBe( + "thinking", + ); + }); + + test("generating once text appears", () => { + expect(deriveAgentPhase({ status: "streaming", message: message([reasoning, text]) })).toBe( + "generating", + ); + }); + + test("running wins over text — the agent is acting, not talking", () => { + expect( + deriveAgentPhase({ status: "streaming", message: message([text, pendingTool]) }), + ).toBe("running"); + }); + + test("a resolved tool no longer counts as running", () => { + expect(deriveAgentPhase({ status: "streaming", message: message([text, doneTool]) })).toBe( + "generating", + ); + }); + + test("dynamic MCP calls count as running too", () => { + expect( + deriveAgentPhase({ + status: "streaming", + message: message([{ type: "dynamic-tool", state: "input-available" }]), + }), + ).toBe("running"); + }); + + test("tool parts with no state yet are treated as pending", () => { + expect( + deriveAgentPhase({ status: "streaming", message: message([{ type: "tool-readFile" }]) }), + ).toBe("running"); + }); +}); + +describe("formatElapsed", () => { + test("shows tenths under a minute", () => { + expect(formatElapsed(2_400)).toBe("2.4s"); + expect(formatElapsed(0)).toBe("0.0s"); + }); + + test("switches to minutes past 60s, like the harness does", () => { + expect(formatElapsed(174_000)).toBe("2m54s"); + }); + + test("never goes negative", () => { + expect(formatElapsed(-500)).toBe("0.0s"); + }); +}); + +describe("formatAgentPhase", () => { + test("idle renders nothing", () => { + expect(formatAgentPhase(null, 1_000)).toBeNull(); + }); + + test("shows the phase with elapsed time", () => { + expect(formatAgentPhase("running", 2_400)).toBe("执行中 2.4s"); + expect(formatAgentPhase("thinking", 1_000)).toBe("思考中 1.0s"); + }); + + test("omits the timer when the elapsed time is unknown", () => { + expect(formatAgentPhase("waiting", null)).toBe("等待首字"); + }); +}); diff --git a/packages/cli/src/lib/agent-phase.ts b/packages/cli/src/lib/agent-phase.ts new file mode 100644 index 0000000..ecc0d43 --- /dev/null +++ b/packages/cli/src/lib/agent-phase.ts @@ -0,0 +1,120 @@ +/** + * What the agent is doing *right now*. + * + * The transcript already records what happened; this answers the question a user + * asks while waiting: "is it thinking, or stuck?" The phase is derived from the + * live message parts rather than tracked separately, so it can never disagree with + * what is on screen. + * + * Pure module: no timers, no I/O. + */ + +export type AgentPhase = "waiting" | "thinking" | "running" | "generating"; + +/** Minimum shape needed from a UI message — keeps this testable without the SDK. */ +export type PhaseMessage = { + parts?: unknown; +}; + +type PartLike = { + type?: unknown; + state?: unknown; +}; + +function partsOf(message: PhaseMessage | null | undefined): PartLike[] { + if (!message || !Array.isArray(message.parts)) return []; + return message.parts as PartLike[]; +} + +/** A tool part that has been requested but not yet resolved. */ +function hasPendingTool(parts: PartLike[]): boolean { + return parts.some((part) => { + const type = typeof part.type === "string" ? part.type : ""; + const isTool = type === "dynamic-tool" || type.startsWith("tool-"); + if (!isTool) return false; + + const state = typeof part.state === "string" ? part.state : ""; + return state !== "output-available" && state !== "output-error"; + }); +} + +function hasPart(parts: PartLike[], type: string): boolean { + return parts.some((part) => part.type === type); +} + +/** + * Whether a run is in progress. + * + * `chat.status` alone is not enough: the loop is client-driven, so between a model + * step and the auto-resubmit the status is `ready` even though the agent is busy + * executing tools. Keying "active" on the status made the phase disappear exactly + * during tool execution — the one moment a user most wants to see it. + */ +export function isRunActive(params: { + /** Chat status from the SDK. */ + status: string; + /** The message being streamed or awaiting tool results. */ + message?: PhaseMessage | null; +}): boolean { + if (params.status === "submitted" || params.status === "streaming") return true; + + return hasPendingTool(partsOf(params.message)); +} + +/** + * Derive the current phase, or `null` when the agent is idle. + * + * Precedence is deliberate: a pending tool means the agent is *acting*, which is + * more useful to show than the text that preceded it. + */ +export function deriveAgentPhase(params: { + /** Chat status from the SDK. */ + status: string; + /** The message being streamed, if any. */ + message?: PhaseMessage | null; +}): AgentPhase | null { + if (!isRunActive(params)) return null; + + const parts = partsOf(params.message); + if (parts.length === 0) return "waiting"; + + if (hasPendingTool(parts)) return "running"; + + const hasText = hasPart(parts, "text"); + const hasReasoning = hasPart(parts, "reasoning"); + + if (hasReasoning && !hasText) return "thinking"; + if (hasText) return "generating"; + + return "waiting"; +} + +/** Human label per phase. Short on purpose — it lives in a one-row status line. */ +const PHASE_LABEL: Record = { + waiting: "等待首字", + thinking: "思考中", + running: "执行中", + generating: "生成中", +}; + +/** Seconds with one decimal, e.g. `2.4s`. */ +export function formatElapsed(ms: number): string { + const seconds = Math.max(0, ms) / 1000; + return seconds < 60 ? `${seconds.toFixed(1)}s` : `${Math.floor(seconds / 60)}m${Math.round(seconds % 60)}s`; +} + +/** + * Status-line segment, e.g. `执行中 2.4s`, or `null` when idle. + * + * `elapsedMs` is omitted from the label when unknown so the line does not show a + * misleading `0.0s` for work that has only just started. + */ +export function formatAgentPhase( + phase: AgentPhase | null, + elapsedMs: number | null, +): string | null { + if (!phase) return null; + + const label = PHASE_LABEL[phase]; + return elapsedMs === null ? label : `${label} ${formatElapsed(elapsedMs)}`; +} diff --git a/packages/cli/src/lib/audit-log.rotation.test.ts b/packages/cli/src/lib/audit-log.rotation.test.ts new file mode 100644 index 0000000..4e9de45 --- /dev/null +++ b/packages/cli/src/lib/audit-log.rotation.test.ts @@ -0,0 +1,121 @@ +/** + * Audit log rotation. + * + * Without rotation a long-lived agent grows `audit.jsonl` without bound — the + * audit trail becomes the outage. These tests pin the two properties that matter: + * the file stays under the limit, and the archive count stays fixed. + */ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { auditRotation, recordAudit, rotateIfNeeded } from "./audit-log"; + +const dir = mkdtempSync(join(tmpdir(), "mocode-audit-rotate-")); +const logPath = join(dir, "audit.jsonl"); + +beforeAll(() => { + process.env.MOCODE_AUDIT_PATH = logPath; + // Tiny limits so a handful of entries trigger rotation. + process.env.MOCODE_AUDIT_MAX_BYTES = "400"; + process.env.MOCODE_AUDIT_MAX_FILES = "2"; +}); + +afterEach(() => { + for (let index = 0; index <= 4; index += 1) { + rmSync(`${logPath}.${index}`, { force: true }); + } + rmSync(logPath, { force: true }); +}); + +afterAll(() => { + delete process.env.MOCODE_AUDIT_PATH; + delete process.env.MOCODE_AUDIT_MAX_BYTES; + delete process.env.MOCODE_AUDIT_MAX_FILES; + rmSync(dir, { recursive: true, force: true }); +}); + +function write(count: number) { + for (let index = 0; index < count; index += 1) { + recordAudit({ tool: "bash", mode: "BUILD", outcome: "ok", detail: `n=${index}` }); + } +} + +describe("policy", () => { + test("defaults are read from the environment when set", () => { + expect(auditRotation()).toEqual({ maxBytes: 400, maxFiles: 2 }); + }); + + test("invalid values fall back to the defaults, never to zero", () => { + process.env.MOCODE_AUDIT_MAX_BYTES = "nonsense"; + process.env.MOCODE_AUDIT_MAX_FILES = "-3"; + try { + const rotation = auditRotation(); + expect(rotation.maxBytes).toBeGreaterThan(0); + expect(rotation.maxFiles).toBeGreaterThan(0); + } finally { + process.env.MOCODE_AUDIT_MAX_BYTES = "400"; + process.env.MOCODE_AUDIT_MAX_FILES = "2"; + } + }); +}); + +describe("rotation", () => { + test("does nothing while the file is under the limit", () => { + writeFileSync(logPath, "x".repeat(50)); + + expect(rotateIfNeeded(logPath)).toBe(false); + expect(existsSync(`${logPath}.1`)).toBe(false); + }); + + test("moves the file aside once it reaches the limit", () => { + writeFileSync(logPath, "x".repeat(600)); + + expect(rotateIfNeeded(logPath)).toBe(true); + expect(existsSync(`${logPath}.1`)).toBe(true); + expect(existsSync(logPath)).toBe(false); + }); + + test("returns false when there is no file yet", () => { + expect(rotateIfNeeded(logPath)).toBe(false); + }); + + test("a burst of writes stays bounded and keeps the newest data", () => { + write(40); + + // Active file must be under the limit — that is the whole point. + const activeSize = existsSync(logPath) ? readFileSync(logPath, "utf8").length : 0; + expect(activeSize).toBeLessThan(400 + 200); // limit + one line of slack + + // The most recent entry survives somewhere in the chain. + const all = [ + logPath, + `${logPath}.1`, + `${logPath}.2`, + ] + .filter((path) => existsSync(path)) + .map((path) => readFileSync(path, "utf8")) + .join(""); + + expect(all).toContain("n=39"); + }); + + test("never keeps more archives than maxFiles", () => { + write(60); + + expect(existsSync(`${logPath}.3`)).toBe(false); + expect(existsSync(`${logPath}.2`)).toBe(true); + }); + + test("every archive line is still valid JSON", () => { + write(30); + + for (const path of [logPath, `${logPath}.1`, `${logPath}.2`]) { + if (!existsSync(path)) continue; + + for (const line of readFileSync(path, "utf8").trim().split("\n").filter(Boolean)) { + expect(() => JSON.parse(line)).not.toThrow(); + } + } + }); +}); diff --git a/packages/cli/src/lib/audit-log.test.ts b/packages/cli/src/lib/audit-log.test.ts new file mode 100644 index 0000000..97c8c0a --- /dev/null +++ b/packages/cli/src/lib/audit-log.test.ts @@ -0,0 +1,151 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { defaultAuditPath, recordAudit, summarizeInput } from "./audit-log"; + +const dir = mkdtempSync(join(tmpdir(), "mocode-audit-")); +const logPath = join(dir, "audit.jsonl"); + +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +function lines(): Array> { + return readFileSync(logPath, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); +} + +describe("summarizeInput", () => { + test("keeps short scalars that identify the call", () => { + expect(summarizeInput({ path: "src/a.ts", line_start: 3 })).toBe("path=src/a.ts line_start=3"); + }); + + test("never records file contents", () => { + const body = "x".repeat(5_000); + expect(summarizeInput({ path: "a.ts", content: body })).toBe("path=a.ts"); + }); + + test("records the command — an audit entry without it cannot be reviewed", () => { + expect(summarizeInput({ command: "cat ~/.ssh/id_rsa", description: "list" })).toBe( + "command=cat ~/.ssh/id_rsa description=list", + ); + }); + + test("masks credentials that appear inside a command", () => { + expect(summarizeInput({ command: "export ANTHROPIC_API_KEY=abcd1234efgh" })).not.toContain( + "abcd1234efgh", + ); + expect(summarizeInput({ command: 'curl -H "Authorization: Bearer sk-live-abcdef123456"' })).not.toContain( + "sk-live-abcdef123456", + ); + expect(summarizeInput({ command: "sk-abcdefghijklmnop tool" })).not.toContain("abcdefghijklmnop"); + }); + + test("masks X-API-Key and other credential headers", () => { + expect( + summarizeInput({ command: 'curl -H "X-API-Key: sk-live-abcdef123456" https://x' }), + ).not.toContain("sk-live-abcdef123456"); + + expect(summarizeInput({ command: "curl -H 'X-Api-Key: plainsecretvalue' https://x" })) + .not.toContain("plainsecretvalue"); + }); + + test("never records edit payloads", () => { + expect( + summarizeInput({ path: "a.ts", oldString: "secret", newString: "other" }), + ).toBe("path=a.ts"); + }); + + test("drops multi-line values even when short", () => { + expect(summarizeInput({ path: "a.ts", description: "a\nb" })).toBe("path=a.ts"); + }); + + test("is bounded regardless of how many keys arrive", () => { + const input = Object.fromEntries( + Array.from({ length: 50 }, (_, i) => [`path${i}`, "v".repeat(50)]), + ); + // Unknown keys are ignored; only the allowlisted `path` field is kept. + expect(summarizeInput({ ...input, path: "src/a.ts" }).length).toBeLessThanOrEqual(240); + expect(summarizeInput({ ...input, path: "src/a.ts" })).toContain("path=src/a.ts"); + }); + + test("non-object input yields nothing", () => { + expect(summarizeInput(null)).toBe(""); + expect(summarizeInput("raw")).toBe(""); + }); +}); + +describe("recordAudit", () => { + test("appends one JSON line per decision", () => { + recordAudit({ tool: "readFile", mode: "BUILD", outcome: "ok", detail: "path=a.ts" }, logPath); + recordAudit( + { tool: "readFile", mode: "BUILD", outcome: "denied", code: "PATH_ESCAPE", detail: "path=/etc" }, + logPath, + ); + + const entries = lines(); + expect(entries).toHaveLength(2); + expect(entries[0]).toMatchObject({ tool: "readFile", outcome: "ok" }); + expect(entries[1]).toMatchObject({ outcome: "denied", code: "PATH_ESCAPE" }); + }); + + test("stamps every entry with an ISO timestamp", () => { + recordAudit({ tool: "bash", mode: "BUILD", outcome: "denied", code: "HARD_DENY" }, logPath); + + const entry = lines().at(-1); + expect(typeof entry?.timestamp).toBe("string"); + expect(Number.isNaN(Date.parse(String(entry?.timestamp)))).toBe(false); + }); + + test("creates the directory when it does not exist", () => { + const nested = join(dir, "deep", "nested", "audit.jsonl"); + expect(recordAudit({ tool: "grep", mode: "BUILD", outcome: "ok" }, nested)).toBe(true); + }); + + test("fails soft instead of throwing when the path is unusable", () => { + // A directory cannot be appended to as a file. + expect(recordAudit({ tool: "bash", mode: "BUILD", outcome: "ok" }, dir)).toBe(false); + }); + + test("honours MOCODE_AUDIT_PATH", () => { + process.env.MOCODE_AUDIT_PATH = logPath; + try { + expect(defaultAuditPath()).toBe(logPath); + } finally { + delete process.env.MOCODE_AUDIT_PATH; + } + }); +}); + +describe("shell commands are auditable", () => { + test("the command itself is recorded — an entry without it is useless", () => { + expect(summarizeInput({ command: "head -1 /etc/passwd" })).toBe("command=head -1 /etc/passwd"); + }); + + test("multi-line commands are collapsed, not dropped", () => { + const summary = summarizeInput({ command: "ls -la\nfile secret.txt" }); + expect(summary).toBe("command=ls -la file secret.txt"); + }); + + test("a command containing a heredoc is capped, not logged in full", () => { + const body = "cat <<'X'\n" + "y".repeat(2_000) + "\nX"; + const summary = summarizeInput({ command: body }); + expect(summary.length).toBeLessThanOrEqual(240); + expect(summary.startsWith("command=")).toBe(true); + }); + + test("file bodies are still dropped", () => { + expect(summarizeInput({ path: "a.ts", content: "x".repeat(500) })).toBe("path=a.ts"); + }); + + test("command and description both survive", () => { + const summary = summarizeInput({ + command: "ls -la secret.txt", + description: "Inspect link", + }); + expect(summary).toContain("command=ls -la secret.txt"); + expect(summary).toContain("description=Inspect link"); + }); +}); diff --git a/packages/cli/src/lib/audit-log.ts b/packages/cli/src/lib/audit-log.ts new file mode 100644 index 0000000..f2ebbbc --- /dev/null +++ b/packages/cli/src/lib/audit-log.ts @@ -0,0 +1,247 @@ +/** + * Append-only audit log for tool authorization decisions. + * + * Enterprise requirement, and the reason the executor — not the model — is the + * authority: every call must leave a record of *what was attempted* and *what was + * decided*, whether it ran or was refused. A refusal that nobody can audit is + * indistinguishable from a bug. + * + * Design constraints: + * + * 1. **Never log file contents.** Only short, single-line scalar arguments are + * recorded, capped hard. An audit log that captures source code is a data leak + * with extra steps. + * 2. **Never break a tool call.** Auditing is best-effort: a full disk or a + * read-only home must not turn a working tool into a failing one. + * 3. **One line per decision, JSONL** — append-only and greppable, so `jq` and log + * shippers work without a parser. + */ +import { appendFileSync, chmodSync, mkdirSync, renameSync, rmSync, statSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join } from "node:path"; + +/** Owner-only directory mode for the audit trail. */ +const AUDIT_DIR_MODE = 0o700; +/** Owner-only file mode for the active log and rotated archives. */ +const AUDIT_FILE_MODE = 0o600; + +/** Best-effort chmod — never fails the tool call that triggered the write. */ +function ownerOnly(path: string, mode: number): void { + try { + chmodSync(path, mode); + } catch { + // Platform/umask may refuse; auditing remains best-effort. + } +} + +/** Rotate once the active file reaches this size. */ +const DEFAULT_MAX_BYTES = 5 * 1024 * 1024; +/** Archives kept alongside the active file (`audit.jsonl.1` … `.N`). */ +const DEFAULT_MAX_FILES = 5; + +export type AuditRotation = { + maxBytes: number; + maxFiles: number; +}; + +/** Read positive numbers from the environment, falling back to the defaults. */ +function readPositive(value: string | undefined, fallback: number): number { + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +/** Rotation policy, overridable for tests and for long-lived agents. */ +export function auditRotation(): AuditRotation { + return { + maxBytes: readPositive(process.env.MOCODE_AUDIT_MAX_BYTES, DEFAULT_MAX_BYTES), + maxFiles: readPositive(process.env.MOCODE_AUDIT_MAX_FILES, DEFAULT_MAX_FILES), + }; +} + +function rotatedName(path: string, index: number): string { + return `${path}.${index}`; +} + +/** + * Move the active file aside once it exceeds `maxBytes`. + * + * Archives are numbered newest-first (`.1` is the most recent) and the oldest is + * dropped, so a chatty agent costs a bounded amount of disk and never grows the + * log indefinitely. Returns `true` when a rotation happened. + * + * Best-effort like the rest of this module: a failed rotation leaves the file in + * place and must not fail the tool call that triggered it. + */ +export function rotateIfNeeded( + path: string = defaultAuditPath(), + rotation: AuditRotation = auditRotation(), +): boolean { + let size: number; + + try { + size = statSync(path).size; + } catch { + // No file yet — nothing to rotate. + return false; + } + + if (size < rotation.maxBytes) return false; + + try { + // Drop the oldest archive before shifting, so the chain never exceeds maxFiles. + rmSync(rotatedName(path, rotation.maxFiles), { force: true }); + + for (let index = rotation.maxFiles - 1; index >= 1; index -= 1) { + try { + const from = rotatedName(path, index); + const to = rotatedName(path, index + 1); + renameSync(from, to); + ownerOnly(to, AUDIT_FILE_MODE); + } catch { + // That archive does not exist yet — fine. + } + } + + const archive = rotatedName(path, 1); + renameSync(path, archive); + ownerOnly(archive, AUDIT_FILE_MODE); + return true; + } catch { + return false; + } +} + +export type AuditOutcome = "ok" | "denied" | "error"; + +export type AuditEvent = { + /** ISO-8601, UTC. */ + timestamp: string; + tool: string; + mode: string; + outcome: AuditOutcome; + /** Machine-readable refusal code when `outcome` is `denied`. */ + code?: string; + /** Short summary of the call — never full argument payloads. */ + /** For shell calls: whether an OS sandbox confined the child process. */ + shellConfined?: boolean; + detail?: string; +}; + +/** Where decisions are appended. `MOCODE_AUDIT_PATH` exists for tests and CI. */ +export function defaultAuditPath(): string { + return process.env.MOCODE_AUDIT_PATH ?? join(homedir(), ".mocode", "audit.jsonl"); +} + +/** Longest single value kept in the log. */ +const MAX_VALUE_LENGTH = 120; +/** Longest total summary kept in the log. */ +const MAX_DETAIL_LENGTH = 240; +/** + * Commands get their own, larger budget. + * + * Auditing a shell call without the command is pointless, and shell commands are + * routinely multi-line. They are collapsed and truncated instead of dropped — the + * cap still bounds how much of a heredoc can leak. + */ +const MAX_COMMAND_LENGTH = 160; + +/** + * Secrets that may appear *inside* a command. + * + * The audit needs the command to be reviewable, but a command is user-controlled + * text and can carry credentials. Masking known shapes beats both extremes: + * logging raw (leaks) and dropping the command (useless). + */ +const SECRET_PATTERNS: ReadonlyArray<[RegExp, string]> = [ + [/(\bsk-[A-Za-z0-9_-]{8,})/g, "sk-***"], + [/(\bBearer\s+)[A-Za-z0-9._~+/-]{8,}/gi, "$1***"], + [/\b((?:[A-Z0-9_]*(?:API_KEY|TOKEN|SECRET|PASSWORD)[A-Z0-9_]*)=)(\S+)/g, "$1***"], + [/(Authorization:\s*)\S+/gi, "$1***"], + // Any header-ish `Name: value` where the name looks like a credential. + [/([\w-]*(?:API[-_]?Key|ApiKey|Token|Secret|Password)[\w-]*\s*:\s*)[^\s"']+/gi, "$1***"], +]; + +/** Mask credential-shaped substrings before a command reaches the log. */ +export function redactSecrets(value: string): string { + let redacted = value; + for (const [pattern, replacement] of SECRET_PATTERNS) { + redacted = redacted.replace(pattern, replacement); + } + return redacted; +} + +/** + * Explicitly approved string fields. Everything else — file bodies, credentials, + * bash commands, edit payloads, free-form prompts — is omitted. + */ +const SAFE_STRING_KEYS = new Set([ + "path", + "pattern", + "include", + "ref", + "description", + "subagent_type", + // A shell audit entry without its command cannot be reviewed. It is collapsed + // and capped rather than dropped — see MAX_COMMAND_LENGTH. + "command", +]); + +/** + * Reduce tool input to a short, single-line summary of approved fields only. + * + * Strings longer than {@link MAX_VALUE_LENGTH} or containing newlines are dropped + * entirely rather than truncated — a truncated file body is still file body. + */ +export function summarizeInput(input: unknown): string { + if (input === null || typeof input !== "object") return ""; + + const parts: string[] = []; + + for (const [key, value] of Object.entries(input as Record)) { + if (typeof value === "number" || typeof value === "boolean") { + parts.push(`${key}=${value}`); + continue; + } + + if (typeof value !== "string" || !SAFE_STRING_KEYS.has(key)) continue; + + if (key === "command") { + const collapsed = redactSecrets(value).replace(/\s+/g, " ").trim(); + if (collapsed) parts.push(`${key}=${collapsed.slice(0, MAX_COMMAND_LENGTH)}`); + continue; + } + + if (value.length <= MAX_VALUE_LENGTH && !value.includes("\n")) { + parts.push(`${key}=${value}`); + } + } + + return parts.join(" ").slice(0, MAX_DETAIL_LENGTH); +} + +/** + * Append one decision. Returns `true` when written, `false` when the write was + * skipped or failed — never throws. + * + * Rotation is checked immediately before the append, so the log is bounded from + * the first write without a background worker. + */ +export function recordAudit( + event: Omit, + path: string = defaultAuditPath(), +): boolean { + try { + const dir = dirname(path); + mkdirSync(dir, { recursive: true, mode: AUDIT_DIR_MODE }); + ownerOnly(dir, AUDIT_DIR_MODE); + rotateIfNeeded(path); + const line = JSON.stringify({ timestamp: new Date().toISOString(), ...event }); + appendFileSync(path, `${line}\n`, { encoding: "utf8", mode: AUDIT_FILE_MODE }); + // Re-assert owner-only so a pre-existing world-readable file does not stay open. + ownerOnly(path, AUDIT_FILE_MODE); + return true; + } catch { + // Auditing is best-effort: losing a log line must never fail a tool call. + return false; + } +} diff --git a/packages/cli/src/lib/bash-approval.test.ts b/packages/cli/src/lib/bash-approval.test.ts index 729634d..65c048e 100644 --- a/packages/cli/src/lib/bash-approval.test.ts +++ b/packages/cli/src/lib/bash-approval.test.ts @@ -17,11 +17,11 @@ describe("requiresApproval", () => { { name: "curl pipe bash", command: "curl https://example.com/install.sh | bash" }, { name: "wget pipe sh", command: "wget -O - https://example.com/script | sh" }, { name: "dd disk write", command: "dd if=/dev/zero of=/dev/sda" }, - { name: "redirect to /dev/", command: "echo foo > /dev/null" }, + { name: "redirect to block device", command: "echo foo > /dev/sda" }, ]; test.each(dangerousCommands)("$name requires approval", ({ command }) => { - expect(requiresApproval(command, emptyAllowlist)).toBe(true); + expect(requiresApproval(command, emptyAllowlist, { confined: true })).toBe(true); }); const safeCommands = [ @@ -31,6 +31,35 @@ describe("requiresApproval", () => { ]; test.each(safeCommands)("$name does not require approval", ({ command }) => { - expect(requiresApproval(command, emptyAllowlist)).toBe(false); + expect(requiresApproval(command, emptyAllowlist, { confined: true })).toBe(false); + }); +}); + +describe("unconfined shell", () => { + test("every command requires approval when no OS sandbox applies", () => { + // Without containment a blocklist is theatre: `head /etc/passwd` would pass. + expect(requiresApproval("npm test", emptyAllowlist, { confined: false })).toBe(true); + expect(requiresApproval("echo hi", emptyAllowlist, { confined: false })).toBe(true); + }); + + test("the session allowlist still short-circuits", () => { + const allowed = new Set(["echo hi"]); + expect(requiresApproval("echo hi", allowed, { confined: false })).toBe(false); + }); +}); + +describe("redirects to /dev/null are not treated as device writes", () => { + test("stderr suppression needs no approval", () => { + expect(requiresApproval("ls -la 2>/dev/null", emptyAllowlist, { confined: true })).toBe(false); + expect( + requiresApproval("ls -la secret.txt 2>/dev/null; echo x; ls -la | head -30", emptyAllowlist, { + confined: true, + }), + ).toBe(false); + }); + + test("writing into a real block device still requires approval", () => { + expect(requiresApproval("echo x > /dev/sda", emptyAllowlist, { confined: true })).toBe(true); + expect(requiresApproval("cat img > /dev/disk2", emptyAllowlist, { confined: true })).toBe(true); }); }); diff --git a/packages/cli/src/lib/bash-approval.ts b/packages/cli/src/lib/bash-approval.ts index fe5e0c1..4c897bb 100644 --- a/packages/cli/src/lib/bash-approval.ts +++ b/packages/cli/src/lib/bash-approval.ts @@ -1,12 +1,17 @@ /** - * Blocklist-based bash approval for Build mode (HARNESS-03). + * Bash approval policy (HARNESS-03). * - * Only commands matching D-13 patterns require user confirmation. - * Session allowlist is owned by use-chat (passed in, not a global singleton). + * Two regimes, because the honest answer depends on the host: + * + * 1. **Confined shell** — an OS sandbox (seatbelt / bwrap) applies, so the + * blocklist is a backup: only listed destructive commands ask. + * 2. **Unconfined shell** — no sandbox is available. An unsandboxed shell can read + * anything the user can, so a blocklist would be theatre and *every* command asks. * - * Limits: whitespace normalization only; obfuscation via env vars or quoting - * may bypass patterns — acceptable per phase discretion. + * Session allowlist is owned by use-chat (passed in, not a global singleton). */ +import { isShellConfined } from "./sandbox-capability"; + export type BashApprovalVerdict = "approve-once" | "reject" | "allow-session"; /** @@ -23,7 +28,9 @@ const BLOCKLIST: RegExp[] = [ /\bchmod\s+[^\n]*(-R|--recursive)\b/, // recursive permission changes /\b(curl|wget)\s+[^\n]*\|\s*(ba)?sh\b/, // pipe remote script directly into shell /\bdd\s+[^\n]*if=/, // raw disk write (dd if=/dev/...) - />\s*\/dev\//, // redirect stdout/stderr into a device node + // Only block devices. `2>/dev/null` is everywhere and harmless; matching it made + // the dialog fire on ordinary commands, which trains users to approve blindly. + />\s*\/dev\/(?:sd|disk|nvme|hd|rdisk)/, // redirect into a block device ]; /** Trim and collapse internal whitespace for blocklist and allowlist keys. */ @@ -36,9 +43,24 @@ export function rememberSessionAllow(sessionAllowed: Set, command: strin sessionAllowed.add(normalizeCommand(command)); } -/** True when command matches the blocklist and is not session-allowed. */ -export function requiresApproval(command: string, sessionAllowed: Set): boolean { +/** + * True when command matches the blocklist and is not session-allowed. + * + * `confined` describes whether the spawned shell is OS-sandboxed. It defaults to + * the real capability, because the distinction changes the answer: an unconfined + * shell can read anything, so a blocklist is not a safety net and *every* command + * needs a human decision. + */ +export function requiresApproval( + command: string, + sessionAllowed: Set, + options: { confined?: boolean } = {}, +): boolean { const normalized = normalizeCommand(command); if (sessionAllowed.has(normalized)) return false; + + const confined = options.confined ?? isShellConfined(); + if (!confined) return true; + return BLOCKLIST.some((re) => re.test(normalized)); } diff --git a/packages/cli/src/lib/chat-fetch.test.ts b/packages/cli/src/lib/chat-fetch.test.ts new file mode 100644 index 0000000..c919098 --- /dev/null +++ b/packages/cli/src/lib/chat-fetch.test.ts @@ -0,0 +1,384 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { createFailoverFetch } from "./chat-fetch"; +import { HttpError } from "./http-errors"; + +const readError = async (response: Response) => { + try { + const data = (await response.json()) as { error?: string }; + return data.error ?? `HTTP ${response.status}`; + } catch { + return `HTTP ${response.status}`; + } +}; + +/** Records every call so tests can assert on the rewritten body. */ +function makeFetch(responses: Array Response)>) { + const calls: Array<{ url: string; body: unknown }> = []; + + const fetchImpl = (async (input: RequestInfo | URL, init?: RequestInit) => { + let body: unknown = init?.body; + if (typeof init?.body === "string") { + try { + body = JSON.parse(init.body); + } catch { + body = init.body; + } + } + calls.push({ url: String(input), body }); + + const next = responses.shift(); + if (!next) throw new Error("unexpected extra fetch call"); + return typeof next === "function" ? next() : next; + }) as typeof globalThis.fetch; + + return { fetchImpl, calls }; +} + +function chatInit(model: string) { + return { + method: "POST", + body: JSON.stringify({ id: "session-1", model, mode: "build", messages: [] }), + } satisfies RequestInit; +} + +/** Build a 200 SSE response shaped like the server's UI message stream. */ +function sseResponse(events: Array>): Response { + const body = events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join(""); + + return new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +describe("createFailoverFetch", () => { + test("returns the primary response without reporting a downgrade", async () => { + const { fetchImpl, calls } = makeFetch([new Response("ok", { status: 200 })]); + const downgrades: unknown[] = []; + + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + attemptsPerModel: 1, + fetchImpl, + readError, + onDowngrade: (info) => downgrades.push(info), + }); + + const response = await fetchWithFallback("http://localhost:3000/chat", chatInit("deepseek-flash")); + + expect(response.status).toBe(200); + expect(calls).toHaveLength(1); + expect(downgrades).toEqual([]); + }); + + test("retries on 403 and reports the downgrade explicitly", async () => { + const { fetchImpl, calls } = makeFetch([ + new Response(JSON.stringify({ error: "Forbidden" }), { status: 403 }), + new Response("ok", { status: 200 }), + ]); + const downgrades: Array<{ from: string; to: string }> = []; + + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + attemptsPerModel: 1, + fetchImpl, + readError, + onDowngrade: (info) => downgrades.push(info), + }); + + const response = await fetchWithFallback("http://localhost:3000/chat", chatInit("gpt-oss-120b")); + + expect(response.status).toBe(200); + expect(calls).toHaveLength(2); + // Second attempt must ask the server for the fallback model, nothing else changes. + expect(calls[1]?.body).toMatchObject({ model: "deepseek-flash", id: "session-1", mode: "build" }); + expect(downgrades).toEqual([ + { + from: "gpt-oss-120b", + to: "deepseek-flash", + failures: [{ modelId: "gpt-oss-120b", detail: "HTTP 403" }], + }, + ]); + }); + + test("does not retry a 400 — the request itself is wrong", async () => { + const { fetchImpl, calls } = makeFetch([ + new Response(JSON.stringify({ error: "Invalid request body" }), { status: 400 }), + ]); + + const fetchWithFallback = createFailoverFetch({ allowFailover: true, attemptsPerModel: 1, fetchImpl, readError }); + + await expect( + fetchWithFallback("http://localhost:3000/chat", chatInit("deepseek-flash")), + ).rejects.toBeInstanceOf(HttpError); + + expect(calls).toHaveLength(1); + }); + + test("throws the last failure when every candidate is blocked", async () => { + const { fetchImpl, calls } = makeFetch([ + new Response("", { status: 403 }), + new Response("", { status: 403 }), + new Response("", { status: 403 }), + ]); + + const fetchWithFallback = createFailoverFetch({ allowFailover: true, attemptsPerModel: 1, fetchImpl, readError }); + + await expect( + fetchWithFallback("http://localhost:3000/chat", chatInit("gpt-oss-120b")), + ).rejects.toBeInstanceOf(HttpError); + + // Default chain is flash only: Pro costs ~4x and is never auto-escalated. + expect(calls.map((call) => (call.body as { model: string }).model)).toEqual([ + "gpt-oss-120b", + "deepseek-flash", + ]); + }); + + test("passes through requests it cannot fail over", async () => { + const { fetchImpl, calls } = makeFetch([ + new Response("ok", { status: 200 }), + new Response("ok", { status: 200 }), + ]); + const fetchWithFallback = createFailoverFetch({ allowFailover: true, attemptsPerModel: 1, fetchImpl, readError }); + + await fetchWithFallback("http://localhost:3000/chat", { method: "POST", body: "not-json" }); + await fetchWithFallback("http://localhost:3000/chat", { + method: "POST", + body: JSON.stringify({ id: "x" }), + }); + + expect(calls).toHaveLength(2); + }); + + test("fails over when the server reports the provider error inside a 200 stream", async () => { + // This is the real shape: the server answers 200 and streams the upstream + // failure as an `error` event, so `response.ok` is true and a status-only + // check would never retry. + const { fetchImpl, calls } = makeFetch([ + sseResponse([{ type: "start" }, { type: "error", errorText: "Forbidden" }]), + sseResponse([ + { type: "start" }, + { type: "text-delta", delta: "recovered" }, + { type: "finish" }, + ]), + ]); + const downgrades: Array<{ from: string; to: string }> = []; + + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + attemptsPerModel: 1, + fetchImpl, + readError, + onDowngrade: (info) => downgrades.push(info), + }); + + const response = await fetchWithFallback( + "http://localhost:3000/chat", + chatInit("gpt-oss-120b"), + ); + + expect(calls).toHaveLength(2); + expect(calls[1]?.body).toMatchObject({ model: "deepseek-flash" }); + expect(downgrades[0]).toMatchObject({ from: "gpt-oss-120b", to: "deepseek-flash" }); + expect(await response.text()).toContain("recovered"); + }); + + test("a healthy stream is returned byte-for-byte, nothing lost by peeking", async () => { + const events = [ + { type: "start" }, + { type: "text-delta", delta: "hello" }, + { type: "text-delta", delta: " world" }, + { type: "finish" }, + ]; + const expected = events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join(""); + + const { fetchImpl, calls } = makeFetch([sseResponse(events)]); + const fetchWithFallback = createFailoverFetch({ allowFailover: true, attemptsPerModel: 1, fetchImpl, readError }); + + const response = await fetchWithFallback( + "http://localhost:3000/chat", + chatInit("deepseek-flash"), + ); + + expect(calls).toHaveLength(1); + expect(await response.text()).toBe(expected); + }); + + test("user abort is classified as AbortError, not a retryable HttpError", async () => { + const user = new AbortController(); + user.abort(); + + const fetchImpl = (async (_input: RequestInfo | URL, init?: RequestInit) => { + // Simulate a racing 403 that arrives as the user hits Esc. + if (init?.signal?.aborted) { + throw new HttpError(403, "Forbidden"); + } + return new Response("ok", { status: 200 }); + }) as typeof globalThis.fetch; + + const fetchWithFallback = createFailoverFetch({ allowFailover: true, attemptsPerModel: 1, fetchImpl, readError }); + + await expect( + fetchWithFallback("http://localhost:3000/chat", { + ...chatInit("deepseek-flash"), + signal: user.signal, + }), + ).rejects.toMatchObject({ name: "AbortError" }); + }); + + test("an SSE body that closes before any output fails over", async () => { + const { fetchImpl, calls } = makeFetch([ + sseResponse([{ type: "start" }]), + sseResponse([{ type: "start" }, { type: "text-delta", delta: "recovered" }, { type: "finish" }]), + ]); + + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + attemptsPerModel: 1, + fetchImpl, + readError, + }); + + const response = await fetchWithFallback( + "http://localhost:3000/chat", + chatInit("gpt-oss-120b"), + ); + + expect(calls).toHaveLength(2); + expect(await response.text()).toContain("recovered"); + }); + + test("without consent the turn is never sent to another provider", async () => { + const { fetchImpl, calls } = makeFetch([new Response("", { status: 403 })]); + + const fetchWithFallback = createFailoverFetch({ fetchImpl, readError }); + + await expect( + fetchWithFallback("http://localhost:3000/chat", chatInit("gpt-oss-120b")), + ).rejects.toBeInstanceOf(HttpError); + + // Exactly one attempt: the user did not authorise another vendor. + expect(calls).toHaveLength(1); + }); + + test("a healthy long stream is not killed by the per-attempt deadline", async () => { + // Regression: the deadline used to cover the whole stream, so any turn longer + // than the timeout died mid-reply with "The operation timed out". + const encoder = new TextEncoder(); + const event = (payload: Record) => + encoder.encode(`data: ${JSON.stringify(payload)}\n\n`); + + const fetchImpl = (async () => + new Response( + new ReadableStream({ + async start(controller) { + controller.enqueue(event({ type: "start" })); + controller.enqueue(event({ type: "text-delta", delta: "first" })); + await new Promise((resolve) => setTimeout(resolve, 150)); + controller.enqueue(event({ type: "text-delta", delta: "late" })); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + )) as typeof globalThis.fetch; + + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + attemptsPerModel: 1, + fetchImpl, + readError, + attemptTimeoutMs: 50, + }); + + const response = await fetchWithFallback( + "http://localhost:3000/chat", + chatInit("deepseek-flash"), + ); + const text = await response.text(); + + expect(text).toContain("first"); + // This chunk arrives well after the 50ms deadline; it must still be delivered. + expect(text).toContain("late"); + }); +}); + +describe("deadline policy", () => { + const original = process.env.MOCODE_CONNECT_TIMEOUT_MS; + const originalStall = process.env.MOCODE_STALL_TIMEOUT_MS; + + afterAll(() => { + if (original === undefined) delete process.env.MOCODE_CONNECT_TIMEOUT_MS; + else process.env.MOCODE_CONNECT_TIMEOUT_MS = original; + if (originalStall === undefined) delete process.env.MOCODE_STALL_TIMEOUT_MS; + else process.env.MOCODE_STALL_TIMEOUT_MS = originalStall; + }); + + test("connect budget defaults to 20s", async () => { + const { connectTimeoutMs } = await import("./chat-fetch"); + delete process.env.MOCODE_CONNECT_TIMEOUT_MS; + expect(connectTimeoutMs()).toBe(20_000); + }); + + test("stall budget defaults to 120s — it bounds silence, not generation", async () => { + const { stallTimeoutMs } = await import("./chat-fetch"); + delete process.env.MOCODE_STALL_TIMEOUT_MS; + expect(stallTimeoutMs()).toBe(120_000); + }); + + test("both are overridable", async () => { + const { connectTimeoutMs, stallTimeoutMs } = await import("./chat-fetch"); + process.env.MOCODE_CONNECT_TIMEOUT_MS = "3000"; + process.env.MOCODE_STALL_TIMEOUT_MS = "5000"; + expect(connectTimeoutMs()).toBe(3_000); + expect(stallTimeoutMs()).toBe(5_000); + }); + + test("nonsense values fall back instead of producing a zero budget", async () => { + const { connectTimeoutMs, stallTimeoutMs } = await import("./chat-fetch"); + process.env.MOCODE_CONNECT_TIMEOUT_MS = "-1"; + process.env.MOCODE_STALL_TIMEOUT_MS = "abc"; + expect(connectTimeoutMs()).toBe(20_000); + expect(stallTimeoutMs()).toBe(120_000); + }); +}); + +describe("a slow but alive stream is never killed", () => { + test("chunks trickling in with gaps under the stall budget complete normally", async () => { + const encoder = new TextEncoder(); + const event = (payload: Record) => + encoder.encode(`data: ${JSON.stringify(payload)}\n\n`); + + // Total duration ~450ms, well past the 120ms stall budget — but no single gap is. + const fetchImpl = (async () => + new Response( + new ReadableStream({ + async start(controller) { + controller.enqueue(event({ type: "start" })); + for (const piece of ["a", "b", "c", "d", "e"]) { + await new Promise((resolve) => setTimeout(resolve, 80)); + controller.enqueue(event({ type: "text-delta", delta: piece })); + } + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + )) as typeof globalThis.fetch; + + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + attemptsPerModel: 1, + fetchImpl, + readError, + stallTimeoutMs: 120, + }); + + const response = await fetchWithFallback( + "http://localhost:3000/chat", + chatInit("deepseek-flash"), + ); + const text = await response.text(); + + expect(text).toContain("text-delta"); + }); +}); diff --git a/packages/cli/src/lib/chat-fetch.ts b/packages/cli/src/lib/chat-fetch.ts new file mode 100644 index 0000000..93cce6d --- /dev/null +++ b/packages/cli/src/lib/chat-fetch.ts @@ -0,0 +1,348 @@ +/** + * Failover fetch for the SaaS chat transport. + * + * The server hard-binds one provider per model, so a region-blocked provider makes + * `POST /chat` answer 403 and the whole turn dies. This wrapper retries the request + * against the next model in the chain, rewriting only the `model` field of the body. + * + * Retry happens **before** any bytes stream: only a non-2xx response is retried, so a + * stream that already started (and was already billed) is never replayed. + * + * The downgrade is reported to the caller, never applied silently. + */ +import type { SupportedChatModelId } from "@mocode/shared"; +import { HttpError } from "./http-errors"; +import { runWithFailover, type FailoverDecision } from "./provider-chain"; + +export type DowngradeInfo = { + /** Model the user asked for. */ + from: string; + /** Model that actually answered. */ + to: string; + failures: Array<{ modelId: string; detail: string }>; +}; + +export type FailoverFetchOptions = { + /** Injectable for tests. */ + fetchImpl?: typeof globalThis.fetch; + /** Reads the `{ error }` envelope off a failed response. */ + readError: (response: Response) => Promise; + fallbacks?: readonly SupportedChatModelId[]; + /** Budget for connect + headers. Does NOT bound generation. */ + connectTimeoutMs?: number; + /** @deprecated alias for {@link connectTimeoutMs}. */ + attemptTimeoutMs?: number; + /** Silence budget while reading; reset on every chunk. */ + stallTimeoutMs?: number; + /** Attempts per model; defaults to `MOCODE_PROVIDER_RETRIES` or 3. */ + attemptsPerModel?: number; + /** Injectable for tests so they do not wait on real backoff. */ + sleep?: (ms: number) => Promise; + /** Explicit opt-in to trying another provider; see `FailoverOptions.consent`. */ + allowFailover?: boolean; + onDowngrade?: (info: DowngradeInfo) => void; + onAttemptFailed?: (event: { modelId: string; decision: FailoverDecision }) => void; +}; + +/** + * Budget for *reaching the stream* — DNS, TCP, TLS, HTTP headers. + * + * This bounds the network, not the model. It is cleared the moment headers arrive, + * because everything after that is generation and generation is allowed to be slow. + */ +export function connectTimeoutMs(): number { + const configured = Number(process.env.MOCODE_CONNECT_TIMEOUT_MS ?? process.env.MOCODE_ATTEMPT_TIMEOUT_MS); + return Number.isFinite(configured) && configured > 0 ? configured : 20_000; +} + +/** + * Silence budget while reading the stream. + * + * Reset on **every chunk**. A provider that keeps emitting bytes is healthy no + * matter how long the whole turn takes; only a provider that goes quiet is a + * stall. This is the primitive Claude Code / Codex-style clients use, and it is + * why a long code generation never trips a timeout. + */ +export function stallTimeoutMs(): number { + const configured = Number(process.env.MOCODE_STALL_TIMEOUT_MS); + return Number.isFinite(configured) && configured > 0 ? configured : 120_000; +} + +/** + * How many chunks to inspect before declaring the stream healthy. + * + * The server answers `200` and puts the upstream failure *inside* the SSE body, so + * `response.ok` says nothing about whether the provider worked. Peeking is the only + * way to fail over without changing the server's streaming contract. + */ +const PEEK_MAX_CHUNKS = 16; + +/** + * Events that prove the provider produced something. + * + * Deliberately excludes `text-start` / `tool-input-start`: those arrive *before* + * any content, so treating them as success would suppress failover for a provider + * that opens a block and then fails. + */ +const HEALTHY_EVENT_TYPES = new Set([ + "text-delta", + "reasoning-delta", + "tool-input-available", + "tool-call", + "finish", + "finish-step", +]); + +type StreamInspection = { error: string | null; response: Response }; + +/** Extract `data: {...}` payloads from a partial SSE buffer. */ +function completeDataLines(buffer: string): string[] { + return buffer + .split("\n") + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice("data:".length).trim()) + .filter((line) => line.length > 0 && line !== "[DONE]"); +} + +/** + * Read just enough of the SSE body to tell "the provider worked" from + * "the provider rejected us", then hand back an equivalent response. + * + * Buffered bytes are replayed into the returned stream, so nothing is lost. + */ +type InspectionOptions = { + /** Silence budget; reset on every chunk. */ + stallMs: number; + /** Cancels the underlying response when the stall budget is exceeded. */ + abort: () => void; +}; + +async function inspectStream( + response: Response, + inspection: InspectionOptions, +): Promise { + if (!response.body) return { error: null, response }; + + const reader = response.body.getReader(); + let stallTimer = setTimeout(inspection.abort, inspection.stallMs); + const resetStall = () => { + clearTimeout(stallTimer); + stallTimer = setTimeout(inspection.abort, inspection.stallMs); + }; + const buffered: Uint8Array[] = []; + const decoder = new TextDecoder(); + let text = ""; + let finished = false; + let sawOutput = false; + + try { + for (let read = 0; read < PEEK_MAX_CHUNKS; read += 1) { + const { done, value } = await reader.read(); + // Any bytes at all mean the provider is alive, however slow the turn is. + resetStall(); + if (done) { + finished = true; + break; + } + + buffered.push(value); + text += decoder.decode(value, { stream: true }); + + for (const payload of completeDataLines(text)) { + let parsed: { type?: unknown; errorText?: unknown; error?: unknown }; + try { + parsed = JSON.parse(payload); + } catch { + continue; + } + + if (parsed.type === "error") { + await reader.cancel().catch(() => {}); + const detail = + typeof parsed.errorText === "string" + ? parsed.errorText + : typeof parsed.error === "string" + ? parsed.error + : "upstream provider error"; + return { error: detail, response }; + } + + if (typeof parsed.type === "string" && HEALTHY_EVENT_TYPES.has(parsed.type)) { + sawOutput = true; + read = PEEK_MAX_CHUNKS; // healthy — stop peeking + break; + } + } + + // Only the newest line can be incomplete; drop it before re-checking next round. + text = text.slice(text.lastIndexOf("\n") + 1); + } + + } finally { + clearTimeout(stallTimer); + } + + // EOF before any real output is not success — but only for SSE bodies. A plain + // JSON/text response is not a stream and must pass through untouched. + const isSse = response.headers.get("content-type")?.includes("text/event-stream") ?? false; + if (finished && isSse && !sawOutput) { + return { error: "stream ended before any output", response }; + } + + const headers = new Headers(response.headers); + headers.delete("content-length"); + headers.delete("content-encoding"); + + const stream = new ReadableStream({ + start(controller) { + for (const chunk of buffered) controller.enqueue(chunk); + + if (finished) { + controller.close(); + return; + } + + void (async () => { + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + controller.close(); + return; + } + controller.enqueue(value); + } + } catch (cause) { + controller.error(cause); + } + })(); + }, + cancel(reason) { + return reader.cancel(reason); + }, + }); + + return { + error: null, + response: new Response(stream, { + status: response.status, + statusText: response.statusText, + headers, + }), + }; +} + +/** Combine the caller's abort signal with our own cancellable deadline. */ +function attemptSignal( + userSignal: AbortSignal | null | undefined, + deadline: AbortSignal, +): AbortSignal { + return userSignal ? AbortSignal.any([userSignal, deadline]) : deadline; +} + +/** Error shape `classifyFailover` treats as a retryable timeout. */ +function timeoutError(): Error { + const error = new Error("attempt timed out before the stream started"); + error.name = "TimeoutError"; + return error; +} + +/** + * Wrap `fetch` so a retryable failure transparently retries on the next model. + * + * Bodies that are not a JSON object with a `model` field pass straight through — + * this wrapper only knows how to fail over chat requests. + */ +export function createFailoverFetch( + options: FailoverFetchOptions, +): typeof globalThis.fetch { + const fetchImpl = options.fetchImpl ?? globalThis.fetch; + const connectMs = options.connectTimeoutMs ?? options.attemptTimeoutMs ?? connectTimeoutMs(); + const stallMs = options.stallTimeoutMs ?? stallTimeoutMs(); + + return (async ( + input: Parameters[0], + init?: Parameters[1], + ) => { + const body = typeof init?.body === "string" ? init.body : null; + if (!body) return fetchImpl(input, init); + + let payload: { model?: unknown }; + try { + payload = JSON.parse(body) as { model?: unknown }; + } catch { + return fetchImpl(input, init); + } + + const primary = typeof payload.model === "string" ? payload.model : null; + if (!primary) return fetchImpl(input, init); + + const outcome = await runWithFailover({ + primary, + fallbacks: options.fallbacks, + consent: options.allowFailover === true, + attemptsPerModel: options.attemptsPerModel, + sleep: options.sleep, + attempt: async (modelId) => { + const attemptBody = + modelId === primary ? body : JSON.stringify({ ...payload, model: modelId }); + + // Phase 1: connect + headers. Bounded, because a dead endpoint should fail + // fast instead of hanging the turn. + const controller = new AbortController(); + const connectDeadline = setTimeout(() => controller.abort(), connectMs); + + try { + const response = await fetchImpl(input, { + ...init, + body: attemptBody, + signal: attemptSignal(init?.signal, controller.signal), + }); + + // Headers arrived: the network is fine. From here on only *silence* is a + // failure, and `inspectStream` resets its timer on every chunk. + clearTimeout(connectDeadline); + + if (!response.ok) { + throw new HttpError(response.status, await options.readError(response)); + } + + // 200 does not mean the provider worked: the server streams upstream + // failures as an `error` event inside the body. Fail over on those too. + const inspected = await inspectStream(response, { + stallMs, + abort: () => controller.abort(), + }); + if (inspected.error) { + throw new HttpError(502, inspected.error); + } + + return inspected.response; + } catch (error) { + // Our deadline must be retryable; a user abort must never be retried — + // even when the underlying throw looks like a retryable HttpError. + if (init?.signal?.aborted) { + const abort = new Error("The operation was aborted"); + abort.name = "AbortError"; + throw abort; + } + if (controller.signal.aborted) throw timeoutError(); + throw error; + } finally { + clearTimeout(connectDeadline); + } + }, + onAttemptFailed: options.onAttemptFailed, + }); + + if (outcome.downgradedFrom) { + options.onDowngrade?.({ + from: outcome.downgradedFrom, + to: outcome.modelId, + failures: outcome.failures, + }); + } + + return outcome.value; + }) as typeof globalThis.fetch; +} diff --git a/packages/cli/src/lib/http-errors.ts b/packages/cli/src/lib/http-errors.ts index 67dc3d5..ed91500 100644 --- a/packages/cli/src/lib/http-errors.ts +++ b/packages/cli/src/lib/http-errors.ts @@ -1,9 +1,20 @@ type ErrorResponse = { - json: () => Promise; - status: number; - statusText: string; + json: () => Promise; + status: number; + statusText: string; }; +/** HTTP failure that keeps the status code so callers can act on it. */ +export class HttpError extends Error { + constructor( + readonly statusCode: number, + message: string, + ) { + super(message); + this.name = "HttpError"; + } +} + /** Extract a human-readable message from the server's `{ error }` JSON envelope. */ export async function getErrorMessage(response: ErrorResponse) { try { @@ -16,4 +27,4 @@ export async function getErrorMessage(response: ErrorResponse) { } return response.statusText || `Request failed with status ${response.status}`; - }; \ No newline at end of file + } diff --git a/packages/cli/src/lib/local-chat-transport.test.ts b/packages/cli/src/lib/local-chat-transport.test.ts index 2198b20..0f483b2 100644 --- a/packages/cli/src/lib/local-chat-transport.test.ts +++ b/packages/cli/src/lib/local-chat-transport.test.ts @@ -46,6 +46,138 @@ function createMockResolvedModel() { }; } +/** Minimal sendMessages args for a single user turn on `model`. */ +function sendArgs(model: string, chatId = "session-1") { + return { + trigger: "submit-message" as const, + chatId, + messageId: undefined, + messages: [ + { + id: "msg-1", + role: "user" as const, + parts: [{ type: "text" as const, text: "hello" }], + metadata: { mode: Mode.BUILD, model }, + }, + ], + abortSignal: undefined, + }; +} + +async function drain(stream: ReadableStream) { + const reader = stream.getReader(); + while (true) { + const { done } = await reader.read(); + if (done) break; + } +} + +describe("LocalChatTransport failover (BYOK)", () => { + test("retries the next model when the primary provider is blocked", async () => { + const downgrades: Array<{ from: string; to: string }> = []; + const transport = new LocalChatTransport({ + resolveModel: (modelId) => { + if (modelId === "gpt-oss-120b") { + throw Object.assign(new Error("blocked"), { statusCode: 403 }); + } + return createMockResolvedModel(); + }, + getMcpManager: createMockManager, + buildSystemPrompt: () => "test system prompt", + allowFailover: true, + attemptsPerModel: 1, + onDowngrade: (info) => downgrades.push(info), + }); + + const stream = await transport.sendMessages(sendArgs("gpt-oss-120b")); + await drain(stream); + + expect(downgrades).toEqual([{ from: "gpt-oss-120b", to: "deepseek-flash" }]); + }); + + test("does not fall back on a non-retryable failure", async () => { + const transport = new LocalChatTransport({ + resolveModel: () => { + throw Object.assign(new Error("bad request"), { statusCode: 400 }); + }, + getMcpManager: createMockManager, + buildSystemPrompt: () => "test system prompt", + }); + + await expect(transport.sendMessages(sendArgs("gpt-oss-120b"))).rejects.toThrow("bad request"); + }); + + test("reports every failed attempt before succeeding", async () => { + const failures: string[] = []; + const transport = new LocalChatTransport({ + resolveModel: (modelId) => { + if (modelId !== "deepseek-flash") { + throw Object.assign(new Error("blocked"), { statusCode: 403 }); + } + return createMockResolvedModel(); + }, + getMcpManager: createMockManager, + buildSystemPrompt: () => "test system prompt", + allowFailover: true, + attemptsPerModel: 1, + onAttemptFailed: ({ modelId, decision }) => failures.push(`${modelId}:${decision.detail}`), + }); + + await drain(await transport.sendMessages(sendArgs("claude-opus-4-6"))); + + expect(failures).toEqual(["claude-opus-4-6:HTTP 403"]); + }); + + test("fails over when the stream emits start then a provider error", async () => { + const downgrades: Array<{ from: string; to: string }> = []; + streamTextMock.mockClear(); + + streamTextMock + .mockImplementationOnce(() => ({ + toUIMessageStream: () => + new ReadableStream({ + start(controller) { + controller.enqueue({ type: "start" }); + controller.enqueue({ type: "error", errorText: "Forbidden" }); + controller.close(); + }, + }), + })) + .mockImplementationOnce(() => ({ + toUIMessageStream: () => + new ReadableStream({ + start(controller) { + controller.enqueue({ type: "start" }); + controller.enqueue({ type: "text-delta", id: "t1", delta: "recovered" }); + controller.close(); + }, + }), + })); + + const transport = new LocalChatTransport({ + resolveModel: () => createMockResolvedModel(), + getMcpManager: createMockManager, + buildSystemPrompt: () => "test system prompt", + allowFailover: true, + attemptsPerModel: 1, + onDowngrade: (info) => downgrades.push(info), + }); + + const chunks: unknown[] = []; + const stream = await transport.sendMessages(sendArgs("gpt-oss-120b")); + const reader = stream.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + } + + expect(streamTextMock.mock.calls.length).toBe(2); + expect(downgrades).toEqual([{ from: "gpt-oss-120b", to: "deepseek-flash" }]); + expect(chunks).toContainEqual({ type: "text-delta", id: "t1", delta: "recovered" }); + }); +}); + describe("LocalChatTransport", () => { beforeEach(() => { lastStreamTextTools = undefined; @@ -127,7 +259,8 @@ describe("reconnectToStream (D-12 BYOK)", () => { controller.close(); return; } - controller.enqueue(new TextEncoder().encode('data: {"type":"text-delta"}\n\n')); + controller.enqueue({ type: "start" }); + controller.enqueue({ type: "text-delta", id: "t1", delta: "hi" }); }, }), })); @@ -197,7 +330,8 @@ describe("reconnectToStream (D-12 BYOK)", () => { toUIMessageStream: () => new ReadableStream({ start(controller) { - controller.enqueue(new TextEncoder().encode('data: {"type":"text-delta"}\n\n')); + controller.enqueue({ type: "start" }); + controller.enqueue({ type: "text-delta", id: "t1", delta: "hi" }); }, }), })); diff --git a/packages/cli/src/lib/local-chat-transport.ts b/packages/cli/src/lib/local-chat-transport.ts index 53ca06f..d2ec8a4 100644 --- a/packages/cli/src/lib/local-chat-transport.ts +++ b/packages/cli/src/lib/local-chat-transport.ts @@ -10,6 +10,12 @@ * 3. Detect MCP intent in last user turn → `mcpRequested` strengthens system prompt * 4. validateUIMessages → convertToModelMessages → streamText → toUIMessageStream * + * Failover: an attempt only counts as started once a *useful* UI chunk arrives + * (not a bare `start`). A provider that rejects the connection — or emits + * `start` then an `error` — therefore fails *before* anything is rendered, and + * the next model in the chain is tried (see `lib/provider-chain.ts`). + * Once a useful chunk has been emitted the turn is visible and is never replayed. + * * Tool execution still happens in `use-chat` `onToolCall` (Phase 11 model). */ import { @@ -22,6 +28,7 @@ import { type UIMessageChunk, } from "ai"; import { Mode, type ModeType, type SupportedChatModelId } from "@mocode/shared"; +import { DEFAULT_COMPACTION_POLICY, compactMessages } from "@mocode/shared"; import { loadMergedMcpConfig } from "../mcp/config"; import type { McpManager } from "../mcp/manager"; import { @@ -32,6 +39,8 @@ import { isMcpToolName } from "../mcp/heuristics"; import type { ResolvedModel } from "./local-model"; import { formatChatStreamError } from "./stream-error"; import { hasVisibleAssistantContent } from "@mocode/shared"; +import { runWithFailover, type FailoverDecision } from "./provider-chain"; +import { stallTimeoutMs } from "./chat-fetch"; type LocalChatMetadata = { mode?: ModeType; @@ -74,6 +83,124 @@ function isMcpUserRequest(text: string): boolean { return /\bmcp\b/i.test(text) || /\bmcp__[\w-]+__[\w-]+/i.test(text); } +/** Per-attempt budget covering connect + first useful chunk only. */ + + +/** Chunks that prove the provider is actually producing a turn. */ +const USEFUL_CHUNK_TYPES = new Set([ + "text-start", + "text-delta", + "reasoning-start", + "reasoning-delta", + "tool-input-start", + "tool-input-delta", + "tool-input-available", + "tool-call", + "finish", + "finish-step", +]); + +function isUsefulChunk(chunk: UIMessageChunk): boolean { + return USEFUL_CHUNK_TYPES.has(chunk.type); +} + +function isErrorChunk(chunk: UIMessageChunk): boolean { + return chunk.type === "error"; +} + +function errorChunkMessage(chunk: UIMessageChunk): string { + if ("errorText" in chunk && typeof chunk.errorText === "string" && chunk.errorText.length > 0) { + return chunk.errorText; + } + return "upstream provider error"; +} + +/** Shape `classifyFailover` treats as a retryable timeout. */ +function attemptTimeoutError(): Error { + const error = new Error("attempt timed out before useful output"); + error.name = "TimeoutError"; + return error; +} + +/** Shape `classifyFailover` treats as a non-retryable user abort. */ +function userAbortError(): Error { + const error = new Error("The operation was aborted"); + error.name = "AbortError"; + return error; +} + +/** + * Read until a useful chunk confirms success, an `error` chunk fails the + * attempt, the stream ends, the per-attempt deadline expires, or the user aborts. + * Leading non-useful chunks (e.g. `start`) are buffered and returned with the reader. + */ +async function readUntilAttemptSettled( + reader: { + read: () => Promise<{ done: boolean; value?: UIMessageChunk }>; + cancel: (reason?: unknown) => Promise; + }, + abortSignal: AbortSignal | undefined, + timeoutMs: number, +): Promise<{ + reader: { + read: () => Promise<{ done: boolean; value?: UIMessageChunk }>; + cancel: (reason?: unknown) => Promise; + }; + prefix: UIMessageChunk[]; + done: boolean; +}> { + const prefix: UIMessageChunk[] = []; + const deadline = new AbortController(); + const timer = setTimeout(() => deadline.abort(), timeoutMs); + + const abortAttempt = () => { + clearTimeout(timer); + // Cancelling is what actually unblocks a pending `read()`. Without this the + // deadline only sets a flag nobody observes and a silent provider hangs forever. + void reader.cancel().catch(() => {}); + }; + + const onAbort = () => abortAttempt(); + + try { + abortSignal?.addEventListener("abort", onAbort, { once: true }); + deadline.signal.addEventListener("abort", onAbort, { once: true }); + + while (true) { + const { done, value } = await reader.read(); + + // Check cancellation *before* interpreting the chunk: a read unblocked by our + // own cancel, or a late error chunk after an abort, must not become a + // retryable 502 that sends the turn to another provider. + if (abortSignal?.aborted) throw userAbortError(); + if (deadline.signal.aborted) throw attemptTimeoutError(); + + if (done || value === undefined) { + clearTimeout(timer); + return { reader, prefix, done: true }; + } + + if (isErrorChunk(value)) { + abortAttempt(); + throw Object.assign(new Error(errorChunkMessage(value)), { statusCode: 502 }); + } + + prefix.push(value); + + if (isUsefulChunk(value)) { + clearTimeout(timer); + return { reader, prefix, done: false }; + } + } + } catch (error) { + clearTimeout(timer); + throw error; + } finally { + abortSignal?.removeEventListener("abort", onAbort); + deadline.signal.removeEventListener("abort", onAbort); + } +} + /** Drops assistant rows with no user-visible content (empty or step-start-only placeholders). */ export function stripIncompleteAssistantMessages( messages: UI_MESSAGE[], @@ -98,6 +225,16 @@ export type LocalChatTransportOptions = { skills?: Array<{ name: string; description: string }>; }) => string; onFinish?: (event: LocalChatTransportFinishEvent) => Promise; + /** Explicit opt-in to trying another provider; see `FailoverOptions.consent`. */ + allowFailover?: boolean; + /** Attempts per model; defaults to `MOCODE_PROVIDER_RETRIES` or 3. */ + attemptsPerModel?: number; + /** Injectable for tests so they do not wait on real backoff. */ + sleep?: (ms: number) => Promise; + /** Called when a fallback model answered — the caller must show this to the user. */ + onDowngrade?: (info: { from: string; to: string }) => void; + /** Called for every failed attempt, for logging. */ + onAttemptFailed?: (event: { modelId: string; decision: FailoverDecision }) => void; cwd?: string; }; @@ -143,94 +280,144 @@ export class LocalChatTransport const mcpToolNames = Object.keys(tools).filter(isMcpToolName); const mcpRequested = isMcpUserRequest(lastUserText(messages)); const systemPrompt = this.opts.buildSystemPrompt({ mode, mcpToolNames, mcpRequested }); - const resolvedModel = this.opts.resolveModel(modelId); - const startTime = Date.now(); - let completedUsage: LanguageModelUsage | null = null; const messagesForValidation = stripIncompleteAssistantMessages(messages); const nextMessages = await validateUIMessages({ messages: messagesForValidation, tools: tools as Parameters>[0]["tools"], }); - const modelMessages = await convertToModelMessages(nextMessages, { tools }); - - const result = streamText({ - model: resolvedModel.model, - system: systemPrompt, - messages: modelMessages, - tools, - providerOptions: resolvedModel.providerOptions, - abortSignal, - onFinish(event) { - completedUsage = event.totalUsage; - }, - }); + // Same policy as the server path: fold old turns out of the *model input* only, + // so the client keeps its full transcript. + const compaction = compactMessages(nextMessages, DEFAULT_COMPACTION_POLICY); + const modelMessages = await convertToModelMessages(compaction.messages, { tools }); + const startTime = Date.now(); const onFinish = this.opts.onFinish; - const self = this; - - const rawStream = result.toUIMessageStream({ - originalMessages: nextMessages, - onError: formatChatStreamError, - messageMetadata({ part }) { - if (part.type === "start") { - return { mode, model: modelId } as never; - } + // Bound to *this* send's stream so an older overlapping cleanup cannot clear a newer one. + let ownedStream: ReadableStream | null = null; + const clearActive = () => { + if ( + this.activeStream?.chatId === chatId && + ownedStream !== null && + this.activeStream.stream === ownedStream + ) { + this.activeStream = null; + } + }; - if (part.type !== "finish") return undefined; + /** + * One attempt = create the stream and wait for useful output (or a decisive + * error). Provider failures reject here, before any chunk is enqueued to the + * caller, which is what makes a retry safe. + */ + const outcome = await runWithFailover({ + primary: modelId, + consent: this.opts.allowFailover === true, + attemptsPerModel: this.opts.attemptsPerModel, + sleep: this.opts.sleep, + attempt: async (candidate) => { + let completedUsage: LanguageModelUsage | null = null; + const resolved = this.opts.resolveModel(candidate); - return { - mode, - model: modelId, - durationMs: Date.now() - startTime, - ...(completedUsage ? { usage: completedUsage } : {}), - } as never; - }, - async onFinish(event) { - if (self.activeStream?.stream === stream) self.activeStream = null; - if (!onFinish) return; - await onFinish({ - messages: event.messages, - usage: completedUsage ?? undefined, + const result = streamText({ + model: resolved.model, + system: systemPrompt, + messages: modelMessages, + tools, + providerOptions: resolved.providerOptions, + abortSignal, + onFinish(event) { + completedUsage = event.totalUsage; + }, }); + + const rawStream = result.toUIMessageStream({ + originalMessages: nextMessages, + onError: formatChatStreamError, + messageMetadata({ part }) { + if (part.type === "start") { + return { mode, model: candidate } as never; + } + + if (part.type !== "finish") return undefined; + + return { + mode, + model: candidate, + durationMs: Date.now() - startTime, + ...(completedUsage ? { usage: completedUsage } : {}), + } as never; + }, + async onFinish(event) { + clearActive(); + if (!onFinish) return; + await onFinish({ + messages: event.messages, + usage: completedUsage ?? undefined, + }); + }, + }); + + const reader = rawStream.getReader(); + return readUntilAttemptSettled( + reader as { + read: () => Promise<{ done: boolean; value?: UIMessageChunk }>; + cancel: (reason?: unknown) => Promise; + }, + abortSignal, + stallTimeoutMs(), + ); }, + onAttemptFailed: this.opts.onAttemptFailed, }); + if (outcome.downgradedFrom) { + this.opts.onDowngrade?.({ from: outcome.downgradedFrom, to: outcome.modelId }); + } + + const { reader, prefix, done } = outcome.value; + const stream = new ReadableStream({ start(controller) { - const reader = rawStream.getReader(); + for (const chunk of prefix) { + controller.enqueue(chunk); + } + + if (done) { + clearActive(); + controller.close(); + return; + } + void (async () => { try { while (true) { - const { done, value } = await reader.read(); - if (done) { - if (self.activeStream?.stream === stream) self.activeStream = null; + const { done: finished, value } = await reader.read(); + if (finished) { + clearActive(); controller.close(); return; } controller.enqueue(value); } } catch (error) { - if (self.activeStream?.stream === stream) self.activeStream = null; + clearActive(); controller.error(error); } })(); }, cancel(reason) { - if (self.activeStream?.stream === stream) self.activeStream = null; - return rawStream.cancel(reason); + clearActive(); + return reader.cancel(reason); }, }); - this.activeStream = { chatId, stream }; - - abortSignal?.addEventListener( - "abort", - () => { - if (this.activeStream?.stream === stream) this.activeStream = null; - }, - { once: true }, - ); + ownedStream = stream; + // A finished attempt (empty or already drained) is not reconnectable. + if (!done) { + this.activeStream = { chatId, stream }; + abortSignal?.addEventListener("abort", clearActive, { once: true }); + } return stream; } diff --git a/packages/cli/src/lib/local-model.ts b/packages/cli/src/lib/local-model.ts index 20b8f80..6d246ee 100644 --- a/packages/cli/src/lib/local-model.ts +++ b/packages/cli/src/lib/local-model.ts @@ -33,6 +33,7 @@ type GoogleModelId = Extract["id"]; type GroqModelId = Extract["id"]; type CerebrasModelId = Extract["id"]; type OpenRouterModelId = Extract["id"]; +type DeepSeekModelId = Extract["id"]; export type ResolvedModel = { model: LanguageModel; @@ -200,6 +201,20 @@ function resolveOpenRouterModel(modelId: OpenRouterModelId, apiKey: string): Res }; } +function resolveDeepSeekModel(modelId: DeepSeekModelId, apiKey: string): ResolvedModel { + // DeepSeek only implements Chat Completions; createOpenAI() defaults to Responses API. + const deepseek = createOpenAI({ + apiKey, + baseURL: "https://api.deepseek.com/v1", + name: "deepseek", + }); + return { + model: deepseek.chat(modelId), + provider: "deepseek", + modelId, + }; +} + function resolveSupportedChatModel( model: SupportedChatModel, keys: ProviderKeys | null, @@ -223,6 +238,8 @@ function resolveSupportedChatModel( model.id, requireProviderApiKey("openrouter", keys), ); + case "deepseek": + return resolveDeepSeekModel(model.id, requireProviderApiKey("deepseek", keys)); default: { const _exhaustive: never = model; throw new Error(`Unsupported provider: ${String(_exhaustive)}`); diff --git a/packages/cli/src/lib/local-tools.audit.test.ts b/packages/cli/src/lib/local-tools.audit.test.ts new file mode 100644 index 0000000..6d84ba2 --- /dev/null +++ b/packages/cli/src/lib/local-tools.audit.test.ts @@ -0,0 +1,128 @@ +/** + * Audit + structured refusal codes. + * + * Enterprise rule: the executor is the authority, so every decision must be + * recordable. These tests assert on the *log*, not on the error message — a + * refusal nobody can audit is indistinguishable from a bug. + */ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Mode } from "@mocode/shared"; +import { executeLocalTool } from "./local-tools"; +import { ToolPermissionError } from "./permission-rules"; + +const dir = mkdtempSync(join(tmpdir(), "mocode-tool-audit-")); +const logPath = join(dir, "audit.jsonl"); + +beforeAll(() => { + process.env.MOCODE_AUDIT_PATH = logPath; +}); + +afterAll(() => { + delete process.env.MOCODE_AUDIT_PATH; + rmSync(dir, { recursive: true, force: true }); +}); + +function entries(): Array> { + return readFileSync(logPath, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); +} + +/** Run a call that is expected to be refused and return the thrown error. */ +async function refused( + tool: string, + input: unknown, + mode: (typeof Mode)[keyof typeof Mode] = Mode.BUILD, +): Promise { + try { + await executeLocalTool(tool, input, mode); + } catch (error) { + expect(error).toBeInstanceOf(ToolPermissionError); + return error as ToolPermissionError; + } + + throw new Error(`expected ${tool} to be refused`); +} + +describe("structured refusal codes", () => { + test("path escape carries PATH_ESCAPE", async () => { + const error = await refused("readFile", { path: "/etc/passwd" }); + expect(error.code).toBe("PATH_ESCAPE"); + }); + + test("hard deny carries HARD_DENY", async () => { + const error = await refused("bash", { command: "mkfs.ext4 /dev/null" }); + expect(error.code).toBe("HARD_DENY"); + }); + + test("plan mode carries MODE_DENIED", async () => { + const error = await refused("writeFile", { path: "a.txt", content: "x" }, Mode.PLAN); + expect(error.code).toBe("MODE_DENIED"); + }); + + test("unknown tool carries UNKNOWN_TOOL", async () => { + const error = await refused("rmMinusRf", { path: "x" }); + expect(error.code).toBe("UNKNOWN_TOOL"); + }); + + test("the message stays readable for the model", async () => { + const error = await refused("readFile", { path: "/etc/passwd" }); + expect(error.message).toContain("outside the project directory"); + }); +}); + +describe("audit trail", () => { + test("a successful call is logged as ok", async () => { + await executeLocalTool("listDirectory", { path: "." }, Mode.BUILD); + const entry = entries().at(-1); + + expect(entry).toMatchObject({ tool: "listDirectory", mode: "BUILD", outcome: "ok" }); + }); + + test("a refusal is logged with its code — not silently dropped", async () => { + await refused("readFile", { path: "../../etc/passwd" }); + const entry = entries().at(-1); + + expect(entry).toMatchObject({ + tool: "readFile", + outcome: "denied", + code: "PATH_ESCAPE", + }); + }); + + test("a non-permission failure is logged as error without a code", async () => { + await expect( + executeLocalTool("readFile", { path: "definitely-missing.txt" }, Mode.BUILD), + ).rejects.toThrow(); + + const entry = entries().at(-1); + expect(entry).toMatchObject({ tool: "readFile", outcome: "error" }); + expect(entry?.code).toBeUndefined(); + }); + + test("file contents never reach the log", async () => { + const secret = "TOP-SECRET-PAYLOAD-" + "y".repeat(500); + + // Refused on purpose: the write must not happen, yet the attempt is logged — + // proving the summary is built from the input without echoing it. + await refused("writeFile", { path: "/etc/audit-probe.txt", content: secret }); + + const raw = readFileSync(logPath, "utf8"); + expect(raw).not.toContain("TOP-SECRET-PAYLOAD"); + expect(raw).toContain("writeFile"); + }); + + test("every decision produces exactly one line", async () => { + const before = entries().length; + + await executeLocalTool("listDirectory", { path: "." }, Mode.BUILD); + await refused("readFile", { path: "/etc/passwd" }); + + expect(entries().length).toBe(before + 2); + }); +}); diff --git a/packages/cli/src/lib/local-tools.readline.test.ts b/packages/cli/src/lib/local-tools.readline.test.ts new file mode 100644 index 0000000..43d0b22 --- /dev/null +++ b/packages/cli/src/lib/local-tools.readline.test.ts @@ -0,0 +1,56 @@ +/** + * `readLinesFromFile` length accounting. + * + * A range whose first selected line is empty used to undercount by one: the code + * treated `selectedLength === 0` as "nothing selected yet", so the *second* line + * skipped its separator. Length is not a proxy for selection. + */ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Mode } from "@mocode/shared"; +import { executeLocalTool } from "./local-tools"; + +const dir = mkdtempSync(join(tmpdir(), "mocode-readline-")); +const originalCwd = process.cwd(); + +// First line empty, then a body long enough to force truncation metadata. +const secondLine = "y".repeat(12_000); +const filePath = "big.txt"; +writeFileSync(join(dir, filePath), `\n${secondLine}\n`); + +beforeAll(() => { + process.chdir(dir); +}); + +afterAll(() => { + process.chdir(originalCwd); + rmSync(dir, { recursive: true, force: true }); +}); + +describe("readLinesFromFile", () => { + test("counts the separator even when the first selected line is empty", async () => { + const result = (await executeLocalTool( + "readFile", + { path: filePath, line_start: 1, line_end: 2 }, + Mode.BUILD, + )) as { content: string; truncated?: boolean; totalLength?: number }; + + expect(result.truncated).toBe(true); + // "" + "\n" + secondLine + expect(result.totalLength).toBe(secondLine.length + 1); + }); + + test("keeps the truncation budget aligned with the counted length", async () => { + const result = (await executeLocalTool( + "readFile", + { path: filePath, line_start: 1, line_end: 2 }, + Mode.BUILD, + )) as { content: string; truncated?: boolean }; + + // 10_000 char budget: the empty first line must not silently buy an extra cell. + expect(result.truncated).toBe(true); + expect(result.content.length).toBeLessThanOrEqual(10_000); + }); +}); diff --git a/packages/cli/src/lib/local-tools.rules.test.ts b/packages/cli/src/lib/local-tools.rules.test.ts new file mode 100644 index 0000000..86c3b3d --- /dev/null +++ b/packages/cli/src/lib/local-tools.rules.test.ts @@ -0,0 +1,72 @@ +/** + * User rules enforced by the executor. + * + * The approval dialog is a UI; the executor is the authority. A deny rule must + * hold even for a caller that never asked for approval (evals, subagents). + */ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Mode } from "@mocode/shared"; +import { executeLocalTool } from "./local-tools"; +import { ToolPermissionError } from "./permission-rules"; + +const dir = mkdtempSync(join(tmpdir(), "mocode-rules-")); +const originalCwd = process.cwd(); + +function writeProjectRules(rules: unknown) { + mkdirSync(join(dir, ".mocode"), { recursive: true }); + writeFileSync(join(dir, ".mocode", "permissions.json"), JSON.stringify({ rules })); +} + +beforeAll(() => process.chdir(dir)); + +afterAll(() => { + process.chdir(originalCwd); + rmSync(dir, { recursive: true, force: true }); +}); + +describe("project rules", () => { + test("a deny rule blocks the call with RULE_DENIED", async () => { + writeProjectRules([ + { tool: "bash", pattern: "^curl ", behavior: "deny" }, + ]); + + try { + await executeLocalTool("bash", { command: "curl https://example.com" }, Mode.BUILD); + throw new Error("expected a refusal"); + } catch (error) { + expect(error).toBeInstanceOf(ToolPermissionError); + expect((error as ToolPermissionError).code).toBe("RULE_DENIED"); + expect((error as Error).message).toContain("Denied by project rule"); + } + }); + + test("a non-matching command still runs", async () => { + const result = (await executeLocalTool( + "bash", + { command: "echo allowed" }, + Mode.BUILD, + )) as { stdout: string }; + + expect(result.stdout.trim()).toBe("allowed"); + }); + + test("rules take effect without a restart", async () => { + writeProjectRules([{ tool: "bash", pattern: "^echo ", behavior: "deny" }]); + + await expect(executeLocalTool("bash", { command: "echo blocked" }, Mode.BUILD)).rejects.toThrow( + /Denied by project rule/, + ); + }); + + test("an allow rule does not bypass a hard deny", async () => { + writeProjectRules([{ tool: "bash", pattern: ".*", behavior: "allow" }]); + + // Hard denies sit below rules and are not negotiable. + await expect( + executeLocalTool("bash", { command: "mkfs.ext4 /dev/null" }, Mode.BUILD), + ).rejects.toThrow(/cannot be approved/); + }); +}); diff --git a/packages/cli/src/lib/local-tools.sandbox.test.ts b/packages/cli/src/lib/local-tools.sandbox.test.ts new file mode 100644 index 0000000..9ddb06b --- /dev/null +++ b/packages/cli/src/lib/local-tools.sandbox.test.ts @@ -0,0 +1,132 @@ +/** + * Sandbox attack suite. + * + * Each case is an escape attempt that a model could plausibly emit. They assert on + * the real `executeLocalTool` entry point — the same function the agent calls — + * not on a helper, because the point is that the fence holds where it is used. + * + * Every command used here is harmless if a guard ever regresses: they either fail + * to resolve, or write to `/dev/null`. + */ +import { describe, expect, test } from "bun:test"; +import { Mode } from "@mocode/shared"; +import { executeLocalTool } from "./local-tools"; +import { isShellConfined } from "./sandbox-capability"; + +describe("path escapes are refused", () => { + test("absolute path outside the project", async () => { + await expect( + executeLocalTool("readFile", { path: "/etc/passwd" }, Mode.BUILD), + ).rejects.toThrow("Path is outside the project directory"); + }); + + test("parent traversal", async () => { + await expect( + executeLocalTool("readFile", { path: "../../../../../../etc/passwd" }, Mode.BUILD), + ).rejects.toThrow("Path is outside the project directory"); + }); + + test("writing outside the project", async () => { + await expect( + executeLocalTool("writeFile", { path: "/tmp/mocode-escape-probe.txt", content: "x" }, Mode.BUILD), + ).rejects.toThrow("Path is outside the project directory"); + }); + + test("listing outside the project", async () => { + await expect( + executeLocalTool("listDirectory", { path: "/etc" }, Mode.BUILD), + ).rejects.toThrow("Path is outside the project directory"); + }); +}); + +describe("hard-denied commands are refused even after approval", () => { + test("raw device write is refused at execution time", async () => { + // Harmless if the guard regresses (writes to /dev/null), but it matches the rule. + await expect( + executeLocalTool("bash", { command: "dd if=/dev/zero of=/dev/null count=1" }, Mode.BUILD), + ).rejects.toThrow("cannot be approved"); + }); + + test("filesystem formatting is refused at execution time", async () => { + await expect( + executeLocalTool("bash", { command: "mkfs.ext4 /dev/null" }, Mode.BUILD), + ).rejects.toThrow("cannot be approved"); + }); +}); + +describe("the shell cannot read secret trees when confined", () => { + // Host-dependent by nature: on a machine without seatbelt/bwrap the executor is + // unconfined, and the control that applies there is per-command approval — not + // this assertion. Declaring the precondition keeps the suite honest instead of + // failing on CI hosts that cannot sandbox. + test.skipIf(!isShellConfined())("reading /etc/passwd is refused by the OS sandbox", async () => { + const result = (await executeLocalTool( + "bash", + { command: "head -1 /etc/passwd 2>&1; echo EXIT:$?" }, + Mode.BUILD, + )) as { stdout: string }; + + // Confined hosts deny the read; unconfined hosts still fail closed via approval, + // but this suite runs the executor directly — assert the sandbox denial when present. + expect(result.stdout).not.toMatch(/^[^:\n]+:[^:\n]*:[0-9]+:/); + expect(result.stdout).toMatch(/Operation not permitted|Permission denied|EXIT:[1-9]/); + }); +}); + +describe("the shell does not inherit secrets", () => { + test("an API key in the CLI process is not visible to the agent's shell", async () => { + const canary = "sk-mocode-canary-should-not-leak"; + process.env.DEEPSEEK_API_KEY = canary; + + try { + const result = (await executeLocalTool( + "bash", + { command: 'echo -n "${DEEPSEEK_API_KEY:-absent}"' }, + Mode.BUILD, + )) as { stdout: string }; + + expect(result.stdout).toBe("absent"); + expect(result.stdout).not.toContain(canary); + } finally { + delete process.env.DEEPSEEK_API_KEY; + } + }); + + test("the shell still works — PATH is present", async () => { + const result = (await executeLocalTool("bash", { command: "echo -n $PATH" }, Mode.BUILD)) as { + stdout: string; + }; + + expect(result.stdout.length).toBeGreaterThan(0); + }); + + test("TERM is forced to dumb so escape codes never reach the TUI", async () => { + const result = (await executeLocalTool("bash", { command: "echo -n $TERM" }, Mode.BUILD)) as { + stdout: string; + }; + + expect(result.stdout).toBe("dumb"); + }); +}); + +describe("plan mode is still read-only at the executor", () => { + test("write tools are refused even if the server contract leaked through", async () => { + await expect( + executeLocalTool("writeFile", { path: "leak.txt", content: "x" }, Mode.PLAN), + ).rejects.toThrow("not available in PLAN mode"); + }); + + test("bash is refused", async () => { + await expect(executeLocalTool("bash", { command: "echo hi" }, Mode.PLAN)).rejects.toThrow( + "not available in PLAN mode", + ); + }); +}); + +describe("shell termination is reported with its real cause", () => { + test("a timeout says timeout — not 'the sandbox failed'", async () => { + await expect( + executeLocalTool("bash", { command: "sleep 5", timeout: 150 }, Mode.BUILD), + ).rejects.toThrow(/timed out/); + }); +}); diff --git a/packages/cli/src/lib/local-tools.ts b/packages/cli/src/lib/local-tools.ts index 534eca0..b4de37f 100644 --- a/packages/cli/src/lib/local-tools.ts +++ b/packages/cli/src/lib/local-tools.ts @@ -18,11 +18,17 @@ import { createReadStream } from "node:fs"; import { createInterface } from "node:readline"; import { mkdir, readFile, readdir, stat, writeFile } from "fs/promises"; -import { dirname, isAbsolute, join, relative, resolve } from "path"; +import { dirname, join, relative, resolve } from "path"; import { toolInputSchemas, Mode, type ModeType } from "@mocode/shared"; import { simpleGit } from "simple-git"; import { runRipgrep } from "./ripgrep"; import { trackToolProcess } from "./tool-process-registry"; +import { resolveInsideCwd as resolveSafePath } from "./sandbox-path"; +import { ToolPermissionError, buildSandboxEnv, findHardDeny } from "./permission-rules"; +import { recordAudit, summarizeInput } from "./audit-log"; +import { checkRules } from "./permissions"; +import { isShellConfined, sandboxBackend } from "./sandbox-capability"; +import { parsePathList, runtimePaths, spawnPlan } from "./sandbox-profile"; /** Max chars returned from readFile before truncation metadata is attached. */ const MAX_FILE_SIZE = 10_000; @@ -45,18 +51,23 @@ const READ_ONLY_TOOLS = [ /** * Resolve `path` relative to `process.cwd()` and reject escapes outside it. + * + * Containment is checked twice — lexically and against real paths — so a symlink + * inside the project cannot point at `/etc`. See `sandbox-path.ts`. + * * @throws when the resolved absolute path leaves the project directory. */ function resolveInsideCwd(path: string) { const cwd = process.cwd(); - const resolved = resolve(cwd, path); - const rel = relative(cwd, resolved); - if (rel.startsWith("..") || isAbsolute(rel)) { - throw new Error("Path is outside the project directory"); + try { + return { cwd, resolved: resolveSafePath(cwd, path) }; + } catch (error) { + throw new ToolPermissionError( + "PATH_ESCAPE", + error instanceof Error ? error.message : "Path is outside the project directory", + ); } - - return { cwd, resolved }; } /** Append a truncation notice when shell output exceeds {@link MAX_OUTPUT}. */ @@ -67,12 +78,19 @@ function truncate(value: string, limit: number) { } -/** Streams only the requested line range — avoids loading entire large files. */ +/** + * Streams only the requested line range. + * + * Keeps at most `limit` characters of output while still counting the full + * selected range length, so truncation metadata stays accurate without holding + * the entire selection in memory. + */ async function readLinesFromFile( resolved: string, lineStart?: number, lineEnd?: number, -): Promise { + limit: number = MAX_FILE_SIZE, +): Promise<{ content: string; truncated?: boolean; totalLength?: number }> { const startLine = lineStart ?? 1; const lines: string[] = []; const rl = createInterface({ @@ -81,14 +99,40 @@ async function readLinesFromFile( }); let lineNumber = 0; + let selectedLength = 0; + let keptLength = 0; + let truncated = false; + // Length is not a proxy for "a line has been selected": the first selected line + // may be empty, and then the *second* line would not count its separator. + let hasSelected = false; + let hasKept = false; + for await (const line of rl) { lineNumber += 1; if (lineNumber < startLine) continue; if (lineEnd !== undefined && lineNumber > lineEnd) break; - lines.push(line); + + selectedLength += (hasSelected ? 1 : 0) + line.length; + hasSelected = true; + + if (truncated) continue; + + const nextKept = keptLength + (hasKept ? 1 : 0) + line.length; + if (nextKept <= limit) { + lines.push(line); + keptLength = nextKept; + hasKept = true; + continue; + } + + const remaining = limit - keptLength - (hasKept ? 1 : 0); + if (remaining > 0) lines.push(line.slice(0, remaining)); + keptLength = limit; + truncated = true; } - return lines.join("\n"); + const content = lines.join("\n"); + return truncated ? { content, truncated: true, totalLength: selectedLength } : { content }; } /** Streams file content up to a character limit — avoids loading entire large files. */ @@ -122,15 +166,6 @@ async function readFileUpToCharLimit( return { content: chunks.join(""), truncated }; } -function applyCharLimit( - content: string, - limit: number, -): { content: string; truncated?: boolean; totalLength?: number } { - return content.length > limit - ? { content: content.slice(0, limit), truncated: true, totalLength: content.length } - : { content }; -} - /** * Dispatch a single tool call to the matching local handler. * @@ -139,12 +174,12 @@ function applyCharLimit( * @param mode - Session mode; PLAN rejects write/bash tools as a second guard * (the server already omits them from `getToolContracts(PLAN)`). */ -export async function executeLocalTool(toolName: string, input: unknown, mode: ModeType) { +async function runTool(toolName: string, input: unknown, mode: ModeType) { if ( mode === Mode.PLAN && !READ_ONLY_TOOLS.includes(toolName as (typeof READ_ONLY_TOOLS)[number]) ) { - throw new Error(`Tool ${toolName} is not available in PLAN mode`); + throw new ToolPermissionError("MODE_DENIED", `Tool ${toolName} is not available in PLAN mode`); } switch (toolName) { @@ -153,15 +188,7 @@ export async function executeLocalTool(toolName: string, input: unknown, mode: M const { resolved } = resolveInsideCwd(path); if (line_start !== undefined || line_end !== undefined) { - const content = await readLinesFromFile(resolved, line_start, line_end); - const limited = applyCharLimit(content, MAX_FILE_SIZE); - return limited.truncated - ? { - content: limited.content, - truncated: true, - totalLength: limited.totalLength, - } - : { content: limited.content }; + return readLinesFromFile(resolved, line_start, line_end, MAX_FILE_SIZE); } const { content, truncated } = await readFileUpToCharLimit(resolved, MAX_FILE_SIZE); @@ -276,22 +303,86 @@ export async function executeLocalTool(toolName: string, input: unknown, mode: M return { success: true as const, path: relative(cwd, resolved) }; } case "bash": { - // Runs in the sandboxed project cwd; TERM=dumb avoids escape-sequence noise in the TUI. + // Hard denies are refused even when the user approved the command: approval + // covers destructive-but-legitimate work, not host-destroying commands. const { command, timeout = DEFAULT_TIMEOUT } = toolInputSchemas.bash.parse(input); - const proc = Bun.spawn(["bash", "-c", command], { - cwd: resolveInsideCwd(".").resolved, - stdout: "pipe", - stderr: "pipe", - env: { ...process.env, TERM: "dumb" }, + const hardDeny = findHardDeny(command); + if (hardDeny) { + throw new ToolPermissionError( + "HARD_DENY", + `Command refused (${hardDeny.id}): ${hardDeny.reason}. This cannot be approved.`, + ); + } + + // Runs in the project cwd with an allowlisted env; TERM=dumb avoids escape + // sequences leaking into the TUI. API keys are deliberately absent. + // + // A shell cannot be contained by path checks, so the spawn is wrapped in the + // host's OS sandbox when one is actually available. When it is not, the run + // is unconfined and the approval policy requires a human for every command. + const cwd = resolveInsideCwd(".").resolved; + const plan = spawnPlan(sandboxBackend(), command, { + cwd, + home: process.env.HOME, + tmpDir: process.env.TMPDIR, + runtimePaths: runtimePaths(), + extraReadPaths: parsePathList(process.env.MOCODE_SANDBOX_EXTRA_READ), + extraWritePaths: parsePathList(process.env.MOCODE_SANDBOX_EXTRA_WRITE), }); + + // Under bwrap the host TMPDIR is a read-only bind; the plan mounts a private + // tmpfs at /tmp, so point the child there. + const childEnv = + plan.backend === "bwrap" ? { ...buildSandboxEnv(), TMPDIR: "/tmp", TMP: "/tmp" } : buildSandboxEnv(); + + const proc = (() => { + try { + return Bun.spawn([plan.file, ...plan.args], { + cwd, + stdout: "pipe", + stderr: "pipe", + env: childEnv, + }); + } catch (error) { + // A sandbox that fails to start must not look like "the command printed + // nothing" — that silently teaches the model to hallucinate output. + const reason = error instanceof Error ? error.message : String(error); + throw new Error( + `Shell could not start (${plan.backend} sandbox): ${reason}. ` + + `Set MOCODE_DISABLE_SANDBOX=1 to run unconfined and diagnose.`, + ); + } + })(); trackToolProcess(proc); - const timer = setTimeout(() => proc.kill(), timeout); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + proc.kill(); + }, timeout); const [stdout, stderr] = await Promise.all([ new Response(proc.stdout).text(), new Response(proc.stderr).text(), ]); const exitCode = await proc.exited; clearTimeout(timer); + + // Distinguish *why* the shell died. A timeout or a user kill also shows up + // as a signal; blaming the sandbox for those was wrong and unactionable. + if (timedOut) { + throw new Error(`Command timed out after ${timeout}ms and was terminated.`); + } + + if (proc.signalCode) { + const sandboxCrash = + plan.confined && (proc.signalCode === "SIGABRT" || proc.signalCode === "SIGSEGV"); + + throw new Error( + sandboxCrash + ? `Shell terminated by ${proc.signalCode} — the ${plan.backend} sandbox failed to apply. ` + + `Set MOCODE_DISABLE_SANDBOX=1 to run unconfined and diagnose.` + : `Command terminated by signal ${proc.signalCode}.`, + ); + } return { stdout: truncate(stdout, MAX_OUTPUT), stderr: truncate(stderr, MAX_OUTPUT), @@ -299,6 +390,49 @@ export async function executeLocalTool(toolName: string, input: unknown, mode: M }; } default: - throw new Error(`Unknown tool: ${toolName}`); + throw new ToolPermissionError("UNKNOWN_TOOL", `Unknown tool: ${toolName}`); + } +}; + +/** + * Authorised tool entry point. + * + * Every call — allowed, refused, or failed — leaves exactly one audit line. That is + * what makes the executor the authority rather than the model: a refusal is a + * recorded decision, not an absence of attempts. + * + * The thrown error keeps a human-readable message for the model while carrying a + * `PermissionCode` for the log. + */ +export async function executeLocalTool( + toolName: string, + input: unknown, + mode: ModeType, +): Promise { + const detail = summarizeInput(input); + + // A deny rule is enforced here, at the authority, not only in the approval UI. + // Checked on every call so a rule edited mid-session applies immediately. + const verdict = checkRules({ tool: toolName, input }); + if (verdict?.behavior === "deny") { + recordAudit({ tool: toolName, mode, outcome: "denied", code: "RULE_DENIED", detail }); + throw new ToolPermissionError("RULE_DENIED", verdict.detail); + } + + // Only shells have a containment question; record the answer alongside the call + // so an audit can distinguish "ran in a sandbox" from "ran unconfined". + const containment = toolName === "bash" ? { shellConfined: isShellConfined() } : {}; + + try { + const output = await runTool(toolName, input, mode); + recordAudit({ tool: toolName, mode, outcome: "ok", detail, ...containment }); + return output; + } catch (error) { + recordAudit( + error instanceof ToolPermissionError + ? { tool: toolName, mode, outcome: "denied", code: error.code, detail, ...containment } + : { tool: toolName, mode, outcome: "error", detail, ...containment }, + ); + throw error; } -}; \ No newline at end of file +} \ No newline at end of file diff --git a/packages/cli/src/lib/permission-rules.test.ts b/packages/cli/src/lib/permission-rules.test.ts new file mode 100644 index 0000000..774a10d --- /dev/null +++ b/packages/cli/src/lib/permission-rules.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, test } from "bun:test"; +import { + buildSandboxEnv, + evaluateToolPermission, + findHardDeny, + isHardDenied, +} from "./permission-rules"; + +describe("hard denies — not approvable in any mode", () => { + test("catches filesystem-destroying commands", () => { + const cases = [ + "rm -rf /", + "rm -rf /;", + "rm -rf /&&", + "rm -rf ~", + "rm -rf $HOME", + "mkfs.ext4 /dev/sda1", + "dd if=/dev/zero of=/dev/sda", + "cat /dev/zero > /dev/sda", + ":(){ :|:& };:", + ]; + + for (const command of cases) { + expect(findHardDeny(command)).not.toBeNull(); + } + }); + + test("leaves legitimate work alone", () => { + const safe = [ + "rm -rf ./dist", + "rm -rf node_modules", + "npm test", + "git status", + "chmod -R 755 ./scripts", + "curl -s https://example.com", + ]; + + for (const command of safe) { + expect(isHardDenied(command)).toBe(false); + } + }); + + test("reports why the command was refused", () => { + expect(findHardDeny("mkfs.ext4 /dev/sda1")).toEqual({ + id: "mkfs", + reason: "formatting a filesystem", + }); + }); +}); + +describe("evaluateToolPermission", () => { + test("read-only tools are allowed everywhere, including plan mode", () => { + for (const mode of ["plan", "default", "acceptEdits", "bypass"] as const) { + expect(evaluateToolPermission("readFile", mode).action).toBe("allow"); + expect(evaluateToolPermission("grep", mode).action).toBe("allow"); + expect(evaluateToolPermission("gitDiff", mode).action).toBe("allow"); + } + }); + + test("plan mode denies every write path", () => { + expect(evaluateToolPermission("writeFile", "plan")).toEqual({ + action: "deny", + reason: "plan mode is read-only", + }); + expect(evaluateToolPermission("editFile", "plan").action).toBe("deny"); + expect(evaluateToolPermission("bash", "plan").action).toBe("deny"); + }); + + test("default mode asks before writing", () => { + expect(evaluateToolPermission("writeFile", "default")).toEqual({ + action: "ask", + reason: "write requires approval", + }); + }); + + test("acceptEdits and bypass allow writing", () => { + expect(evaluateToolPermission("editFile", "acceptEdits").action).toBe("allow"); + expect(evaluateToolPermission("editFile", "bypass").action).toBe("allow"); + }); + + test("unknown tools ask rather than guess", () => { + expect(evaluateToolPermission("mcp__fs__write", "default").action).toBe("ask"); + }); +}); + +describe("buildSandboxEnv", () => { + const source = { + PATH: "/usr/bin", + HOME: "/Users/dev", + LANG: "zh_CN.UTF-8", + TERM: "xterm-256color", + DEEPSEEK_API_KEY: "sk-should-not-leak", + CLERK_SECRET_KEY: "sk_test_should_not_leak", + DATABASE_URL: "postgres://user:pass@host/db", + }; + + test("keeps the keys a shell legitimately needs", () => { + const env = buildSandboxEnv(source); + + expect(env.PATH).toBe("/usr/bin"); + expect(env.HOME).toBe("/Users/dev"); + expect(env.LANG).toBe("zh_CN.UTF-8"); + }); + + test("drops every secret", () => { + const env = buildSandboxEnv(source); + const serialized = JSON.stringify(env); + + expect(env.DEEPSEEK_API_KEY).toBeUndefined(); + expect(env.CLERK_SECRET_KEY).toBeUndefined(); + expect(env.DATABASE_URL).toBeUndefined(); + expect(serialized).not.toContain("sk-should-not-leak"); + expect(serialized).not.toContain("postgres://"); + }); + + test("forces TERM=dumb so escape codes never reach the TUI", () => { + expect(buildSandboxEnv(source).TERM).toBe("dumb"); + }); + + test("skips unset keys instead of writing undefined", () => { + const env = buildSandboxEnv({ PATH: "/bin" }); + expect(env.HOME).toBeUndefined(); + expect(Object.values(env).every((value) => typeof value === "string")).toBe(true); + }); +}); diff --git a/packages/cli/src/lib/permission-rules.ts b/packages/cli/src/lib/permission-rules.ts new file mode 100644 index 0000000..779763f --- /dev/null +++ b/packages/cli/src/lib/permission-rules.ts @@ -0,0 +1,190 @@ +/** + * Permission policy for agent tools. + * + * Two independent concerns live here: + * + * 1. **Hard denies** — commands that are refused even when the user clicks + * "approve". Approval is for *destructive but legitimate* work; it is not a + * licence to format a disk. The earlier blocklist could only *ask*, so a user + * clicking through the dialog could still run `mkfs`. + * + * 2. **Environment isolation** — bash used to inherit the full `process.env`, + * which meant the agent could read `DEEPSEEK_API_KEY`, `CLERK_SECRET_KEY` and + * every other secret the CLI was started with. The sandbox env is an explicit + * allowlist. + * + * Deliberately a policy module: no I/O, no React, no process spawning. + */ + +/** Coarse permission mode, mirroring the Plan/Build split the TUI already exposes. */ +export type PermissionMode = "plan" | "default" | "acceptEdits" | "bypass"; + +export type PermissionAction = "allow" | "deny" | "ask"; + +/** Machine-readable refusal reasons — audited and aggregatable. */ +export type PermissionCode = + | "PATH_ESCAPE" + | "HARD_DENY" + | "MODE_DENIED" + | "UNKNOWN_TOOL" + | "RULE_DENIED"; + +/** + * Refusal raised by the executor. + * + * The message stays human-readable (the model is told why), while `code` is what + * audit logs and dashboards group by. Prose messages drift; codes do not. + */ +export class ToolPermissionError extends Error { + constructor( + readonly code: PermissionCode, + message: string, + ) { + super(message); + this.name = "ToolPermissionError"; + } +} + +export type ToolPermissionDecision = { + action: PermissionAction; + reason: string; +}; + +/** Tools that only read state; safe in every mode, including Plan. */ +const READ_ONLY_TOOLS = new Set([ + "readFile", + "listDirectory", + "glob", + "grep", + "gitStatus", + "gitDiff", + "task", +]); + +/** Tools that mutate the project tree. */ +const WRITE_TOOLS = new Set(["writeFile", "editFile"]); + +/** + * Commands that are never executed, regardless of mode or approval. + * Kept narrow on purpose — a broad list would block legitimate cleanup work. + */ +const HARD_DENY_RULES: ReadonlyArray<{ id: string; pattern: RegExp; reason: string }> = [ + { + id: "rm-root", + // `rm -rf /;` and `rm -rf /&&` are just as final as `rm -rf /`. + pattern: /\brm\s+[^\n]*\s\/(?:\s|;|&|\||$)/, + reason: "deleting the filesystem root", + }, + { + id: "rm-home", + pattern: /\brm\s+[^\n]*\s(?:~|\$HOME)\/?\s?(?:$|;|\|)/, + reason: "deleting the home directory", + }, + { + id: "mkfs", + pattern: /\bmkfs(?:\.\w+)?\b/, + reason: "formatting a filesystem", + }, + { + id: "dd-device", + pattern: /\bdd\s+[^\n]*\bof=\/dev\//, + reason: "writing raw bytes to a device node", + }, + { + id: "dev-redirect", + pattern: />\s*\/dev\/(?:sd|disk|nvme|hd)/, + reason: "redirecting output into a block device", + }, + { + id: "fork-bomb", + pattern: /:\(\)\s*\{[^}]*\};?\s*:/, + reason: "fork bomb", + }, +]; + +/** Returns the matching hard-deny rule, or null when the command is merely risky. */ +export function findHardDeny(command: string): { id: string; reason: string } | null { + for (const rule of HARD_DENY_RULES) { + if (rule.pattern.test(command)) return { id: rule.id, reason: rule.reason }; + } + return null; +} + +export function isHardDenied(command: string): boolean { + return findHardDeny(command) !== null; +} + +/** + * Decide whether a tool may run in the current mode. + * + * Bash is intentionally `allow` here: the finer-grained "is this specific command + * dangerous" question is answered by `bash-approval.ts`, which drives the TUI + * dialog. This function only answers "may this *class* of tool run at all". + */ +export function evaluateToolPermission( + tool: string, + mode: PermissionMode, +): ToolPermissionDecision { + if (READ_ONLY_TOOLS.has(tool)) { + return { action: "allow", reason: "read-only" }; + } + + if (WRITE_TOOLS.has(tool)) { + switch (mode) { + case "plan": + return { action: "deny", reason: "plan mode is read-only" }; + case "acceptEdits": + return { action: "allow", reason: "acceptEdits mode" }; + case "bypass": + return { action: "allow", reason: "bypass mode" }; + case "default": + return { action: "ask", reason: "write requires approval" }; + } + } + + if (tool === "bash") { + return mode === "plan" + ? { action: "deny", reason: "plan mode is read-only" } + : { action: "allow", reason: "command checked against the blocklist separately" }; + } + + // MCP and unknown tools: ask rather than guess. + return { action: "ask", reason: "unrecognised tool" }; +} + +/** + * Environment keys a spawned shell may see. Everything else — API keys especially — + * stays out of the child process. + */ +export const SANDBOX_ENV_KEYS = [ + "PATH", + "HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "SHELL", + "TMPDIR", + "TMP", + "USER", + "LOGNAME", + "PWD", +] as const; + +/** + * Build the child environment from an explicit allowlist. + * + * `TERM` is forced to `dumb` so escape sequences never leak into the TUI. + */ +export function buildSandboxEnv( + source: Record = process.env, +): Record { + const env: Record = {}; + + for (const key of SANDBOX_ENV_KEYS) { + const value = source[key]; + if (typeof value === "string") env[key] = value; + } + + env.TERM = "dumb"; + return env; +} diff --git a/packages/cli/src/lib/permissions.test.ts b/packages/cli/src/lib/permissions.test.ts new file mode 100644 index 0000000..2b33b19 --- /dev/null +++ b/packages/cli/src/lib/permissions.test.ts @@ -0,0 +1,156 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + evaluateRules, + loadPermissionRules, + loadRuleFile, + parseRules, + ruleSubject, + type LoadedRule, +} from "./permissions"; + +const dir = mkdtempSync(join(tmpdir(), "mocode-perms-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +function writeRules(path: string, content: string): string { + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, content); + return path; +} + +describe("parseRules", () => { + test("accepts the documented shape", () => { + const result = parseRules( + { rules: [{ tool: "bash", pattern: "^npm ", behavior: "allow" }] }, + "project", + ); + + expect(result.errors).toEqual([]); + expect(result.rules).toHaveLength(1); + expect(result.rules[0]).toMatchObject({ tool: "bash", behavior: "allow", source: "project" }); + }); + + test("drops a rule with a bad behaviour instead of coercing it", () => { + // Coercing `"denied"` into an allow would be the worst possible failure. + const result = parseRules({ rules: [{ tool: "bash", behavior: "denied" }] }, "user"); + + expect(result.rules).toEqual([]); + expect(result.errors[0]).toContain("behavior"); + }); + + test("drops a rule with an invalid regex and says so", () => { + const result = parseRules({ rules: [{ tool: "bash", pattern: "([", behavior: "deny" }] }, "user"); + + expect(result.rules).toEqual([]); + expect(result.errors[0]).toContain("invalid regex"); + }); + + test("requires a tool name", () => { + const result = parseRules({ rules: [{ behavior: "deny" }] }, "user"); + expect(result.errors[0]).toContain("tool"); + }); + + test("reports a non-array rules field", () => { + expect(parseRules({ rules: "nope" }, "user").errors[0]).toContain("array"); + }); + + test("an empty object is not an error", () => { + expect(parseRules({}, "user").errors).toEqual([]); + }); +}); + +describe("loadRuleFile", () => { + test("a missing file is normal, not an error", () => { + expect(loadRuleFile(join(dir, "nope.json"), "user")).toEqual({ rules: [], errors: [] }); + }); + + test("invalid JSON is reported and yields no rules", () => { + const path = writeRules(join(dir, "bad.json"), "{ not json"); + const result = loadRuleFile(path, "project"); + + expect(result.rules).toEqual([]); + expect(result.errors[0]).toContain("invalid JSON"); + }); +}); + +describe("loadPermissionRules", () => { + test("merges user and project files, tagging the source", () => { + const userPath = writeRules( + join(dir, "user.json"), + JSON.stringify({ rules: [{ tool: "bash", behavior: "ask" }] }), + ); + const projectPath = writeRules( + join(dir, "project.json"), + JSON.stringify({ rules: [{ tool: "writeFile", behavior: "deny" }] }), + ); + + const result = loadPermissionRules(dir, { user: userPath, project: projectPath }); + + expect(result.rules.map((rule) => rule.source)).toEqual(["user", "project"]); + }); +}); + +describe("evaluateRules", () => { + function rules(...entries: Array>): LoadedRule[] { + return entries.map((entry) => ({ + tool: "bash", + behavior: "ask", + source: "project", + pattern: undefined, + regex: null, + ...entry, + })) as LoadedRule[]; + } + + test("a deny anywhere wins over an allow everywhere", () => { + const list = rules( + { tool: "bash", behavior: "allow", source: "user" }, + { tool: "bash", behavior: "deny", source: "project" }, + ); + + expect(evaluateRules(list, "bash", "rm -rf /")).toMatchObject({ behavior: "deny" }); + }); + + test("an allow suppresses the ask", () => { + const list = rules({ tool: "bash", behavior: "allow", regex: /^npm / }); + + expect(evaluateRules(list, "bash", "npm test")).toMatchObject({ behavior: "allow" }); + expect(evaluateRules(list, "bash", "curl x")).toBeNull(); + }); + + test("wildcard tools match everything", () => { + const list = rules({ tool: "*", behavior: "deny" }); + expect(evaluateRules(list, "writeFile", "a.txt")).toMatchObject({ behavior: "deny" }); + }); + + test("first match wins within one behaviour", () => { + const list = rules( + { tool: "bash", behavior: "deny", regex: /^git push/ }, + { tool: "bash", behavior: "deny", regex: /^git/ }, + ); + + expect(evaluateRules(list, "bash", "git push origin main")?.rule.regex?.source).toBe( + "^git push", + ); + }); + + test("no match returns null so the caller keeps its own policy", () => { + expect(evaluateRules(rules({ tool: "writeFile", behavior: "deny" }), "bash", "ls")).toBeNull(); + }); +}); + +describe("ruleSubject", () => { + test("bash matches on the command", () => { + expect(ruleSubject("bash", { command: "npm test" })).toBe("npm test"); + }); + + test("file tools match on the path", () => { + expect(ruleSubject("writeFile", { path: "src/a.ts", content: "x" })).toBe("src/a.ts"); + }); + + test("unknown input yields an empty subject", () => { + expect(ruleSubject("bash", null)).toBe(""); + }); +}); diff --git a/packages/cli/src/lib/permissions.ts b/packages/cli/src/lib/permissions.ts new file mode 100644 index 0000000..17f62a7 --- /dev/null +++ b/packages/cli/src/lib/permissions.ts @@ -0,0 +1,218 @@ +/** + * User-writable permission rules. + * + * The previous model was a hardcoded blocklist plus an in-session allow set: a user + * could approve something for an hour, never for a project, and could not deny + * anything the blocklist did not already know about. + * + * Rules live in JSON so the policy is reviewable, diffable and shareable: + * + * ~/.mocode/permissions.json user rules, apply to every project + * /.mocode/permissions.json project rules, committed with the repo + * + * Precedence is **deny > allow > ask**, matching how a safety rule should behave: + * a deny anywhere wins over an allow everywhere. Within one behaviour the first + * matching rule wins, so the file reads top-down. + * + * Pure: no I/O in the evaluator, so the precedence table is testable on its own. + */ +import { readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +export type PermissionBehavior = "allow" | "deny" | "ask"; + +export type PermissionRule = { + /** Exact tool name, or `*` for every tool. */ + tool: string; + /** Optional JavaScript regex tested against the tool's subject. */ + pattern?: string; + behavior: PermissionBehavior; +}; + +export type RuleSource = "user" | "project"; + +export type LoadedRule = PermissionRule & { + source: RuleSource; + /** Compiled pattern, or null when the rule matches on tool name alone. */ + regex: RegExp | null; +}; + +export type RuleLoadResult = { + rules: LoadedRule[]; + /** Human-readable problems. A bad rule is dropped, never guessed at. */ + errors: string[]; +}; + +const BEHAVIORS: ReadonlySet = new Set(["allow", "deny", "ask"]); + +/** + * Validate raw parsed JSON. + * + * A malformed rule is dropped rather than coerced: silently turning + * `{"behavior": "denied"}` into an allow would be the worst possible failure. + */ +export function parseRules(raw: unknown, source: RuleSource): RuleLoadResult { + const errors: string[] = []; + const rules: LoadedRule[] = []; + + const list = (raw as { rules?: unknown })?.rules; + + // An absent `rules` key is an empty policy; a present but wrong-typed one is a + // mistake worth reporting. + if (list === undefined) return { rules, errors }; + + if (!Array.isArray(list)) { + errors.push("`rules` must be an array"); + return { rules, errors }; + } + + list.forEach((entry, index) => { + const where = `rules[${index}]`; + if (entry === null || typeof entry !== "object") { + errors.push(`${where}: not an object`); + return; + } + + const { tool, pattern, behavior } = entry as Record; + + if (typeof tool !== "string" || tool.length === 0) { + errors.push(`${where}: missing "tool"`); + return; + } + + if (typeof behavior !== "string" || !BEHAVIORS.has(behavior)) { + errors.push(`${where}: "behavior" must be allow, deny or ask`); + return; + } + + if (pattern !== undefined && typeof pattern !== "string") { + errors.push(`${where}: "pattern" must be a string`); + return; + } + + let regex: RegExp | null = null; + if (typeof pattern === "string") { + try { + regex = new RegExp(pattern); + } catch { + errors.push(`${where}: invalid regex ${JSON.stringify(pattern)}`); + return; + } + } + + rules.push({ tool, pattern, behavior: behavior as PermissionBehavior, source, regex }); + }); + + return { rules, errors }; +} + +/** Read one rule file. A missing file is normal; a malformed one is reported. */ +export function loadRuleFile(path: string, source: RuleSource): RuleLoadResult { + let text: string; + + try { + text = readFileSync(path, "utf8"); + } catch { + return { rules: [], errors: [] }; + } + + try { + return parseRules(JSON.parse(text), source); + } catch { + return { rules: [], errors: [`${path}: invalid JSON`] }; + } +} + +export const USER_RULES_PATH = () => join(homedir(), ".mocode", "permissions.json"); +export const PROJECT_RULES_PATH = (cwd: string) => join(cwd, ".mocode", "permissions.json"); + +/** + * Load user rules then project rules. + * + * Order only decides *within* a behaviour; deny always beats allow regardless of + * which file it came from. + */ +export function loadPermissionRules( + cwd: string = process.cwd(), + paths: { user?: string; project?: string } = {}, +): RuleLoadResult { + const user = loadRuleFile(paths.user ?? USER_RULES_PATH(), "user"); + const project = loadRuleFile(paths.project ?? PROJECT_RULES_PATH(cwd), "project"); + + return { + rules: [...user.rules, ...project.rules], + errors: [...user.errors, ...project.errors], + }; +} + +export type RuleMatch = { + behavior: PermissionBehavior; + rule: LoadedRule; +}; + +/** + * Evaluate rules for one tool call. + * + * Returns `null` when nothing matches, which the caller treats as "fall back to the + * built-in policy". Deny is scanned first across **all** rules, then allow, then ask. + */ +export function evaluateRules( + rules: readonly LoadedRule[], + tool: string, + subject: string, +): RuleMatch | null { + const matches = (rule: LoadedRule): boolean => { + if (rule.tool !== "*" && rule.tool !== tool) return false; + return rule.regex === null || rule.regex.test(subject); + }; + + for (const behavior of ["deny", "allow", "ask"] as const) { + const rule = rules.find((candidate) => candidate.behavior === behavior && matches(candidate)); + if (rule) return { behavior, rule }; + } + + return null; +} + +/** The string a rule's pattern is tested against. */ +export function ruleSubject(tool: string, input: unknown): string { + if (input === null || typeof input !== "object") return ""; + + const record = input as Record; + if (tool === "bash" && typeof record.command === "string") return record.command; + if (typeof record.path === "string") return record.path; + + return ""; +} + +export type RuleVerdict = { + behavior: PermissionBehavior; + detail: string; +}; + +/** + * Load the rule files and evaluate one call. + * + * Loaded per call on purpose: a rule edited in the project file takes effect on the + * very next tool call, which is what makes the policy demoable without a restart. + * Two small JSON reads per tool call is noise next to the model request. + */ +export function checkRules( + params: { tool: string; input: unknown }, + cwd: string = process.cwd(), +): RuleVerdict | null { + const { rules } = loadPermissionRules(cwd); + if (rules.length === 0) return null; + + const match = evaluateRules(rules, params.tool, ruleSubject(params.tool, params.input)); + if (!match) return null; + + const origin = `${match.rule.source} rule`; + const pattern = match.rule.pattern ? ` /${match.rule.pattern}/` : ""; + + return { + behavior: match.behavior, + detail: `${match.behavior === "allow" ? "Allowed" : match.behavior === "deny" ? "Denied" : "Approval required"} by ${origin} (${match.rule.tool}${pattern})`, + }; +} diff --git a/packages/cli/src/lib/provider-chain.test.ts b/packages/cli/src/lib/provider-chain.test.ts new file mode 100644 index 0000000..9dec710 --- /dev/null +++ b/packages/cli/src/lib/provider-chain.test.ts @@ -0,0 +1,293 @@ +import { describe, expect, test } from "bun:test"; +import { + DEFAULT_FALLBACK_CHAIN, + retryDelayMs, + buildModelChain, + classifyFailover, + runWithFailover, +} from "./provider-chain"; + +/** Error carrying an HTTP status, shaped like the providers' failures. */ +function httpError(status: number): Error & { statusCode: number } { + const error = new Error(`upstream failed with ${status}`) as Error & { statusCode: number }; + error.statusCode = status; + return error; +} + +function namedError(name: string, message = name): Error { + const error = new Error(message); + error.name = name; + return error; +} + +describe("buildModelChain", () => { + test("puts the primary first and appends fallbacks", () => { + expect(buildModelChain("claude-opus-4-6")).toEqual(["claude-opus-4-6", "deepseek-flash"]); + }); + + test("drops the primary when it also appears in the fallback chain", () => { + expect(buildModelChain("deepseek-flash")).toEqual(["deepseek-flash"]); + }); + + test("never retries the same model twice", () => { + const chain = buildModelChain("a", ["a", "b", "a", "c"]); + expect(chain).toEqual(["a", "b", "c"]); + expect(new Set(chain).size).toBe(chain.length); + }); + + test("empty primary is ignored rather than attempted", () => { + expect(buildModelChain("", ["b"])).toEqual(["b"]); + }); +}); + +describe("classifyFailover", () => { + test("retries region blocks and auth failures — another provider may work", () => { + for (const status of [401, 403, 429, 502, 503]) { + const decision = classifyFailover(httpError(status)); + expect(decision.retryable).toBe(true); + expect(decision.kind).toBe("http"); + expect(decision.detail).toBe(`HTTP ${status}`); + } + }); + + test("does NOT retry a 400 — that is our own malformed request", () => { + expect(classifyFailover(httpError(400)).retryable).toBe(false); + }); + + test("does NOT retry a user abort — Esc must stay cancelled", () => { + const decision = classifyFailover(namedError("AbortError")); + expect(decision).toMatchObject({ retryable: false, retrySameProvider: false, kind: "abort" }); + }); + + test("retries a timeout — a hung provider should not end the turn", () => { + expect(classifyFailover(namedError("TimeoutError")).retryable).toBe(true); + }); + + test("retries connection-level failures", () => { + expect(classifyFailover(new Error("fetch failed")).kind).toBe("network"); + expect(classifyFailover(new Error("connect ECONNREFUSED 127.0.0.1:443")).retryable).toBe(true); + }); + + test("unknown errors are not retried", () => { + expect(classifyFailover(new Error("boom")).retryable).toBe(false); + }); + + test("reads status from a fetch-style response object", () => { + expect(classifyFailover({ response: { status: 403 } }).detail).toBe("HTTP 403"); + }); +}); + +describe("runWithFailover", () => { + test("returns the primary result without a downgrade when it works", async () => { + const outcome = await runWithFailover({ + primary: "deepseek-flash", + consent: true, + attempt: async (modelId) => `answer from ${modelId}`, + }); + + expect(outcome.value).toBe("answer from deepseek-flash"); + expect(outcome.modelId).toBe("deepseek-flash"); + expect(outcome.downgradedFrom).toBeNull(); + expect(outcome.failures).toEqual([]); + }); + + test("falls through a blocked provider and reports the downgrade", async () => { + const outcome = await runWithFailover({ + primary: "gpt-oss-120b", + consent: true, + attempt: async (modelId) => { + if (modelId === "gpt-oss-120b") throw httpError(403); + return `answer from ${modelId}`; + }, + }); + + expect(outcome.value).toBe("answer from deepseek-flash"); + expect(outcome.downgradedFrom).toBe("gpt-oss-120b"); + expect(outcome.failures).toEqual([{ modelId: "gpt-oss-120b", detail: "HTTP 403" }]); + }); + + test("tries every candidate before giving up, then rethrows the last error", async () => { + const tried: string[] = []; + + await expect( + runWithFailover({ + primary: "a", + consent: true, + fallbacks: ["b", "c"], + attemptsPerModel: 1, + attempt: async (modelId) => { + tried.push(modelId); + throw httpError(503); + }, + }), + ).rejects.toThrow("upstream failed with 503"); + + expect(tried).toEqual(["a", "b", "c"]); + }); + + test("stops immediately on a non-retryable failure", async () => { + const tried: string[] = []; + + await expect( + runWithFailover({ + primary: "a", + consent: true, + fallbacks: ["b"], + attempt: async (modelId) => { + tried.push(modelId); + throw httpError(400); + }, + }), + ).rejects.toThrow("upstream failed with 400"); + + expect(tried).toEqual(["a"]); + }); + + test("does not burn the chain when the user aborts", async () => { + const tried: string[] = []; + + await expect( + runWithFailover({ + primary: "a", + consent: true, + fallbacks: ["b"], + attempt: async (modelId) => { + tried.push(modelId); + throw namedError("AbortError"); + }, + }), + ).rejects.toThrow(); + + expect(tried).toEqual(["a"]); + }); + + test("reports every failed attempt to the caller", async () => { + const events: string[] = []; + + await runWithFailover({ + primary: "a", + consent: true, + fallbacks: ["b", "c"], + attemptsPerModel: 1, + attempt: async (modelId) => { + if (modelId !== "c") throw httpError(403); + return "ok"; + }, + onAttemptFailed: ({ modelId, decision }) => events.push(`${modelId}:${decision.detail}`), + }); + + expect(events).toEqual(["a:HTTP 403", "b:HTTP 403"]); + }); + + test("default chain starts with the reachable provider", () => { + expect(DEFAULT_FALLBACK_CHAIN[0]).toBe("deepseek-flash"); + }); + + test("the 4x-cost model is NOT a default fallback", () => { + // Silently escalating someone's spend is not this code's call. + expect(DEFAULT_FALLBACK_CHAIN).not.toContain("deepseek-v4-pro"); + }); + + test("failover requires explicit consent", async () => { + const tried: string[] = []; + + await expect( + runWithFailover({ + primary: "gpt-oss-120b", + consent: false, + attemptsPerModel: 1, + attempt: async (modelId) => { + tried.push(modelId); + throw httpError(403); + }, + }), + ).rejects.toThrow(); + + expect(tried).toEqual(["gpt-oss-120b"]); + }); +}); + +describe("retries keep a valid key sufficient", () => { + const noWait = async () => {}; + + test("a transient failure is retried and the turn completes", async () => { + let calls = 0; + + const outcome = await runWithFailover({ + primary: "deepseek-flash", + consent: false, + attemptsPerModel: 3, + sleep: noWait, + attempt: async (modelId) => { + calls += 1; + if (calls < 3) throw httpError(429); + return `ok from ${modelId}`; + }, + }); + + expect(outcome.value).toBe("ok from deepseek-flash"); + expect(calls).toBe(3); + }); + + test("exhausting retries surfaces the last error", async () => { + let calls = 0; + + await expect( + runWithFailover({ + primary: "deepseek-flash", + consent: false, + attemptsPerModel: 2, + sleep: noWait, + attempt: async () => { + calls += 1; + throw httpError(503); + }, + }), + ).rejects.toThrow("upstream failed with 503"); + + expect(calls).toBe(2); + }); + + test("a non-retryable error is not retried", async () => { + let calls = 0; + + await expect( + runWithFailover({ + primary: "deepseek-flash", + consent: false, + attemptsPerModel: 3, + sleep: noWait, + attempt: async () => { + calls += 1; + throw httpError(400); + }, + }), + ).rejects.toThrow(); + + expect(calls).toBe(1); + }); + + test("A timeout is retried — a stalled stream is not the user's fault", async () => { + let calls = 0; + + await runWithFailover({ + primary: "deepseek-flash", + consent: false, + attemptsPerModel: 2, + sleep: noWait, + attempt: async () => { + calls += 1; + if (calls === 1) throw namedError("TimeoutError"); + return "recovered"; + }, + }); + + expect(calls).toBe(2); + }); + + test("backoff grows and stays bounded", () => { + expect(retryDelayMs(1, 500)).toBeLessThan(1_001); + expect(retryDelayMs(2, 500)).toBeLessThan(1_501); + expect(retryDelayMs(10, 500)).toBeLessThanOrEqual(8_500); + }); +}); diff --git a/packages/cli/src/lib/provider-chain.ts b/packages/cli/src/lib/provider-chain.ts new file mode 100644 index 0000000..ad3af7f --- /dev/null +++ b/packages/cli/src/lib/provider-chain.ts @@ -0,0 +1,285 @@ +/** + * Model failover — retry a turn on the next model when the current provider fails. + * + * Why: `resolveChatModel` hard-binds one provider per model. On a network where that + * provider is region-blocked, 13 of 15 catalog models were unusable (see + * `packages/server/scripts/provider-failover-bench.ts`). A single unreachable + * provider meant 100% of turns failed with a bare `Forbidden`. + * + * This module is pure orchestration — no I/O, no React. Callers supply an `attempt` + * thunk; the module decides whether a failure is worth retrying, walks the chain, + * and reports which model actually answered so the downgrade can be shown to the user. + * + * A downgrade is never silent: `downgradedFrom` is non-null whenever a fallback + * answered, and callers must surface it. + */ +import type { SupportedChatModelId } from "@mocode/shared"; + +/** + * Ordered fallback chain. + * + * `deepseek-v4-pro` is deliberately NOT here: it costs roughly four times + * `deepseek-flash`, and silently escalating a user's spend because the cheap model + * was rate-limited is not a decision this code gets to make. Opt in explicitly via + * `MOCODE_FALLBACK_CHAIN` when the extra cost is acceptable. + */ +export const DEFAULT_FALLBACK_CHAIN: readonly SupportedChatModelId[] = ["deepseek-flash"]; + +/** + * Whether the operator has authorised cross-provider failover. + * + * Off unless explicitly enabled: retrying on another vendor sends the same code to + * a different company, which needs a decision, not a default. + */ +export function failoverConsentFromEnv(): boolean { + return process.env.MOCODE_ALLOW_MODEL_FAILOVER === "1"; +} + +/** Fallback chain, overridable by the operator. */ +export function fallbackChainFromEnv(): readonly string[] { + const configured = process.env.MOCODE_FALLBACK_CHAIN?.split(",") + .map((id) => id.trim()) + .filter(Boolean); + + return configured && configured.length > 0 ? configured : DEFAULT_FALLBACK_CHAIN; +} + +/** HTTP statuses that mean "this provider cannot serve us — try another". */ +const RETRYABLE_STATUS = new Set([401, 403, 404, 408, 429, 500, 502, 503, 504]); + +/** Subset that is worth retrying against the *same* provider. */ +const SAME_PROVIDER_RETRY_STATUS = new Set([408, 429, 500, 502, 503, 504]); + +export type FailoverDecision = { + /** Worth trying *somewhere* — another provider or a later attempt. */ + retryable: boolean; + /** + * Worth retrying against the **same** provider. + * + * A 403/401/404 will answer identically in 500ms; retrying it only delays the + * fallback and burns the user's patience. 429/5xx/timeouts are transient and do + * deserve a backed-off retry. + */ + retrySameProvider: boolean; + /** Short machine-readable bucket, used in logs and tests. */ + kind: "http" | "timeout" | "abort" | "network" | "unknown"; + detail: string; +}; + +/** + * Build the attempt order for a primary model: primary first, then the fallbacks, + * de-duplicated. A fallback equal to the primary is dropped rather than retried + * against the same broken provider. + */ +export function buildModelChain( + primary: string, + fallbacks: readonly string[] = DEFAULT_FALLBACK_CHAIN, +): string[] { + const seen = new Set(); + const chain: string[] = []; + + for (const id of [primary, ...fallbacks]) { + if (!id || seen.has(id)) continue; + seen.add(id); + chain.push(id); + } + + return chain; +} + +/** Read a numeric HTTP status off an SDK/`fetch` error, if there is one. */ +function statusOf(error: unknown): number | null { + if (typeof error !== "object" || error === null) return null; + + const candidate = error as { + statusCode?: unknown; + status?: unknown; + response?: { status?: unknown }; + }; + + for (const value of [candidate.statusCode, candidate.status, candidate.response?.status]) { + if (typeof value === "number") return value; + } + + return null; +} + +/** + * Decide whether a failed attempt is worth retrying on another model. + * + * Deliberately conservative: a 400 means *we* built a bad request, and a user abort + * (Esc) must never be retried — retrying either would mask the real problem. + */ +export function classifyFailover(error: unknown): FailoverDecision { + const status = statusOf(error); + if (status !== null) { + return { + retryable: RETRYABLE_STATUS.has(status), + // Auth/URL rejections are not transient; rate limits and server faults are. + retrySameProvider: SAME_PROVIDER_RETRY_STATUS.has(status), + kind: "http", + detail: `HTTP ${status}`, + }; + } + + const name = error instanceof Error ? error.name : ""; + + if (name === "AbortError") { + return { retryable: false, retrySameProvider: false, kind: "abort", detail: "aborted by caller" }; + } + + if (name === "TimeoutError") { + return { retryable: true, retrySameProvider: true, kind: "timeout", detail: "attempt timed out" }; + } + + const message = error instanceof Error ? error.message : String(error); + + if (/fetch failed|ECONNREFUSED|ENOTFOUND|ETIMEDOUT|socket hang up|network/i.test(message)) { + return { retryable: true, retrySameProvider: true, kind: "network", detail: message.slice(0, 80) }; + } + + return { retryable: false, retrySameProvider: false, kind: "unknown", detail: message.slice(0, 80) }; +} + +/** What the caller must surface: which model answered, and what it replaced. */ +export type FailoverOutcome = { + value: T; + /** Model that actually produced the result. */ + modelId: string; + /** Non-null when a fallback answered — the UI must show this. */ + downgradedFrom: string | null; + /** Failures that happened before the successful attempt. */ + failures: Array<{ modelId: string; detail: string }>; +}; + +/** Attempts per model before giving up on it. Retries are what make a valid key + * sufficient: a transient 429 or stalled connection should not end the turn. */ +export function attemptsPerModelFromEnv(): number { + const configured = Number(process.env.MOCODE_PROVIDER_RETRIES); + return Number.isFinite(configured) && configured >= 1 ? Math.floor(configured) : 3; +} + +/** Exponential backoff with jitter: 500ms, 1s, 2s … capped at 8s. */ +export function retryDelayMs(attempt: number, baseMs = 500): number { + const exponential = Math.min(baseMs * 2 ** (attempt - 1), 8_000); + return exponential + Math.floor(Math.random() * baseMs); +} + +export type FailoverOptions = { + primary: string; + fallbacks?: readonly string[]; + /** Attempts per model. Defaults to `MOCODE_PROVIDER_RETRIES` or 3. */ + attemptsPerModel?: number; + /** Base backoff delay. */ + retryBaseMs?: number; + /** Injectable for tests so they do not actually wait. */ + sleep?: (ms: number) => Promise; + /** + * Explicit authorisation to send the turn to a *different* provider. + * + * Required, with no default: switching provider means the user's code leaves for + * a vendor they did not pick. Without consent only the selected model is tried + * and its failure surfaces unchanged. + */ + consent: boolean; + /** One attempt against one model. Throw to signal failure. */ + attempt: (modelId: string) => Promise; + /** Called for every failure so callers can log it. */ + onAttemptFailed?: (event: { modelId: string; decision: FailoverDecision }) => void; +}; + +/** + * Run `attempt` down the chain until one succeeds. + * + * Re-throws the *last* error when every candidate fails, so the original cause is + * not swallowed. Non-retryable failures (bad request, user abort) throw immediately + * without burning fallbacks. + */ +export async function runWithFailover({ + primary, + fallbacks = fallbackChainFromEnv(), + consent, + attemptsPerModel = attemptsPerModelFromEnv(), + retryBaseMs = 500, + sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + attempt, + onAttemptFailed, +}: FailoverOptions): Promise> { + if (!consent) { + // No authorisation to switch providers — but retrying the *selected* provider + // is not a privacy decision, and a transient failure must not end the turn. + const value = await attemptWithRetries(primary, attempt, { + attemptsPerModel, + retryBaseMs, + sleep, + onAttemptFailed, + }); + return { value, modelId: primary, downgradedFrom: null, failures: [] }; + } + + const chain = buildModelChain(primary, fallbacks); + const failures: FailoverOutcome["failures"] = []; + let lastError: unknown; + + for (const modelId of chain) { + try { + const value = await attemptWithRetries(modelId, attempt, { + attemptsPerModel, + retryBaseMs, + sleep, + onAttemptFailed, + }); + + return { + value, + modelId, + downgradedFrom: modelId === primary ? null : primary, + failures, + }; + } catch (error) { + const decision = classifyFailover(error); + failures.push({ modelId, detail: decision.detail }); + if (!decision.retryable) throw error; + lastError = error; + } + } + + throw lastError ?? new Error("No model candidates available"); +} + +type RetryContext = { + attemptsPerModel: number; + retryBaseMs: number; + sleep: (ms: number) => Promise; + onAttemptFailed?: FailoverOptions["onAttemptFailed"]; +}; + +/** One model, up to `attemptsPerModel` tries with backoff. Non-retryable throws at once. */ +async function attemptWithRetries( + modelId: string, + attempt: (modelId: string) => Promise, + context: RetryContext, +): Promise { + let lastError: unknown; + + for (let attemptNo = 1; attemptNo <= context.attemptsPerModel; attemptNo += 1) { + try { + return await attempt(modelId); + } catch (error) { + const decision = classifyFailover(error); + context.onAttemptFailed?.({ modelId, decision }); + + if (!decision.retryable) throw error; + lastError = error; + + // A rejection that will repeat verbatim is not worth waiting for. + if (!decision.retrySameProvider) throw error; + + if (attemptNo < context.attemptsPerModel) { + await context.sleep(retryDelayMs(attemptNo, context.retryBaseMs)); + } + } + } + + throw lastError ?? new Error(`All ${context.attemptsPerModel} attempts failed for ${modelId}`); +} diff --git a/packages/cli/src/lib/sandbox-capability.test.ts b/packages/cli/src/lib/sandbox-capability.test.ts new file mode 100644 index 0000000..ed08d0e --- /dev/null +++ b/packages/cli/src/lib/sandbox-capability.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, test } from "bun:test"; +import { + detectSandbox, + probeBwrap, + probeSeatbelt, + sandboxCapability, + resetSandboxCapability, + type ProbeRunner, +} from "./sandbox-capability"; + +const applies: ProbeRunner = () => ({ exitCode: 0, output: "" }); +const refuses: ProbeRunner = () => ({ + exitCode: 1, + output: "sandbox-exec: sandbox_apply: Operation not permitted", +}); + +describe("seatbelt probe", () => { + test("available when the profile actually applies", () => { + expect(probeSeatbelt(applies)).toMatchObject({ backend: "seatbelt", available: true }); + }); + + test("unavailable when the binary exists but cannot apply", () => { + // The real-world case: /usr/bin/sandbox-exec is present, nesting is refused. + const capability = probeSeatbelt(refuses); + expect(capability.available).toBe(false); + expect(capability.detail).toContain("Operation not permitted"); + }); +}); + +describe("bwrap probe", () => { + test("available when a bind mount succeeds", () => { + expect(probeBwrap(applies)).toMatchObject({ backend: "bwrap", available: true }); + }); + + test("unavailable when bwrap is missing", () => { + const capability = probeBwrap(() => ({ exitCode: 127, output: "bwrap: command not found" })); + expect(capability.available).toBe(false); + }); +}); + +describe("detectSandbox", () => { + test("picks the backend by platform and never claims capability it cannot prove", () => { + const capability = detectSandbox(refuses); + expect(capability.available).toBe(false); + expect(["seatbelt", "bwrap", "none"]).toContain(capability.backend); + }); +}); + +describe("sandboxCapability memoisation", () => { + test("env opt-out reports unconfined without probing", () => { + resetSandboxCapability(); + process.env.MOCODE_ALLOW_UNCONFINED_SHELL = "1"; + try { + expect(sandboxCapability().available).toBe(false); + expect(sandboxCapability().detail).toContain("unconfined"); + } finally { + delete process.env.MOCODE_ALLOW_UNCONFINED_SHELL; + resetSandboxCapability(); + } + }); + + test("the answer is stable for the process lifetime", () => { + resetSandboxCapability(); + const first = sandboxCapability(); + const second = sandboxCapability(); + expect(second).toBe(first); + resetSandboxCapability(); + }); +}); + +describe("capability vs enforcement", () => { + test("a successful probe now reports enforcement, because the spawn site wraps it", () => { + const capability = probeSeatbelt(applies); + expect(capability.available).toBe(true); + expect(capability.enforced).toBe(true); + }); + + test("a failed probe reports neither", () => { + const capability = probeSeatbelt(refuses); + expect(capability.available).toBe(false); + expect(capability.enforced).toBe(false); + }); + + test("isShellConfined mirrors the live probe on this host", async () => { + const { detectSandbox, isShellConfined } = await import("./sandbox-capability"); + resetSandboxCapability(); + const capability = detectSandbox(); + expect(isShellConfined()).toBe(capability.available && capability.enforced); + }); + + test("sandboxBackend mirrors the live probe on this host", async () => { + const { detectSandbox, sandboxBackend } = await import("./sandbox-capability"); + resetSandboxCapability(); + const capability = detectSandbox(); + expect(sandboxBackend()).toBe( + capability.available && capability.enforced ? capability.backend : "none", + ); + }); + + test("MOCODE_DISABLE_SANDBOX forces none", async () => { + const { sandboxBackend } = await import("./sandbox-capability"); + process.env.MOCODE_DISABLE_SANDBOX = "1"; + try { + expect(sandboxBackend()).toBe("none"); + } finally { + delete process.env.MOCODE_DISABLE_SANDBOX; + resetSandboxCapability(); + } + }); +}); diff --git a/packages/cli/src/lib/sandbox-capability.ts b/packages/cli/src/lib/sandbox-capability.ts new file mode 100644 index 0000000..7cf0a90 --- /dev/null +++ b/packages/cli/src/lib/sandbox-capability.ts @@ -0,0 +1,174 @@ +/** + * Shell sandbox capability probe. + * + * Why this exists: path containment (`sandbox-path.ts`) only guards the file + * tools. A shell cannot be contained by path checks — `head /etc/passwd`, + * `cat link-to-etc`, `cd ..` all escape anything enforced in-process. The only + * real boundary is an OS-level sandbox. + * + * So the question is not "should bash be sandboxed" but "is this machine even + * capable of it". This module answers that once, at first use, and the rest of the + * system fails closed when the answer is no. + * + * Probes are injectable so tests do not depend on the host kernel. + */ + +import { BWRAP_BIN, SEATBELT_BIN } from "./sandbox-profile"; + +export type SandboxBackend = "seatbelt" | "bwrap" | "none"; + +export type SandboxCapability = { + backend: SandboxBackend; + /** The host *can* apply a sandbox (a probe actually ran successfully). */ + available: boolean; + /** + * The executor *does* apply it to spawned shells. + * + * Capability is not enforcement — reporting `available` as if it were confined + * is exactly the class of bug this whole module exists to prevent. Until the + * spawn site wraps the profile, every backend reports `enforced: false` and the + * approval policy stays fail-closed. + */ + enforced: boolean; + /** Why it is unavailable — surfaced in audit and diagnostics. */ + detail: string; +}; + +/** Runs a command and reports the exit code; split out so tests can fake it. */ +export type ProbeRunner = (command: string[]) => { exitCode: number; output: string }; + +const defaultRunner: ProbeRunner = (command) => { + try { + const result = Bun.spawnSync(command, { stdout: "pipe", stderr: "pipe" }); + return { + exitCode: result.exitCode ?? 1, + output: `${result.stdout.toString()}${result.stderr.toString()}`.trim(), + }; + } catch (error) { + return { exitCode: 1, output: error instanceof Error ? error.message : "probe failed" }; + } +}; + +/** + * Deny-default probe that mirrors the production seatbelt shape. + * + * `(allow default)` is too weak: it succeeds on hosts where the real + * deny-default profile SIGABRTs. Probe with deny-default + the broad read + * allow that modern macOS needs, so "available" means spawn will work. + */ +const SEATBELT_PROBE_PROFILE = [ + "(version 1)", + "(deny default)", + "(allow process*)", + "(allow sysctl-read)", + "(allow mach-lookup)", + '(allow file-read* (subpath "/"))', + "(allow file-map-executable)", +].join(""); + +/** + * macOS: `sandbox-exec` may exist yet fail to apply (nested sandbox, hardened + * runtime, SIP). Existence is not capability — we must run a deny-default profile. + */ +export function probeSeatbelt(run: ProbeRunner = defaultRunner): SandboxCapability { + const probe = run(["sandbox-exec", "-p", SEATBELT_PROBE_PROFILE, "/usr/bin/true"]); + + if (probe.exitCode === 0) { + return { + backend: "seatbelt", + available: true, + enforced: true, + detail: "sandbox-exec applies; profile applied at spawn", + }; + } + + return { + backend: "seatbelt", + available: false, + enforced: false, + detail: probe.output.slice(0, 120) || "sandbox-exec present but could not apply", + }; +} + +/** Linux: bubblewrap is the portable answer; a failed bind mount means no capability. */ +export function probeBwrap(run: ProbeRunner = defaultRunner): SandboxCapability { + const probe = run([BWRAP_BIN, "--ro-bind", "/", "/", "--dev", "/dev", "true"]); + + if (probe.exitCode === 0) { + return { backend: "bwrap", available: true, enforced: true, detail: "bwrap applies; profile applied at spawn" }; + } + + return { + backend: "bwrap", + available: false, + enforced: false, + detail: probe.output.slice(0, 120) || "bwrap unavailable", + }; +} + +/** Detect the platform's backend without caching. */ +export function detectSandbox(run: ProbeRunner = defaultRunner): SandboxCapability { + if (process.platform === "darwin") return probeSeatbelt(run); + if (process.platform === "linux") return probeBwrap(run); + + return { + backend: "none", + available: false, + enforced: false, + detail: `no sandbox backend on ${process.platform}`, + }; +} + +let cached: SandboxCapability | null = null; + +/** + * Cached capability for the process. + * + * An explicit opt-out exists because failing closed is heavy: when no sandbox is + * available every bash command needs approval. `MOCODE_ALLOW_UNCONFINED_SHELL=1` + * accepts that risk knowingly rather than silently. + */ +export function sandboxCapability(): SandboxCapability { + if (cached) return cached; + + if (process.env.MOCODE_ALLOW_UNCONFINED_SHELL === "1") { + cached = { backend: "none", available: false, enforced: false, detail: "unconfined shell allowed by env" }; + return cached; + } + + cached = detectSandbox(); + return cached; +} + +/** + * True only when a spawned shell is *actually* contained. + * + * Deliberately `available && enforced`: a host that can sandbox but whose spawn + * site does not wrap the profile is still unconfined. + */ +export function isShellConfined(): boolean { + // Must be derived from what the spawn site will actually do, not from what the + // host *can* do: `MOCODE_DISABLE_SANDBOX=1` turns confinement off without + // changing the probe result, and an unconfined shell must never be reported as + // confined — the approval policy keys off this. + return sandboxBackend() !== "none"; +} + +/** + * Backend the spawn site should use right now, or `none`. + * + * `MOCODE_DISABLE_SANDBOX=1` exists for environments where seatbelt/bwrap breaks + * legitimate work; it turns confinement off *and* leaves the approval policy + * fail-closed, because `isShellConfined()` then reports false. + */ +export function sandboxBackend(): SandboxBackend { + if (process.env.MOCODE_DISABLE_SANDBOX === "1") return "none"; + + const capability = sandboxCapability(); + return capability.available && capability.enforced ? capability.backend : "none"; +} + +/** Test seam: forget the memoised probe. */ +export function resetSandboxCapability(): void { + cached = null; +} diff --git a/packages/cli/src/lib/sandbox-path.test.ts b/packages/cli/src/lib/sandbox-path.test.ts new file mode 100644 index 0000000..f9b7a38 --- /dev/null +++ b/packages/cli/src/lib/sandbox-path.test.ts @@ -0,0 +1,92 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveInsideCwd } from "./sandbox-path"; + +const root = mkdtempSync(join(tmpdir(), "mocode-sandbox-")); +const outside = mkdtempSync(join(tmpdir(), "mocode-outside-")); + +writeFileSync(join(root, "inside.txt"), "ok"); +writeFileSync(join(outside, "secret.txt"), "top secret"); +mkdirSync(join(root, "nested"), { recursive: true }); + +// The escape hatch: a link that lives inside the project but points outside it. +symlinkSync(outside, join(root, "escape")); +// A link to a single outside file. +symlinkSync(join(outside, "secret.txt"), join(root, "escape-file")); +// A link whose target does not exist: walking past it would resolve the parent +// (inside cwd) and accept a path that writes through the link. +symlinkSync(join(outside, "missing-target"), join(root, "dangling")); + +afterAll(() => { + rmSync(root, { recursive: true, force: true }); + rmSync(outside, { recursive: true, force: true }); +}); + +describe("resolveInsideCwd — lexical escapes", () => { + test("accepts a plain relative path", () => { + expect(resolveInsideCwd(root, "inside.txt")).toBe(join(root, "inside.txt")); + }); + + test("accepts the project root itself", () => { + expect(resolveInsideCwd(root, ".")).toBe(root); + }); + + test("accepts a nested path that does not exist yet (writeFile case)", () => { + expect(resolveInsideCwd(root, "a/b/c/new.ts")).toBe(join(root, "a/b/c/new.ts")); + }); + + test("rejects parent traversal", () => { + expect(() => resolveInsideCwd(root, "../../etc/passwd")).toThrow( + "Path is outside the project directory", + ); + }); + + test("accepts a contained filename that merely begins with two dots", () => { + writeFileSync(join(root, "..config"), "ok"); + expect(resolveInsideCwd(root, "..config")).toBe(join(root, "..config")); + }); + + test("rejects traversal that lands back inside via a sibling prefix", () => { + // `..` must not be treated as a prefix of a legitimate directory name. + expect(() => resolveInsideCwd(root, "../mocode-sandbox-evil/x")).toThrow( + "Path is outside the project directory", + ); + }); + + test("rejects an absolute path outside the project", () => { + expect(() => resolveInsideCwd(root, "/etc/passwd")).toThrow( + "Path is outside the project directory", + ); + }); + + test("does not expand ~ into the home directory", () => { + // `~` is not special to the tool layer; it must stay a literal name inside cwd. + expect(resolveInsideCwd(root, "~/.ssh/id_rsa")).toBe(join(root, "~/.ssh/id_rsa")); + }); +}); + +describe("resolveInsideCwd — symlink escapes", () => { + test("rejects a directory symlink pointing outside the project", () => { + expect(() => resolveInsideCwd(root, "escape/secret.txt")).toThrow( + "Path is outside the project directory", + ); + }); + + test("rejects a file symlink pointing outside the project", () => { + expect(() => resolveInsideCwd(root, "escape-file")).toThrow( + "Path is outside the project directory", + ); + }); + + test("rejects a dangling symlink instead of walking past it", () => { + expect(() => resolveInsideCwd(root, "dangling")).toThrow( + "Path is outside the project directory", + ); + }); + + test("still accepts a normal nested directory", () => { + expect(resolveInsideCwd(root, "nested/file.ts")).toBe(join(root, "nested/file.ts")); + }); +}); diff --git a/packages/cli/src/lib/sandbox-path.ts b/packages/cli/src/lib/sandbox-path.ts new file mode 100644 index 0000000..2cd03ee --- /dev/null +++ b/packages/cli/src/lib/sandbox-path.ts @@ -0,0 +1,85 @@ +/** + * Path containment for agent tools. + * + * The previous check was purely lexical: `resolve()` then reject anything starting + * with `..`. That stops `../../etc/passwd` but not a symlink, because the symlink + * itself lives inside the project while pointing outside it: + * + * ln -s /etc ./docs + * readFile("docs/passwd") ← lexically inside, actually /etc/passwd + * + * This module checks both: the lexical path *and* the real path of the deepest + * existing ancestor (so it works for files that do not exist yet, e.g. writeFile + * creating a new directory tree). + */ +import { lstatSync, realpathSync } from "node:fs"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; + +function outsideError(): Error { + return new Error("Path is outside the project directory"); +} + +/** Throw unless `candidate` is `root` or lives beneath it. */ +function assertInside(root: string, candidate: string): void { + const rel = relative(root, candidate); + if (rel === "") return; + // Reject only the parent segment itself (`..`) or a parent walk (`../…`). + // Filenames that merely begin with two dots (e.g. `..config`) are contained. + if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) throw outsideError(); +} + +/** Real path, or the input when the file does not exist. */ +function realpathOrSelf(path: string): string { + try { + return realpathSync(path); + } catch { + return path; + } +} + +/** + * Real path of the deepest ancestor that exists. Resolving the ancestor (rather + * than the leaf) keeps this usable for paths that are about to be created. + */ +function realAncestor(path: string): string { + let current = path; + + while (true) { + try { + return realpathSync(current); + } catch { + // A dangling symlink is NOT a "missing component". Walking past it would + // resolve the parent (inside cwd) and accept a path that actually writes + // through the link to a target outside the project. + if (isDanglingSymlink(current)) throw outsideError(); + + const parent = dirname(current); + if (parent === current) return current; + current = parent; + } + } +} + +/** True when `path` exists as a symlink whose target cannot be resolved. */ +function isDanglingSymlink(path: string): boolean { + try { + return lstatSync(path).isSymbolicLink(); + } catch { + return false; + } +} + +/** + * Resolve `target` under `cwd`, rejecting both lexical escapes and symlink escapes. + * + * @throws when the path leaves the project directory by either route. + */ +export function resolveInsideCwd(cwd: string, target: string): string { + const resolved = resolve(cwd, target); + assertInside(cwd, resolved); + + // Symlink check: compare real paths so `docs -> /etc` is caught. + assertInside(realpathOrSelf(cwd), realAncestor(resolved)); + + return resolved; +} diff --git a/packages/cli/src/lib/sandbox-profile.test.ts b/packages/cli/src/lib/sandbox-profile.test.ts new file mode 100644 index 0000000..af1d6e5 --- /dev/null +++ b/packages/cli/src/lib/sandbox-profile.test.ts @@ -0,0 +1,124 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + buildBwrapArgs, + buildSeatbeltProfile, + parsePathList, + runtimePaths, + spawnPlan, +} from "./sandbox-profile"; + +const base = { cwd: "/Users/dev/project", home: "/Users/dev", tmpDir: "/tmp" }; + +describe("seatbelt profile", () => { + const profile = buildSeatbeltProfile(base); + + test("denies by default — that is the whole guarantee", () => { + expect(profile).toContain("(version 1)"); + expect(profile.split("\n")[1]).toBe("(deny default)"); + }); + + test("allows the project tree", () => { + expect(profile).toContain('(subpath "/Users/dev/project")'); + }); + + test("allows broad reads under / (required on modern macOS) then denies secrets", () => { + // Narrow subpath allowlists SIGABRT on dyld/cryptex hosts; deny comes after. + expect(profile).toContain('(allow file-read* (subpath "/"))'); + expect(profile).toContain('(subpath "/etc/passwd")'); + expect(profile).toContain('(subpath "/Users/dev/.ssh")'); + const denyLine = profile.split("\n").find((line) => line.startsWith("(deny file-read*")); + expect(denyLine).toBeDefined(); + }); + + test("keeps the writable set tight", () => { + const writeLine = profile.split("\n").find((line) => line.startsWith("(allow file-write*")); + expect(writeLine).toContain('(subpath "/Users/dev/project")'); + expect(writeLine).not.toContain('(subpath "/Users/dev")'); + }); + + test("escapes quotes so a hostile directory name cannot break the profile", () => { + const tricky = buildSeatbeltProfile({ cwd: '/tmp/we"ird\\(dir)' }); + expect(tricky).toContain('/tmp/we\\"ird\\\\(dir)'); + }); + + test("lists the project once in the write allow", () => { + const duplicated = buildSeatbeltProfile({ + ...base, + extraWritePaths: ["/Users/dev/project", "/tmp/extra"], + }); + const occurrences = duplicated.split('(subpath "/Users/dev/project")').length - 1; + expect(occurrences).toBe(1); + }); +}); + +describe("bwrap args", () => { + // bwrap refuses to start when a bind target is missing, so masks are only emitted + // for paths that exist. Build a throwaway home that has them. + const fakeHome = mkdtempSync(join(tmpdir(), "mocode-bwrap-home-")); + mkdirSync(join(fakeHome, ".ssh"), { recursive: true }); + writeFileSync(join(fakeHome, ".npmrc"), "//registry/:_authToken=x"); + + const args = buildBwrapArgs({ ...base, home: fakeHome }); + + test("mounts a read-only root and a writable project", () => { + expect(args.join(" ")).toContain("--ro-bind / /"); + expect(args.join(" ")).toContain("--bind /Users/dev/project /Users/dev/project"); + }); + + test("masks sensitive home paths", () => { + const joined = args.join(" "); + // Directories are emptied with a tmpfs; binding a *file* over a directory makes + // bubblewrap refuse to start. + expect(joined).not.toContain(`--ro-bind /dev/null ${fakeHome}/.ssh`); + // Files are bound over /dev/null. + expect(joined).toContain(`--ro-bind /dev/null ${fakeHome}/.npmrc`); + // Directories are emptied instead. + expect(joined).toContain(`--tmpfs ${fakeHome}/.ssh`); + }); + + test("always mounts a private tmpfs for TMPDIR", () => { + expect(args.join(" ")).toContain("--tmpfs /tmp"); + }); + + test("does not share the parent's pid namespace", () => { + expect(args).not.toContain("--unshare-pid"); + }); + + afterAll(() => rmSync(fakeHome, { recursive: true, force: true })); +}); + + +describe("spawnPlan", () => { + test("seatbelt wraps bash in sandbox-exec", () => { + const plan = spawnPlan("seatbelt", "echo hi", base); + expect(plan).toMatchObject({ file: "/usr/bin/sandbox-exec", confined: true, backend: "seatbelt" }); + expect(plan.args.slice(-3)).toEqual(["bash", "-c", "echo hi"]); + expect(plan.args[0]).toBe("-p"); + }); + + test("bwrap prefixes the command", () => { + const plan = spawnPlan("bwrap", "echo hi", base); + expect(plan).toMatchObject({ file: "/usr/bin/bwrap", confined: true }); + expect(plan.args.slice(-3)).toEqual(["bash", "-c", "echo hi"]); + }); + + test("none is honest: returns a working shell and says it is unconfined", () => { + const plan = spawnPlan("none", "echo hi", base); + expect(plan).toMatchObject({ file: "bash", confined: false, backend: "none" }); + expect(plan.args).toEqual(["-c", "echo hi"]); + }); +}); + +describe("helpers", () => { + test("parsePathList splits, trims and drops empties", () => { + expect(parsePathList("/a:/b : /c::")).toEqual(["/a", "/b", "/c"]); + expect(parsePathList(undefined)).toEqual([]); + }); + + test("runtimePaths keeps the interpreter directory readable", () => { + expect(runtimePaths("/Users/dev/.bun/bin/bun")).toEqual(["/Users/dev/.bun/bin"]); + }); +}); diff --git a/packages/cli/src/lib/sandbox-profile.ts b/packages/cli/src/lib/sandbox-profile.ts new file mode 100644 index 0000000..9283d2b --- /dev/null +++ b/packages/cli/src/lib/sandbox-profile.ts @@ -0,0 +1,247 @@ +/** + * OS-level sandbox profiles and spawn plans. + * + * The path checks in `sandbox-path.ts` cannot contain a shell — this module is what + * actually does. Given a backend, it produces either an SBPL profile (macOS) or a + * bubblewrap argument list (Linux), plus the exact argv to spawn. + * + * Everything here is a pure function on purpose: a sandbox whose profile is wrong + * fails silently in the worst possible direction, so the profile is asserted in + * tests rather than eyeballed in production. + * + * Policy: **deny by default, then allow what a shell needs, then deny secrets.** + * On modern macOS a narrow read-allowlist SIGABRTs (dyld/cryptex paths), so reads + * are allowed under `/` and sensitive trees are denied afterward. Writes stay + * limited to the project and tmp. + */ +import { existsSync } from "node:fs"; +import { dirname } from "node:path"; + +/** Sandbox binaries are addressed absolutely: a bare name resolves against the + * allowlisted env we hand the child, and a miss looks like "the sandbox broke". */ +export const SEATBELT_BIN = "/usr/bin/sandbox-exec"; +export const BWRAP_BIN = "/usr/bin/bwrap"; + +export type SandboxBackend = "seatbelt" | "bwrap" | "none"; + +export type SandboxProfileOptions = { + /** Project root: the only tree that is writable. */ + cwd: string; + /** Used for tmp paths and the extra allowlist defaults. */ + home?: string; + tmpDir?: string; + /** Additional read-only trees (colon-separated `MOCODE_SANDBOX_EXTRA_READ`). */ + extraReadPaths?: string[]; + /** Additional writable trees (`MOCODE_SANDBOX_EXTRA_WRITE`). */ + extraWritePaths?: string[]; + /** Runtime install dirs that must stay readable (bun/node themselves). */ + runtimePaths?: string[]; +}; + +/** Escape a path for SBPL string literals. */ +function sbplString(value: string): string { + return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; +} + +function uniq(values: string[]): string[] { + return [...new Set(values.filter((value) => value.length > 0))]; +} + +/** Split a colon-separated env var into paths. */ +export function parsePathList(value: string | undefined): string[] { + if (!value) return []; + return value + .split(":") + .map((entry) => entry.trim()) + .filter(Boolean); +} + +/** + * Sensitive paths denied after the broad filesystem read allow. + * + * Mirrors the bwrap mask list so both backends refuse the same secret trees. + */ +function deniedReadPaths(home: string | undefined): string[] { + const system = [ + "/etc/passwd", + "/etc/shadow", + "/etc/sudoers", + "/private/etc/passwd", + "/private/etc/shadow", + "/private/etc/sudoers", + "/var/root", + "/root", + ]; + const user = home + ? [ + `${home}/.ssh`, + `${home}/.aws`, + `${home}/.gnupg`, + `${home}/.config/gh`, + `${home}/.config/gcloud`, + `${home}/.kube`, + `${home}/.docker`, + // Package-manager config routinely carries registry tokens. + `${home}/.npmrc`, + `${home}/.yarnrc`, + `${home}/.pypirc`, + `${home}/.netrc`, + ] + : []; + return uniq([...system, ...user]); +} + +/** + * Build the macOS seatbelt profile. + * + * `(deny default)` first. Broad `(allow file-read* (subpath "/"))` is required on + * current macOS — a narrow subpath allowlist aborts the child before it starts. + * Sensitive paths are denied afterward (later SBPL rules win for the same op). + * Writes stay limited to the project / tmp. + */ +export function buildSeatbeltProfile(options: SandboxProfileOptions): string { + const { cwd } = options; + const writePaths = uniq([ + cwd, + ...(options.tmpDir ? [options.tmpDir] : []), + ...(options.extraWritePaths ?? []), + "/dev/null", + "/dev/tty", + ]); + const denied = deniedReadPaths(options.home); + + const writeRules = writePaths.map((path) => `(subpath ${sbplString(path)})`).join(" "); + const denyRules = denied.map((path) => `(subpath ${sbplString(path)})`).join(" "); + + return [ + "(version 1)", + "(deny default)", + // Process control is required by any shell. + "(allow process*)", + "(allow signal)", + "(allow sysctl-read)", + // Lookups for system services; keep to the operation names that have been + // stable across macOS releases. An unknown operation makes `sandbox_apply` + // fail as a whole, which shows up as "the command produced no output". + "(allow mach-lookup)", + "(allow file-read-metadata)", + // Broad read: narrow allowlists SIGABRT on dyld/cryptex hosts. + '(allow file-read* (subpath "/"))', + // Executables must be mappable, otherwise the dynamic loader fails. + "(allow file-map-executable)", + `(allow file-write* ${writeRules})`, + "(allow network*)", + // Later rules win — block the secret trees after the broad allow. + `(deny file-read* ${denyRules})`, + ].join("\n"); +} + +/** + * Masked paths are bound to `/dev/null` so the sandbox cannot read them even + * though bwrap's default is a full read-only filesystem view. + */ +const BWRAP_MASKED_FILES = [ + "/etc/shadow", + "/etc/sudoers", + ".netrc", + ".npmrc", + ".yarnrc", + ".pypirc", +] as const; + +/** Directories need an empty dir, not a file bind: binding a file over a + * directory makes bubblewrap refuse to start. */ +const BWRAP_MASKED_DIRS = [".ssh", ".aws", ".gnupg", ".config/gh", ".config/gcloud", ".kube", ".docker"] as const; + +/** + * Build the bubblewrap argument list. + * + * bwrap cannot express "deny by default" without rebuilding the whole root, so the + * approach is: read-only root, writable project, and sensitive paths masked. + */ +export function buildBwrapArgs(options: SandboxProfileOptions): string[] { + const { cwd } = options; + const args = [ + "--die-with-parent", + "--ro-bind", + "/", + "/", + "--bind", + cwd, + cwd, + "--dev", + "/dev", + "--proc", + "/proc", + "--chdir", + cwd, + ]; + + for (const extra of options.extraWritePaths ?? []) { + args.push("--bind", extra, extra); + } + + for (const masked of BWRAP_MASKED_FILES) { + const path = masked.startsWith("/") ? masked : options.home ? `${options.home}/${masked}` : null; + if (path && existsSync(path)) args.push("--ro-bind", "/dev/null", path); + } + + // Absent dirs are skipped: bwrap fails the whole sandbox if a target is missing. + for (const masked of BWRAP_MASKED_DIRS) { + const path = masked.startsWith("/") ? masked : options.home ? `${options.home}/${masked}` : null; + if (path && existsSync(path)) args.push("--tmpfs", path); + } + + // Private scratch space: without this, TMPDIR points at a read-only host path. + args.push("--tmpfs", "/tmp"); + + return args; +} + +export type SpawnPlan = { + file: string; + args: string[]; + /** Recorded in the audit log so "confined" is provable after the fact. */ + confined: boolean; + backend: SandboxBackend; +}; + +/** + * Decide how to spawn a shell command. + * + * `none` is the honest fallback: the caller still gets a working shell, and the + * approval policy stays fail-closed because `confined` is false. + */ +export function spawnPlan( + backend: SandboxBackend, + command: string, + options: SandboxProfileOptions, +): SpawnPlan { + if (backend === "seatbelt") { + return { + // Absolute path on purpose: `Bun.spawn` resolves a bare name against the env + // it is given, and we hand it an allowlisted env. A resolution miss looks + // exactly like "the sandbox broke" (exit null, empty output). + file: SEATBELT_BIN, + args: ["-p", buildSeatbeltProfile(options), "bash", "-c", command], + confined: true, + backend, + }; + } + + if (backend === "bwrap") { + return { + file: BWRAP_BIN, + args: [...buildBwrapArgs(options), "bash", "-c", command], + confined: true, + backend, + }; + } + + return { file: "bash", args: ["-c", command], confined: false, backend: "none" }; +} + +/** Runtime dirs that must stay readable so `bun`/`node` can start inside the sandbox. */ +export function runtimePaths(execPath: string = process.execPath): string[] { + return uniq([dirname(execPath)]); +} diff --git a/packages/cli/src/lib/tool-timing.test.ts b/packages/cli/src/lib/tool-timing.test.ts new file mode 100644 index 0000000..449078e --- /dev/null +++ b/packages/cli/src/lib/tool-timing.test.ts @@ -0,0 +1,58 @@ +import { beforeEach, describe, expect, test } from "bun:test"; +import { + clearToolTiming, + formatToolElapsed, + markToolStarted, + resetToolTimings, + toolElapsedMs, +} from "./tool-timing"; + +beforeEach(() => resetToolTimings()); + +describe("tool timing registry", () => { + test("reports elapsed time from the mark", () => { + markToolStarted("call-1", 1_000); + expect(toolElapsedMs("call-1", 3_400)).toBe(2_400); + }); + + test("unknown calls report null instead of a fake zero", () => { + expect(toolElapsedMs("never-seen", 5_000)).toBeNull(); + }); + + test("the first mark wins, so a retry does not reset the clock", () => { + markToolStarted("call-1", 1_000); + markToolStarted("call-1", 9_000); + expect(toolElapsedMs("call-1", 3_000)).toBe(2_000); + }); + + test("never reports negative time if the clock jumps backwards", () => { + markToolStarted("call-1", 5_000); + expect(toolElapsedMs("call-1", 1_000)).toBe(0); + }); + + test("clearing removes the entry", () => { + markToolStarted("call-1", 1_000); + clearToolTiming("call-1"); + expect(toolElapsedMs("call-1", 2_000)).toBeNull(); + }); + + test("stays bounded across a long session", () => { + for (let index = 0; index < 700; index += 1) { + markToolStarted(`call-${index}`, index); + } + + // Oldest evicted, newest retained — no unbounded growth. + expect(toolElapsedMs("call-0", 1_000)).toBeNull(); + expect(toolElapsedMs("call-699", 1_000)).not.toBeNull(); + }); +}); + +describe("formatToolElapsed", () => { + test("tenths under a minute", () => { + expect(formatToolElapsed(2_300)).toBe("2.3s"); + }); + + test("minutes past 60s", () => { + expect(formatToolElapsed(125_000)).toBe("2m5s"); + }); +}); diff --git a/packages/cli/src/lib/tool-timing.ts b/packages/cli/src/lib/tool-timing.ts new file mode 100644 index 0000000..75a92fc --- /dev/null +++ b/packages/cli/src/lib/tool-timing.ts @@ -0,0 +1,47 @@ +/** + * Per-tool start times, so a running tool can show how long it has been running. + * + * Deliberately a tiny side-table rather than a field on the message part: adding a + * field would mean mutating SDK-owned message data, and the number is presentation + * only. Entries are keyed by `toolCallId`, which is unique per call. + * + * The map is bounded — a long session with hundreds of calls must not leak. + */ + +const startedAt = new Map(); + +/** Cap on retained entries; oldest are evicted first. */ +const MAX_ENTRIES = 512; + +/** Record when a tool call began. Safe to call twice (first write wins). */ +export function markToolStarted(toolCallId: string, now: number = Date.now()): void { + if (startedAt.has(toolCallId)) return; + + if (startedAt.size >= MAX_ENTRIES) { + const oldest = startedAt.keys().next(); + if (!oldest.done) startedAt.delete(oldest.value); + } + + startedAt.set(toolCallId, now); +} + +/** Milliseconds since the call began, or `null` when it was never marked. */ +export function toolElapsedMs(toolCallId: string, now: number = Date.now()): number | null { + const start = startedAt.get(toolCallId); + return start === undefined ? null : Math.max(0, now - start); +} + +export function clearToolTiming(toolCallId: string): void { + startedAt.delete(toolCallId); +} + +/** Drop all entries — used when a new run starts and by tests. */ +export function resetToolTimings(): void { + startedAt.clear(); +} + +/** Compact label for a running tool, e.g. `2.3s`. */ +export function formatToolElapsed(ms: number): string { + const seconds = Math.max(0, ms) / 1000; + return seconds < 60 ? `${seconds.toFixed(1)}s` : `${Math.floor(seconds / 60)}m${Math.round(seconds % 60)}s`; +} diff --git a/packages/cli/src/screens/session.tsx b/packages/cli/src/screens/session.tsx index ac95321..d453ecc 100644 --- a/packages/cli/src/screens/session.tsx +++ b/packages/cli/src/screens/session.tsx @@ -90,6 +90,10 @@ function SessionChat({ turnInterrupted, subagentRunning, subagentType, + agentProgress, + agentDowngrade, + agentPhase, + agentElapsedMs, submit, abort, interrupt, @@ -234,6 +238,10 @@ function SessionChat({ interruptible={isLoading} subagentRunning={subagentRunning} subagentType={subagentType} + agentProgress={agentProgress} + agentDowngrade={agentDowngrade} + agentPhase={agentPhase} + agentElapsedMs={agentElapsedMs} inputDisabled={isLoading} composerRestoreText={composerRestoreText} composerRestoreToken={composerRestoreToken} diff --git a/packages/cli/src/testing/render.tsx b/packages/cli/src/testing/render.tsx new file mode 100644 index 0000000..d5ad896 --- /dev/null +++ b/packages/cli/src/testing/render.tsx @@ -0,0 +1,73 @@ +/** + * Headless render harness for the TUI. + * + * The CLI had no component-level test infrastructure: every existing test covered + * pure logic, so rendered output (status line, dialogs, transcript rows, width + * behaviour) could only be checked by hand in a real terminal. `@opentui/react` + * ships a test renderer whose frame we can assert on as plain text, which is what + * this module wraps. + * + * What it gives us: + * - `renderText(node)` → the captured character frame, trailing spaces trimmed + * - provider wrapping (theme + prompt config) so components render in isolation + * - `renderLines(node)` → the frame as lines, for row-count / width assertions + * + * It does NOT cover: keyboard input, scrolling, or anything that needs a real + * TTY. Those still need a manual run (`bun run dev`). + */ +import { act } from "react"; +import type { ReactNode } from "react"; +import { testRender } from "@opentui/react/test-utils"; +import { ThemeProvider } from "../providers/theme"; +import { PromptConfigProvider } from "../providers/prompt-config"; + +export type RenderOptions = { + /** Terminal columns. Default 100 — wide enough that nothing wraps by accident. */ + width?: number; + /** Terminal rows. Default 6. */ + height?: number; +}; + +/** + * Render `node` in a headless terminal and return the captured frame. + * + * Lines are right-trimmed and trailing blank rows dropped, so assertions read + * like `expect(frame).toContain("turn 3/24")` instead of comparing padding. + */ +export async function renderText( + node: ReactNode, + { width = 100, height = 6 }: RenderOptions = {}, +): Promise { + const setup = await testRender( + + {node} + , + { width, height }, + ); + + try { + await act(async () => { + await setup.renderOnce(); + }); + return normalizeFrame(setup.captureCharFrame()); + } finally { + await act(async () => { + setup.renderer.destroy(); + }); + } +} + +/** {@link renderText} split into rows — use for "must stay on one row" assertions. */ +export async function renderLines(node: ReactNode, options: RenderOptions = {}): Promise { + const frame = await renderText(node, options); + return frame.length === 0 ? [] : frame.split("\n"); +} + +/** Right-trim every line and drop trailing blank rows. */ +export function normalizeFrame(frame: string): string { + return frame + .split("\n") + .map((line) => line.replace(/\s+$/, "")) + .join("\n") + .replace(/\n+$/, ""); +} diff --git a/packages/database/package.json b/packages/database/package.json index 68f9ecc..62ab26c 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -8,7 +8,8 @@ "./client": "./src/client.ts" }, "scripts": { - "db:generate": "bunx prisma generate" + "db:generate": "prisma generate", + "postinstall": "DATABASE_URL=${DATABASE_URL:-postgresql://localhost:5432/mocode} prisma generate" }, "devDependencies": { "@types/node": "^25.9.3", diff --git a/packages/server/package.json b/packages/server/package.json index 093ec3f..34455c3 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,40 +1,39 @@ { - "name": "@mocode/server", - "version": "0.0.1", - "private": true, - "type": "module", - "exports": { - ".": "./src/index.ts" - }, - "scripts": { - "dev": "bun run --hot src/index.ts", - "build": "bun build src/index.ts --outdir dist --target bun", - "postinstall": "bun run --cwd ../database db:generate", - "test": "bun test", - "test:providers": "bun run scripts/test-providers.ts" - }, - "dependencies": { - "@ai-sdk/anthropic": "^3.0.85", - "@ai-sdk/cerebras": "^2.0.51", - "@ai-sdk/gateway": "^3.0.133", - "@ai-sdk/google": "^3.0.30", - "@ai-sdk/groq": "^3.0.21", - "@ai-sdk/openai": "^3.0.72", - "@ai-sdk/provider-utils": "^4.0.30", - "@clerk/backend": "^3.8.2", - "@hono/zod-validator": "^0.8.0", - "@mocode/database": "workspace:*", - "@mocode/shared": "workspace:*", - "@openrouter/ai-sdk-provider": "^2.9.1", - "@polar-sh/sdk": "^0.48.1", - "@sentry/bun": "^10.58.0", - "@sentry/hono": "^10.58.0", - "ai": "^6.0.207", - "dotenv": "^17.4.2", - "hono": "^4.12.25", - "zod": "^4.4.3" - }, - "devDependencies": { - "@types/bun": "latest" - } -} \ No newline at end of file + "name": "@mocode/server", + "version": "0.0.1", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "dev": "bun run --hot src/index.ts", + "build": "bun build src/index.ts --outdir dist --target bun", + "test": "bun test", + "test:providers": "bun run scripts/test-providers.ts" + }, + "dependencies": { + "@ai-sdk/anthropic": "^3.0.85", + "@ai-sdk/cerebras": "^2.0.51", + "@ai-sdk/gateway": "^3.0.133", + "@ai-sdk/google": "^3.0.30", + "@ai-sdk/groq": "^3.0.21", + "@ai-sdk/openai": "^3.0.72", + "@ai-sdk/provider-utils": "^4.0.30", + "@clerk/backend": "^3.8.2", + "@hono/zod-validator": "^0.8.0", + "@mocode/database": "workspace:*", + "@mocode/shared": "workspace:*", + "@openrouter/ai-sdk-provider": "^2.9.1", + "@polar-sh/sdk": "^0.48.1", + "@sentry/bun": "^10.58.0", + "@sentry/hono": "^10.58.0", + "ai": "^6.0.207", + "dotenv": "^17.4.2", + "hono": "^4.12.25", + "zod": "^4.4.3" + }, + "devDependencies": { + "@types/bun": "latest" + } +} diff --git a/packages/server/scripts/context-compaction-bench.ts b/packages/server/scripts/context-compaction-bench.ts new file mode 100644 index 0000000..a09bf4a --- /dev/null +++ b/packages/server/scripts/context-compaction-bench.ts @@ -0,0 +1,76 @@ +/** + * Context-compaction bench. + * + * Measures what bounds the prompt actually buys on a long session. No API calls — + * the conversation is synthetic but sized like real agent traffic (tool outputs + * dominate, which is the whole reason compaction matters). + * + * Run from the repo root: + * bun run packages/server/scripts/context-compaction-bench.ts + */ +import { + DEFAULT_COMPACTION_POLICY, + compactMessages, + estimateMessagesTokens, +} from "@mocode/shared"; + +type Msg = { id: string; role: string; parts: unknown[] }; + +/** One user turn plus the agent's tool round-trip, sized like real traffic. */ +function turn(index: number): Msg[] { + return [ + { + id: `u${index}`, + role: "user", + parts: [{ type: "text", text: `检查 src/module-${index}.ts 的实现并说明问题。`.repeat(6) }], + }, + { + id: `a${index}`, + role: "assistant", + parts: [ + { type: index % 2 === 0 ? "tool-readFile" : "tool-bash", input: { path: `src/module-${index}.ts` }, output: "x".repeat(10_000) }, + { type: "text", text: `模块 ${index} 的问题是边界条件缺失。`.repeat(12) }, + ], + }, + ]; +} + +function session(turns: number): Msg[] { + return Array.from({ length: turns }, (_, index) => turn(index)).flat(); +} + +const TURNS = 50; +const messages = session(TURNS); + +console.log(`=== ${TURNS} 轮会话(每轮 1 次工具调用 + 10k 字符输出(= readFile 截断上限))===\n`); +console.log("turn".padEnd(8), "无压缩 prompt tokens".padEnd(24), "有压缩"); +console.log("-".repeat(60)); + +for (let turns = 5; turns <= TURNS; turns += 5) { + const slice = messages.slice(0, turns * 2); + const before = estimateMessagesTokens(slice); + const compacted = compactMessages(slice, DEFAULT_COMPACTION_POLICY); + const after = compacted.compacted ? compacted.tokensAfter : before; + const saved = before === 0 ? 0 : Math.round(((before - after) / before) * 100); + + console.log( + String(turns).padEnd(8), + String(before).padEnd(24), + `${after}${compacted.compacted ? ` (-${saved}%)` : ""}`, + ); +} + +const before = estimateMessagesTokens(messages); +const result = compactMessages(messages, DEFAULT_COMPACTION_POLICY); +const reduction = Math.round(((before - result.tokensAfter) / before) * 100); + +console.log("\n=== 结论(完整 50 轮) ==="); +console.log("压缩触发:", result.compacted ? "是" : "否"); +console.log("prompt tokens:", before, "→", result.tokensAfter, `(-${reduction}%)`); +console.log("消息条数:", messages.length, "→", result.messages.length); +console.log("策略:", JSON.stringify(DEFAULT_COMPACTION_POLICY)); + +if (result.digest) { + console.log("\n=== 折叠出的摘要(进上下文的就是这一段) ==="); + console.log(result.digest.slice(0, 400)); +} diff --git a/packages/server/scripts/provider-failover-bench.ts b/packages/server/scripts/provider-failover-bench.ts new file mode 100644 index 0000000..51dfbd1 --- /dev/null +++ b/packages/server/scripts/provider-failover-bench.ts @@ -0,0 +1,216 @@ +/** + * Provider availability + failover bench. + * + * Why this exists: `resolveChatModel` hard-binds one provider per model. When that + * provider is unreachable the whole agent turn dies with a bare `403 Forbidden`, + * which is what actually happened on a mainland-China network. + * + * The fault source is real, not mocked: Cerebras / Groq / Anthropic answer 403 + * behind a country block, and the OpenAI / Google endpoints do not connect at all. + * That makes them a free, reproducible fault injector — every failure below is a + * genuine upstream failure, which is the whole point. + * + * It measures two policies against the same probe results: + * baseline — one model, one provider, no fallback + * failover — if the primary fails, walk FALLBACK_CHAIN + * + * Run it from the repo root so `.env` is picked up: + * bun run packages/server/scripts/provider-failover-bench.ts + * + * Note: this makes real API calls. The reachable providers cost a few tokens each. + */ +import { generateText, tool } from "ai"; +import { z } from "zod"; +import { SUPPORTED_CHAT_MODELS } from "@mocode/shared"; +import { resolveChatModel } from "../src/lib/model"; + +/** Per-model budget. Must exceed normal latency but stay under user patience. */ +const TIMEOUT_MS = 12_000; + +/** + * The failover policy under test. Ordered, cheapest-capable-first. + * Today no such chain exists in the code — this script measures what adding it + * would buy, using the probe results we already paid for (no extra requests). + */ +const FALLBACK_CHAIN = ["deepseek-flash", "deepseek-v4-pro"] as const; + +/** Minimal tool so the probe also proves the provider can emit tool calls. */ +const pingTool = tool({ + description: "List the entries of a directory.", + inputSchema: z.object({ path: z.string() }), + execute: async ({ path }) => ({ path, entries: ["a.ts"] }), +}); + +type Probe = { + modelId: string; + provider: string; + ok: boolean; + ms: number; + detail: string; +}; + +/** Collapse SDK errors into something a table can show. */ +function classify(error: unknown): string { + if (error instanceof Error && (error.name === "TimeoutError" || error.name === "AbortError")) { + return "timeout"; + } + + const status = (error as { statusCode?: number }).statusCode; + if (typeof status === "number") return `HTTP ${status}`; + + const message = error instanceof Error ? error.message : String(error); + if (/is not set/.test(message)) return "no-key"; + return message.replace(/\s+/g, " ").slice(0, 58); +} + +async function probe(modelId: string, provider: string): Promise { + const started = Date.now(); + + try { + const resolved = resolveChatModel(modelId); + const result = await generateText({ + model: resolved.model, + messages: [{ role: "user", content: "List the current directory using the ping tool." }], + tools: { ping: pingTool }, + maxOutputTokens: 64, + abortSignal: AbortSignal.timeout(TIMEOUT_MS), + }); + + // Answering without calling the tool is not a working agent turn — a model + // that chats instead of acting would otherwise score as healthy. + const calledPing = result.toolCalls.some((call) => call.toolName === "ping"); + + return { + modelId, + provider, + ok: calledPing, + ms: Date.now() - started, + detail: calledPing ? "tool_calls=1" : `no ping call (tool_calls=${result.toolCalls.length})`, + }; + } catch (error) { + return { modelId, provider, ok: false, ms: Date.now() - started, detail: classify(error) }; + } +} + +function pct(part: number, total: number): string { + return `${part}/${total} (${Math.round((part / total) * 100)}%)`; +} + +const rows: Probe[] = []; + +for (const model of SUPPORTED_CHAT_MODELS) { + process.stdout.write(` probing ${model.id} … `); + const row = await probe(model.id, model.provider); + rows.push(row); + console.log(row.ok ? `ok ${(row.ms / 1000).toFixed(1)}s` : `FAIL ${row.detail}`); +} + +const byId = new Map(rows.map((row) => [row.modelId, row])); + +console.log("\n=== 逐模型探测(真实请求;失败即为上游真实故障) ==="); +console.log("model".padEnd(30), "provider".padEnd(12), "result"); +for (const row of rows) { + const result = row.ok + ? `ok ${(row.ms / 1000).toFixed(1)}s ${row.detail}` + : `FAIL ${(row.ms / 1000).toFixed(1)}s ${row.detail}`; + console.log(row.modelId.padEnd(30), row.provider.padEnd(12), result); +} + +const baselineOk = rows.filter((row) => row.ok).length; +const failoverOk = rows.filter( + (row) => row.ok || FALLBACK_CHAIN.some((id) => byId.get(id)?.ok), +).length; + +console.log("\n=== 结论 ==="); +console.log("无回退(单 provider 硬绑):", pct(baselineOk, rows.length)); +console.log("带回退(失败后走", FALLBACK_CHAIN.join(" → ") + "):", pct(failoverOk, rows.length)); + +const failures = rows.filter((row) => !row.ok); +const fastFail = failures.filter((row) => row.ms < 5_000); +const slowFail = failures.filter((row) => row.ms >= 5_000); + +console.log("\n=== 失败耗时分布(决定回退策略要不要设超时) ==="); +console.log("快速失败 <5s(如 403):", fastFail.length, "→", fastFail.map((r) => r.detail).join(", ") || "-"); +console.log("慢失败 ≥5s(连接挂起):", slowFail.length, "→", slowFail.map((r) => r.detail).join(", ") || "-"); + +if (slowFail.length > 0) { + console.log( + `\n注意:有 ${slowFail.length} 个模型是"挂到超时"才失败的。若回退链不设 per-attempt 超时,` + + `一次请求会被最慢的那个 provider 拖到 ${TIMEOUT_MS / 1000}s 之后。`, + ); +} + +// --------------------------------------------------------------------------- +// Phase 2 — end-to-end failover, real requests. +// +// Phase 1 above only *simulates* the policy by re-reading probe results. This +// phase actually walks the chain with real calls, so the wall-clock number +// includes the failed attempts a user would wait through. +// +// The policy below mirrors `packages/cli/src/lib/provider-chain.ts` (same +// retryable-status set). It is duplicated here on purpose: the server package may +// not import the CLI package, and a script must not be the thing that defines +// shipped behaviour. +// --------------------------------------------------------------------------- + +const RETRYABLE = new Set([401, 403, 404, 408, 429, 500, 502, 503, 504]); + +/** Three representative primaries: fast-fail, region-block, already-working. */ +const E2E_SAMPLE = ["claude-sonnet-4-6", "gpt-oss-120b", "deepseek-flash"] as const; + +function retryable(error: unknown): boolean { + const status = (error as { statusCode?: number }).statusCode; + if (typeof status === "number") return RETRYABLE.has(status); + const name = error instanceof Error ? error.name : ""; + if (name === "AbortError") return false; // user cancel must never retry + return name === "TimeoutError"; +} + +async function e2eFailover(primary: string) { + const chain = [...new Set([primary, ...FALLBACK_CHAIN])]; + const started = Date.now(); + const failures: string[] = []; + + for (const modelId of chain) { + try { + const resolved = resolveChatModel(modelId); + await generateText({ + model: resolved.model, + messages: [{ role: "user", content: "List the current directory using the ping tool." }], + tools: { ping: pingTool }, + maxOutputTokens: 64, + abortSignal: AbortSignal.timeout(TIMEOUT_MS), + }); + + return { + primary, + answeredBy: modelId, + downgraded: modelId !== primary, + totalMs: Date.now() - started, + failures, + }; + } catch (error) { + failures.push(`${modelId}: ${classify(error)}`); + if (!retryable(error)) break; + } + } + + return { + primary, + answeredBy: null, + downgraded: false, + totalMs: Date.now() - started, + failures, + }; +} + +console.log("\n=== 端到端回退(真实请求,含失败的等待时间) ==="); +for (const primary of E2E_SAMPLE) { + const result = await e2eFailover(primary); + const verdict = result.answeredBy + ? `→ ${result.answeredBy}${result.downgraded ? " (downgraded)" : ""} ${(result.totalMs / 1000).toFixed(1)}s` + : `FAILED ${(result.totalMs / 1000).toFixed(1)}s`; + console.log(primary.padEnd(30), verdict); + for (const failure of result.failures) console.log("".padEnd(30), ` ✗ ${failure}`); +} + diff --git a/packages/server/scripts/test-providers.ts b/packages/server/scripts/test-providers.ts index 3f8ace7..0d2b7d1 100644 --- a/packages/server/scripts/test-providers.ts +++ b/packages/server/scripts/test-providers.ts @@ -48,6 +48,14 @@ const FREE_MODELS = [ modelId: "openai/gpt-oss-120b:free", envKey: "OPENROUTER_API_KEY", }, + { + modelId: "deepseek-flash", + envKey: "DEEPSEEK_API_KEY", + }, + { + modelId: "deepseek-v4-pro", + envKey: "DEEPSEEK_API_KEY", + }, ] as const; const BASIC_PROMPT = "Reply with exactly: OK"; diff --git a/packages/server/src/lib/model.ts b/packages/server/src/lib/model.ts index b76235e..add343f 100644 --- a/packages/server/src/lib/model.ts +++ b/packages/server/src/lib/model.ts @@ -6,7 +6,7 @@ import { anthropic } from "@ai-sdk/anthropic"; import { cerebras } from "@ai-sdk/cerebras"; import { google } from "@ai-sdk/google"; import { groq } from "@ai-sdk/groq"; -import { openai } from "@ai-sdk/openai"; +import { createOpenAI, openai } from "@ai-sdk/openai"; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { @@ -26,6 +26,7 @@ type GoogleModelId = Extract["id"]; type GroqModelId = Extract["id"]; type CerebrasModelId = Extract["id"]; type OpenRouterModelId = Extract["id"]; +type DeepSeekModelId = Extract["id"]; export type ResolvedModel = { model: LanguageModel; @@ -173,6 +174,24 @@ function resolveOpenRouterModel(modelId: OpenRouterModelId): ResolvedModel { }; } +function resolveDeepSeekModel(modelId: DeepSeekModelId): ResolvedModel { + const apiKey = process.env.DEEPSEEK_API_KEY; + if (!apiKey) { + throw new Error("DEEPSEEK_API_KEY is not set"); + } + // DeepSeek only implements Chat Completions; createOpenAI() defaults to Responses API. + const deepseek = createOpenAI({ + apiKey, + baseURL: "https://api.deepseek.com/v1", + name: "deepseek", + }); + return { + model: deepseek.chat(modelId), + provider: "deepseek", + modelId, + }; +} + function resolveSupportedChatModel(model: SupportedChatModel): ResolvedModel { switch (model.provider) { case "anthropic": @@ -187,6 +206,8 @@ function resolveSupportedChatModel(model: SupportedChatModel): ResolvedModel { return resolveCerebrasModel(model.id); case "openrouter": return resolveOpenRouterModel(model.id); + case "deepseek": + return resolveDeepSeekModel(model.id); default: { const _exhaustive: never = model; throw new Error(`Unsupported provider: ${String(_exhaustive)}`); diff --git a/packages/server/src/routes/chat.ts b/packages/server/src/routes/chat.ts index 451deb6..0cd156e 100644 --- a/packages/server/src/routes/chat.ts +++ b/packages/server/src/routes/chat.ts @@ -2,7 +2,10 @@ * Chat HTTP routes: submit a user turn and stream the assistant reply via SSE. * * Phase 8 extends the stream with: - * - Multi-step agent loop via `streamText({ tools, stopWhen: stepCountIs(50) })` + * - Exactly one model step per request: tools carry no executors here, so the + * stream ends after the model asks for tools. The CLI runs them and re-posts; + * the loop budget (turns, tool caps, repeat guard) lives in + * `packages/cli/src/lib/agent-loop.ts`. * - Reasoning, tool-call, and tool-result SSE events (mirrored in Message.parts) * - System prompt and cwd-scoped tools when session.cwd is set * @@ -63,12 +66,15 @@ import { } from "../lib/stream-interrupt"; import { shouldPersistOnFinish } from "../lib/chat-abort"; import { resolveSubagentChatFinish } from "../lib/chat-subagent"; +import { DEFAULT_COMPACTION_POLICY, compactMessages } from "@mocode/shared"; type ChatMessageMetadata = { mode?: ModeType; model?: string; durationMs?: number; usage?: LanguageModelUsage; + /** Present when old turns were folded to stay inside the context budget. */ + compaction?: { tokensBefore: number; tokensAfter: number }; }; type MocodeUIMessage = UIMessage>; @@ -179,7 +185,10 @@ const app = new Hono() messages: stripIncompleteAssistantMessages(mergedMessages), tools, }); - const modelMessages = await convertToModelMessages(nextMessages, { tools }); + // Compaction shapes only the *model input*. The client's message list stays + // intact and is still what gets persisted and echoed back. + const compaction = compactMessages(nextMessages, DEFAULT_COMPACTION_POLICY); + const modelMessages = await convertToModelMessages(compaction.messages, { tools }); let completedUsage: LanguageModelUsage | null = null; const replayBuffer = new StreamReplayBuffer(); @@ -208,6 +217,14 @@ const app = new Hono() model, durationMs: Date.now() - startTime, ...(completedUsage ? { usage: completedUsage } : {}), + ...(compaction.compacted + ? { + compaction: { + tokensBefore: compaction.tokensBefore, + tokensAfter: compaction.tokensAfter, + }, + } + : {}), }; }, consumeSseStream: ({ stream }) => { diff --git a/packages/shared/src/context-compaction.test.ts b/packages/shared/src/context-compaction.test.ts new file mode 100644 index 0000000..cbe2a76 --- /dev/null +++ b/packages/shared/src/context-compaction.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, test } from "bun:test"; +import { + DEFAULT_COMPACTION_POLICY, + compactMessages, + estimateMessageTokens, + estimateMessagesTokens, + estimateTextTokens, +} from "./context-compaction"; + +type Msg = { id: string; role: string; parts: unknown[] }; + +function user(id: string, text: string): Msg { + return { id, role: "user", parts: [{ type: "text", text }] }; +} + +function assistantText(id: string, text: string): Msg { + return { id, role: "assistant", parts: [{ type: "text", text }] }; +} + +function assistantTool(id: string, name: string, input: unknown, output: string): Msg { + return { + id, + role: "assistant", + parts: [ + { type: `tool-${name}`, input, output }, + { type: "text", text: `called ${name}` }, + ], + }; +} + +describe("estimateTextTokens", () => { + test("empty text is free", () => { + expect(estimateTextTokens("")).toBe(0); + }); + + test("ascii is roughly 4 chars per token", () => { + expect(estimateTextTokens("abcd")).toBe(1); + expect(estimateTextTokens("a".repeat(400))).toBe(100); + }); + + test("cjk costs about one token per character", () => { + expect(estimateTextTokens("修仙欠费中")).toBe(5); + }); + + test("mixed content adds both", () => { + expect(estimateTextTokens("abcd修仙")).toBe(3); + }); +}); + +describe("estimateMessageTokens", () => { + test("counts text parts", () => { + expect(estimateMessageTokens(user("u", "hello world"))).toBeGreaterThan(0); + }); + + test("counts tool input and output payloads", () => { + const small = estimateMessageTokens(assistantTool("a", "readFile", { path: "a.ts" }, "x")); + const large = estimateMessageTokens( + assistantTool("a", "readFile", { path: "a.ts" }, "x".repeat(4_000)), + ); + + expect(large).toBeGreaterThan(small); + }); + + test("a message with no parts costs nothing", () => { + expect(estimateMessageTokens({ id: "x", role: "user" })).toBe(0); + }); + + test("conversation total is the sum", () => { + const messages = [user("u", "abcd"), assistantText("a", "abcd")]; + expect(estimateMessagesTokens(messages)).toBe( + estimateMessageTokens(messages[0]!) + estimateMessageTokens(messages[1]!), + ); + }); +}); + +/** 20 turns of realistic, token-heavy history. */ +function longConversation(): Msg[] { + const messages: Msg[] = []; + + for (let turn = 0; turn < 20; turn += 1) { + messages.push(user(`u${turn}`, `请检查第 ${turn} 个文件并说明问题`.padEnd(200, "。"))); + messages.push( + assistantTool( + `a${turn}`, + turn % 2 === 0 ? "bash" : "readFile", + { path: `src/module-${turn}.ts`, command: "ls" }, + "x".repeat(2_000), + ), + ); + } + + return messages; +} + +describe("compactMessages", () => { + test("leaves a small conversation alone", () => { + const messages = [user("u", "hi"), assistantText("a", "hello")]; + const result = compactMessages(messages); + + expect(result.compacted).toBe(false); + expect(result.messages).toBe(messages); + expect(result.digest).toBeNull(); + }); + + test("does not compact when the message count is still low", () => { + const messages = [user("u", "hi"), assistantText("a", "x".repeat(1_000_000))]; + const result = compactMessages(messages, { ...DEFAULT_COMPACTION_POLICY, maxTokens: 10 }); + + expect(result.compacted).toBe(false); + }); + + test("folds old turns once the budget is exceeded", () => { + const messages = longConversation(); + const result = compactMessages(messages, { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 6, + minMessages: 8, + }); + + expect(result.compacted).toBe(true); + expect(result.tokensAfter).toBeLessThan(result.tokensBefore); + expect(result.messages.length).toBeLessThan(messages.length); + }); + + test("keeps the most recent messages verbatim", () => { + const messages = longConversation(); + const result = compactMessages(messages, { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 4, + minMessages: 8, + }); + + const kept = result.messages.slice(1); + expect(kept).toEqual(messages.slice(messages.length - kept.length)); + }); + + test("never splits a user turn from what followed it", () => { + const messages = longConversation(); + const result = compactMessages(messages, { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 7, + minMessages: 8, + }); + + // Index 0 is the digest; everything after it must start on a user message. + expect(result.messages[1]?.role).toBe("user"); + }); + + test("never compacts when there is no user boundary to cut at", () => { + // One opening user turn, then a long assistant-only tail: any cut would land + // inside the active turn and drop the request that produced it. + const messages: Msg[] = [user("u0", "开始")]; + for (let index = 1; index < 20; index += 1) { + messages.push(assistantText(`a${index}`, "x".repeat(2_000))); + } + + const result = compactMessages(messages, { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 100, + keepRecent: 2, + minMessages: 4, + }); + + expect(result.compacted).toBe(false); + }); + + test("digest records the tools and files the agent already touched", () => { + const result = compactMessages(longConversation(), { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 4, + minMessages: 8, + }); + + expect(result.digest).toContain("[compacted]"); + expect(result.digest).toContain("Tools already used:"); + expect(result.digest).toMatch(/(bash|readFile)×\d+/); + expect(result.digest).toContain("Files touched:"); + expect(result.digest).toContain("src/module-0.ts"); + }); + + test("digest tells the model the content is gone, not that it was summarized", () => { + const result = compactMessages(longConversation(), { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 4, + minMessages: 8, + }); + + expect(result.digest).toContain("re-read a file"); + }); + + test("compaction is bounded — running twice does not compound", () => { + const first = compactMessages(longConversation(), { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 4, + minMessages: 8, + }); + + const second = compactMessages(first.messages, { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 4, + minMessages: 8, + }); + + expect(second.tokensAfter).toBeLessThanOrEqual(first.tokensAfter); + }); +}); diff --git a/packages/shared/src/context-compaction.ts b/packages/shared/src/context-compaction.ts new file mode 100644 index 0000000..65318fd --- /dev/null +++ b/packages/shared/src/context-compaction.ts @@ -0,0 +1,229 @@ +/** + * Context budget + compaction. + * + * Problem: the server replays the whole stored session on every turn, so the prompt + * grows without bound. A long session eventually either overflows the model window + * or costs a fortune in input tokens for history nobody re-reads. + * + * Approach: when the estimated prompt exceeds a budget, keep the most recent turns + * verbatim and replace everything older with a compact deterministic digest. The + * digest is not an LLM summary on purpose — it adds no latency, no cost, and no + * extra failure mode, and what the agent actually needs from old turns is *what it + * already did* (tools used, files touched), not the prose around it. + * + * Pure module: no I/O, no model calls. + */ + +/** Rough token estimate. ASCII ≈ 4 chars/token; CJK ≈ 1 char/token. */ +export function estimateTextTokens(text: string): number { + if (text.length === 0) return 0; + + let narrow = 0; + let wide = 0; + + for (const char of text) { + const code = char.codePointAt(0) ?? 0; + // CJK, kana, hangul and full-width forms are ~1 token per character. + if (code > 0x2e80) wide += 1; + else narrow += 1; + } + + return Math.ceil(narrow / 4) + wide; +} + +type PartLike = { + type: string; + text?: unknown; + toolName?: unknown; + input?: unknown; + output?: unknown; +}; + +type MessageLike = { + id: string; + role: string; + parts?: unknown; +}; + +function partsOf(message: MessageLike): PartLike[] { + return Array.isArray(message.parts) ? (message.parts as PartLike[]) : []; +} + +/** Token estimate for one message, counting text and tool payloads. */ +export function estimateMessageTokens(message: MessageLike): number { + let total = 0; + + for (const part of partsOf(message)) { + if (typeof part.text === "string") total += estimateTextTokens(part.text); + if (part.input !== undefined) total += estimateTextTokens(safeJson(part.input)); + if (part.output !== undefined) total += estimateTextTokens(safeJson(part.output)); + // Fixed overhead per part keeps a long tool-call chain from looking free. + total += 4; + } + + return total; +} + +/** Token estimate for a whole conversation. */ +export function estimateMessagesTokens(messages: MessageLike[]): number { + return messages.reduce((sum, message) => sum + estimateMessageTokens(message), 0); +} + +function safeJson(value: unknown): string { + try { + return JSON.stringify(value) ?? ""; + } catch { + return ""; + } +} + +/** Tool name for a part, handling both static `tool-` and dynamic MCP calls. */ +function toolNameOf(part: PartLike): string | null { + if (part.type === "dynamic-tool" && typeof part.toolName === "string") return part.toolName; + if (part.type.startsWith("tool-")) return part.type.slice("tool-".length); + return null; +} + +/** File-path-ish strings inside a tool input, for the digest. */ +function pathsIn(input: unknown, into: Set): void { + if (input === null || typeof input !== "object") return; + + for (const [key, value] of Object.entries(input as Record)) { + if (typeof value === "string") { + if (/path|file|dir/i.test(key) && !value.includes("\n") && value.length <= 120) { + into.add(value); + } + } else if (value !== null && typeof value === "object") { + pathsIn(value, into); + } + } +} + +export type CompactionPolicy = { + /** Estimated prompt tokens allowed before compaction runs. */ + maxTokens: number; + /** Most recent messages kept verbatim. */ + keepRecent: number; + /** Below this many messages compaction is pointless — skip it. */ + minMessages: number; +}; + +export const DEFAULT_COMPACTION_POLICY: CompactionPolicy = { + maxTokens: 60_000, + keepRecent: 10, + minMessages: 16, +}; + +export type CompactionResult = { + messages: MESSAGE[]; + compacted: boolean; + tokensBefore: number; + tokensAfter: number; + /** Human-readable record of what was folded away; null when nothing happened. */ + digest: string | null; +}; + +/** + * Fold old turns into one digest message when the prompt exceeds `maxTokens`. + * + * The cut is placed on a user-message boundary so a tool call and its result are + * never separated — splitting those would produce an invalid conversation. + */ +export function compactMessages( + messages: MESSAGE[], + policy: CompactionPolicy = DEFAULT_COMPACTION_POLICY, +): CompactionResult { + const tokensBefore = estimateMessagesTokens(messages); + + const noop: CompactionResult = { + messages, + compacted: false, + tokensBefore, + tokensAfter: tokensBefore, + digest: null, + }; + + if (tokensBefore <= policy.maxTokens) return noop; + if (messages.length < policy.minMessages) return noop; + + const cut = findCutIndex(messages, policy.keepRecent); + if (cut <= 0) return noop; + + const dropped = messages.slice(0, cut); + const kept = messages.slice(cut); + const digest = buildDigest(dropped); + + const digestMessage = { + id: `compaction-${dropped.length}`, + role: "assistant", + parts: [{ type: "text", text: digest }], + } as unknown as MESSAGE; + + const compacted = [digestMessage, ...kept]; + const tokensAfter = estimateMessagesTokens(compacted); + + return { + messages: compacted, + compacted: true, + tokensBefore, + tokensAfter, + digest, + }; +} + +/** + * Index to cut at: far enough back to keep `keepRecent` messages, snapped forward + * to the next user message so no assistant/tool group is split. + */ +function findCutIndex(messages: MessageLike[], keepRecent: number): number { + const target = Math.max(1, messages.length - keepRecent); + + // Prefer a boundary at or after the target: keeps the most recent turns intact. + for (let index = target; index < messages.length; index += 1) { + if (messages[index]?.role === "user") return index; + } + + // No user boundary ahead — fall back to the closest one *before* it. Returning + // `target` would cut inside a turn and could drop the user's active request. + for (let index = Math.min(target, messages.length - 1); index >= 1; index -= 1) { + if (messages[index]?.role === "user") return index; + } + + // Nothing safe to remove; compaction is skipped rather than guessed. + return 0; +} + +/** Deterministic record of what the dropped turns did. */ +function buildDigest(dropped: MessageLike[]): string { + const toolCounts = new Map(); + const paths = new Set(); + let userTurns = 0; + + for (const message of dropped) { + if (message.role === "user") userTurns += 1; + + for (const part of partsOf(message)) { + const toolName = toolNameOf(part); + if (toolName) { + toolCounts.set(toolName, (toolCounts.get(toolName) ?? 0) + 1); + pathsIn(part.input, paths); + } + } + } + + const tools = [...toolCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([name, count]) => `${name}×${count}`) + .join(", "); + + const fileList = [...paths].slice(0, 12).join(", "); + + const lines = [ + `[compacted] ${dropped.length} earlier messages folded away (${userTurns} user turns).`, + tools ? `Tools already used: ${tools}.` : null, + fileList ? `Files touched: ${fileList}.` : null, + "Those turns are no longer in context; re-read a file if you need its exact contents.", + ].filter(Boolean); + + return lines.join(" "); +} diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 42c0a4d..e75a41b 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -28,6 +28,17 @@ export { type SerializedMcpTool, } from "./mcp-tools"; +export { + /** Context budget: token estimation + fold-based compaction (server and BYOK). */ + DEFAULT_COMPACTION_POLICY, + compactMessages, + estimateMessageTokens, + estimateMessagesTokens, + estimateTextTokens, + type CompactionPolicy, + type CompactionResult, +} from "./context-compaction"; + export { hasVisibleAssistantContent, type AssistantMessageLike, diff --git a/packages/shared/src/models.test.ts b/packages/shared/src/models.test.ts index b3530ae..e6c6667 100644 --- a/packages/shared/src/models.test.ts +++ b/packages/shared/src/models.test.ts @@ -8,8 +8,8 @@ import { } from "./models"; describe("model catalog hints", () => { - test("default model is Cerebras gpt-oss-120b", () => { - expect(DEFAULT_CHAT_MODEL_ID).toBe("gpt-oss-120b"); + test("default model is DeepSeek Flash", () => { + expect(DEFAULT_CHAT_MODEL_ID).toBe("deepseek-flash"); }); test("marks free-tier tool-calling guidance", () => { diff --git a/packages/shared/src/models.ts b/packages/shared/src/models.ts index 7332f3a..785caea 100644 --- a/packages/shared/src/models.ts +++ b/packages/shared/src/models.ts @@ -15,7 +15,8 @@ export type ModelPricing = { | "google" | "groq" | "cerebras" - | "openrouter"; + | "openrouter" + | "deepseek"; type SupportedChatModelDefinition = { id: string; @@ -130,6 +131,27 @@ export type ModelPricing = { outputUsdPerMillionTokens: 0.1, }, }, + { + // DeepSeek V4.1 Flash (1M context, tool calls + vision). Pricing is peak + // cache-miss input + peak output so credit estimates over- rather than + // under-charge; cache hits and off-peak hours are half or less. + // Source: api-docs.deepseek.com/quick_start/pricing + id: "deepseek-flash", + provider: "deepseek", + pricing: { + inputUsdPerMillionTokens: 0.3, + outputUsdPerMillionTokens: 1.2, + }, + }, + { + // DeepSeek V4 Pro — same pricing basis as deepseek-flash above. + id: "deepseek-v4-pro", + provider: "deepseek", + pricing: { + inputUsdPerMillionTokens: 1.32, + outputUsdPerMillionTokens: 3.96, + }, + }, ] as const satisfies readonly SupportedChatModelDefinition[]; export type SupportedChatModel = (typeof SUPPORTED_CHAT_MODELS)[number]; @@ -147,8 +169,12 @@ export type ModelPricing = { { message: "Unsupported chat model" }, ); - /** Default model when the session UI does not expose an explicit picker yet. */ - export const DEFAULT_CHAT_MODEL_ID: SupportedChatModelId = "gpt-oss-120b"; + /** + * Default model. DeepSeek is reachable from mainland China without a proxy, + * unlike the Cerebras / Groq / Anthropic endpoints (which answer 403 "Forbidden" + * behind a country block). Switch with `/models`. + */ + export const DEFAULT_CHAT_MODEL_ID: SupportedChatModelId = "deepseek-flash"; /** BYOK catalog hints for the `/models` picker (free-tier tool-calling guidance). */ export type ModelCatalogHint = "free-recommended" | "free-ok" | "weak-tools";