From e68f89d45182fb4c41236cb789f00f08eb377e11 Mon Sep 17 00:00:00 2001 From: moyun Date: Thu, 12 Nov 2026 09:35:20 +0800 Subject: [PATCH 01/11] feat(models): add DeepSeek provider and make flash the default The existing catalog pointed at providers that answer 403 behind a country block, so the out-of-the-box experience was "Forbidden" on every message. - add a `deepseek` provider and the two ids the API actually serves (`deepseek-flash`, `deepseek-v4-pro`); the old `deepseek-chat` / `deepseek-reasoner` ids no longer exist - price them from the official table, using peak cache-miss input + peak output so credit estimates over- rather than under-charge - default to `deepseek-flash`: reachable without a proxy, 1M context, and it emits tool calls correctly - smoke-test both ids through `test-providers.ts` --- .env.example | 3 ++ README.md | 12 ++++--- README.zh-CN.md | 12 ++++--- bun.lock | 4 +-- .../components/dialogs/keys-wizard-dialog.tsx | 1 + packages/cli/src/lib/local-model.ts | 17 ++++++++++ packages/server/scripts/test-providers.ts | 8 +++++ packages/server/src/lib/model.ts | 23 ++++++++++++- packages/shared/src/models.test.ts | 4 +-- packages/shared/src/models.ts | 32 +++++++++++++++++-- 10 files changed, 100 insertions(+), 16 deletions(-) diff --git a/.env.example b/.env.example index 036d8f4..c233a7b 100644 --- a/.env.example +++ b/.env.example @@ -26,6 +26,9 @@ CEREBRAS_API_KEY= # OpenRouter — https://openrouter.ai/keys OPENROUTER_API_KEY= +# DeepSeek — https://platform.deepseek.com/api_keys +DEEPSEEK_API_KEY= + # Clerk OAuth (phase 9) — https://clerk.com/keys # Server: CLERK_SECRET_KEY + CLERK_PUBLISHABLE_KEY validate Bearer tokens on /sessions and /chat. # CLI: CLERK_FRONTEND_API + CLERK_OAUTH_CLIENT_ID drive the browser /login PKCE flow. diff --git a/README.md b/README.md index 1d549c8..396915c 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ MoCode is a Bun monorepo that pairs a rich **TUI client** with a **Hono API serv |---|---| | **Dual agent modes** | **Plan** — read-only exploration and architecture. **Build** — full read/write/bash toolset for implementation. | | **Local tool execution** | The server defines tools; the CLI runs them against `process.cwd()` with path sandboxing. | -| **Multi-provider models** | Anthropic, OpenAI, Google, Groq, Cerebras, and OpenRouter — switch models mid-session with `/models`. | +| **Multi-provider models** | Anthropic, OpenAI, Google, Groq, Cerebras, OpenRouter, and DeepSeek — switch models mid-session with `/models`. | | **Session persistence & recovery** | PostgreSQL (SaaS) or local JSON (`--local`). Esc keeps partial text; tool Esc marks `Interrupted by user` and stops immediately; auto-resume on user-only tail; partial assistant uses `/resume` (regenerate). See [Session recovery](#session-recovery). | | **Rich TUI** | Built with [OpenTUI](https://github.com/opentui/opentui) + React 19 — themes, dialogs, keyboard layers, and slash commands. | | **OAuth sign-in** | Browser-based PKCE flow via Clerk; tokens stored at `~/.mocode/auth.json`. | @@ -148,6 +148,7 @@ mocode | `GROQ_API_KEY` | Server | Groq API key | | `CEREBRAS_API_KEY` | Server | Cerebras API key | | `OPENROUTER_API_KEY` | Server | OpenRouter API key | +| `DEEPSEEK_API_KEY` | Server | DeepSeek API key | | `CLERK_SECRET_KEY` | Server | Clerk secret key (auth) | | `CLERK_PUBLISHABLE_KEY` | Server | Clerk publishable key | | `CLERK_FRONTEND_API` | CLI | Clerk frontend API domain | @@ -232,16 +233,19 @@ Implementation details: [`doc/harness-phase-03-stream-reliability-notes.md`](./d | `llama-3.3-70b-versatile` | Groq (free; weak tool calling) | | `openai/gpt-oss-20b` | Groq (free) | | `openai/gpt-oss-120b` | Groq (free) | -| `gpt-oss-120b` | Cerebras (default for BYOK; free recommended) | +| `gpt-oss-120b` | Cerebras (free recommended) | | `openai/gpt-oss-120b:free` | OpenRouter | +| `deepseek-flash` | DeepSeek (**default**; 1M context; reachable from mainland China) | +| `deepseek-v4-pro` | DeepSeek (stronger, ~4x the price) | The canonical list lives in `packages/shared/src/models.ts`. ### BYOK (`mocode --local`) -- **Default model:** `gpt-oss-120b` (Cerebras). Set provider keys with **`/keys`** → `~/.mocode/keys.json` (local only; never sent to the MoCode server). +- **Default model:** `deepseek-flash` (DeepSeek). Set provider keys with **`/keys`** → `~/.mocode/keys.json` (local only; never sent to the MoCode server). +- **Why not Cerebras/Groq:** those endpoints answer `403 Forbidden` from networks where the provider is region-blocked. DeepSeek is reachable without a proxy; use `/models` to switch. - **`/models` hints:** *Free recommended* / *Free OK* / *Weak tool calling* — avoid weak-tool models for `/explore` and other multi-tool turns. -- **Free-tier picks:** Cerebras `gpt-oss-120b` or Groq `openai/gpt-oss-120b`. Groq Llama ids often malform tool calls in subagent loops. +- **Free-tier picks:** Cerebras `gpt-oss-120b` or Groq `openai/gpt-oss-120b` (only where those APIs are reachable). Groq Llama ids often malform tool calls in subagent loops. Smoke-test server `.env` keys: `bun run packages/server/scripts/test-providers.ts` diff --git a/README.zh-CN.md b/README.zh-CN.md index ed3dde9..72fdc7e 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -18,7 +18,7 @@ MoCode 是一个基于 Bun 的 monorepo,由功能丰富的 **TUI 客户端** |---|---| | **双 Agent 模式** | **Plan** — 只读探索与架构分析。**Build** — 完整读写/bash 工具集,用于实现代码。 | | **本地工具执行** | 服务端定义工具契约,CLI 在 `process.cwd()` 下执行,并做路径沙箱隔离。 | -| **多模型支持** | Anthropic、OpenAI、Google、Groq、Cerebras、OpenRouter — 会话中随时用 `/models` 切换。 | +| **多模型支持** | Anthropic、OpenAI、Google、Groq、Cerebras、OpenRouter、DeepSeek — 会话中随时用 `/models` 切换。 | | **会话持久化与恢复** | PostgreSQL(SaaS)或本地 JSON(`--local`)。Esc 保留 partial 正文;工具执行中 Esc 标记 `Interrupted by user` 并立即停止;末条 user 重开自动续写;partial 助手需 `/resume`(regenerate)。详见 [会话恢复](#会话恢复)。 | | **富 TUI 界面** | 基于 [OpenTUI](https://github.com/opentui/opentui) + React 19 — 主题、对话框、键盘层与斜杠命令。 | | **OAuth 登录** | 通过 Clerk 的浏览器 PKCE 流程;Token 保存在 `~/.mocode/auth.json`。 | @@ -148,6 +148,7 @@ mocode | `GROQ_API_KEY` | 服务端 | Groq API Key | | `CEREBRAS_API_KEY` | 服务端 | Cerebras API Key | | `OPENROUTER_API_KEY` | 服务端 | OpenRouter API Key | +| `DEEPSEEK_API_KEY` | 服务端 | DeepSeek API Key | | `CLERK_SECRET_KEY` | 服务端 | Clerk Secret Key(认证) | | `CLERK_PUBLISHABLE_KEY` | 服务端 | Clerk Publishable Key | | `CLERK_FRONTEND_API` | CLI | Clerk Frontend API 域名 | @@ -232,16 +233,19 @@ macOS Terminal.app 可运行 `mocode --terminal-setup` 一次,启用 Option-as | `llama-3.3-70b-versatile` | Groq(免费;工具调用不稳定) | | `openai/gpt-oss-20b` | Groq(免费) | | `openai/gpt-oss-120b` | Groq(免费) | -| `gpt-oss-120b` | Cerebras(BYOK 默认;免费推荐) | +| `gpt-oss-120b` | Cerebras(免费推荐) | | `openai/gpt-oss-120b:free` | OpenRouter | +| `deepseek-flash` | DeepSeek(**默认**;1M 上下文;国内可直连) | +| `deepseek-v4-pro` | DeepSeek(更强,价格约为 4 倍) | 权威列表见 `packages/shared/src/models.ts`。 ### BYOK(`mocode --local`) -- **默认模型:** `gpt-oss-120b`(Cerebras)。用 **`/keys`** 写入 `~/.mocode/keys.json`(仅存本机,不上传服务端)。 +- **默认模型:** `deepseek-flash`(DeepSeek)。用 **`/keys`** 写入 `~/.mocode/keys.json`(仅存本机,不上传服务端)。 +- **为什么不用 Cerebras/Groq:** 在厂商按地区封锁的网络里,这两个端点会返回 `403 Forbidden`。DeepSeek 可直连;用 `/models` 切换。 - **`/models` 标注:** *免费推荐* / *免费可用* / *工具调用不稳定* — `/explore` 等多工具场景请避开不稳定项。 -- **免费档推荐:** Cerebras `gpt-oss-120b` 或 Groq `openai/gpt-oss-120b`。Groq Llama 系列在子代理多步 tool loop 中易格式错误。 +- **免费档推荐:** Cerebras `gpt-oss-120b` 或 Groq `openai/gpt-oss-120b`(仅在这两家可达的网络下)。Groq Llama 系列在子代理多步 tool loop 中易格式错误。 冒烟测试服务端 `.env` Key:`bun run packages/server/scripts/test-providers.ts` diff --git a/bun.lock b/bun.lock index 02caa27..7121343 100644 --- a/bun.lock +++ b/bun.lock @@ -333,7 +333,7 @@ "@tanstack/query-core": ["@tanstack/query-core@5.101.0", "https://registry.npmmirror.com/@tanstack/query-core/-/query-core-5.101.0.tgz", {}, "sha512-cQetA74EB+seWySv1TTKr828TnP0u39m6LykwDXIo84SNortpDkp30TMEjkqtYCNP9c40uT/iwl6MLiufEt0Ow=="], - "@types/bun": ["@types/bun@1.3.14", "https://registry.npmmirror.com/@types/bun/-/bun-1.3.14.tgz", { "dependencies": { "bun-types": "1.3.14" } }, "sha512-h1hFqFVcvAvD9j9K7ZW7vd82aSA+rTdznZa+5bwvCwqSB1jmmfLcbIWhOLx1/+boy/xmjgCs/OMUL8hRJSmnPw=="], + "@types/bun": ["@types/bun@1.4.4", "https://registry.npmmirror.com/@types/bun/-/bun-1.4.4.tgz", { "dependencies": { "bun-types": "1.4.4" } }, "sha512-9wkRDXUXK124pEnAutE6n/l18j+i9GC5IAHK8TmX/2nlySETmky0OU6yL5U8pW1AoKh++AN9auDCr647BfurjQ=="], "@types/node": ["@types/node@25.9.3", "https://registry.npmmirror.com/@types/node/-/node-25.9.3.tgz", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-603BddQMv3pUcr4U2dhujk83N2tTDVr/34wII2B6bJy6g+8WD6yUb11jszNs0gdi4PesVWl7ABt8nYMVpnLUcg=="], @@ -393,7 +393,7 @@ "bun-ffi-structs": ["bun-ffi-structs@0.2.3", "https://registry.npmmirror.com/bun-ffi-structs/-/bun-ffi-structs-0.2.3.tgz", { "peerDependencies": { "typescript": "^5" } }, "sha512-pgJiXP+hEgFo9qG51J6ItfY4ocs3vniwNzJ9WhoakB3QB2GdzQxX2EXssentPYlB2hOfJrTjO6iIQkWYzUodpg=="], - "bun-types": ["bun-types@1.3.14", "https://registry.npmmirror.com/bun-types/-/bun-types-1.3.14.tgz", { "dependencies": { "@types/node": "*" } }, "sha512-4N0ig0fEomHt5R0KCFWjovxow98rIoRwKolrYdCcknNwMekCXRnWEUvgu5soYV8QXtVsrUD8B95MBOZGPvr6KQ=="], + "bun-types": ["bun-types@1.4.4", "https://registry.npmmirror.com/bun-types/-/bun-types-1.4.4.tgz", { "dependencies": { "@types/node": "*", "undici-types": "*" } }, "sha512-yjrJh5oGX7mMPKGd19YXyIjmEuj4G4zNC30bs7PM9Az7NQ3ndF3j6hbeRdvJTRQEhJ/FYQwW0a2moweEi2hMxQ=="], "bundle-name": ["bundle-name@4.1.0", "https://registry.npmmirror.com/bundle-name/-/bundle-name-4.1.0.tgz", { "dependencies": { "run-applescript": "^7.0.0" } }, "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q=="], diff --git a/packages/cli/src/components/dialogs/keys-wizard-dialog.tsx b/packages/cli/src/components/dialogs/keys-wizard-dialog.tsx index c173aaa..00c7562 100644 --- a/packages/cli/src/components/dialogs/keys-wizard-dialog.tsx +++ b/packages/cli/src/components/dialogs/keys-wizard-dialog.tsx @@ -21,6 +21,7 @@ const PROVIDERS = [ "groq", "cerebras", "openrouter", + "deepseek", ] as const; type ProviderId = (typeof PROVIDERS)[number]; diff --git a/packages/cli/src/lib/local-model.ts b/packages/cli/src/lib/local-model.ts index 20b8f80..6d246ee 100644 --- a/packages/cli/src/lib/local-model.ts +++ b/packages/cli/src/lib/local-model.ts @@ -33,6 +33,7 @@ type GoogleModelId = Extract["id"]; type GroqModelId = Extract["id"]; type CerebrasModelId = Extract["id"]; type OpenRouterModelId = Extract["id"]; +type DeepSeekModelId = Extract["id"]; export type ResolvedModel = { model: LanguageModel; @@ -200,6 +201,20 @@ function resolveOpenRouterModel(modelId: OpenRouterModelId, apiKey: string): Res }; } +function resolveDeepSeekModel(modelId: DeepSeekModelId, apiKey: string): ResolvedModel { + // DeepSeek only implements Chat Completions; createOpenAI() defaults to Responses API. + const deepseek = createOpenAI({ + apiKey, + baseURL: "https://api.deepseek.com/v1", + name: "deepseek", + }); + return { + model: deepseek.chat(modelId), + provider: "deepseek", + modelId, + }; +} + function resolveSupportedChatModel( model: SupportedChatModel, keys: ProviderKeys | null, @@ -223,6 +238,8 @@ function resolveSupportedChatModel( model.id, requireProviderApiKey("openrouter", keys), ); + case "deepseek": + return resolveDeepSeekModel(model.id, requireProviderApiKey("deepseek", keys)); default: { const _exhaustive: never = model; throw new Error(`Unsupported provider: ${String(_exhaustive)}`); diff --git a/packages/server/scripts/test-providers.ts b/packages/server/scripts/test-providers.ts index 3f8ace7..0d2b7d1 100644 --- a/packages/server/scripts/test-providers.ts +++ b/packages/server/scripts/test-providers.ts @@ -48,6 +48,14 @@ const FREE_MODELS = [ modelId: "openai/gpt-oss-120b:free", envKey: "OPENROUTER_API_KEY", }, + { + modelId: "deepseek-flash", + envKey: "DEEPSEEK_API_KEY", + }, + { + modelId: "deepseek-v4-pro", + envKey: "DEEPSEEK_API_KEY", + }, ] as const; const BASIC_PROMPT = "Reply with exactly: OK"; diff --git a/packages/server/src/lib/model.ts b/packages/server/src/lib/model.ts index b76235e..add343f 100644 --- a/packages/server/src/lib/model.ts +++ b/packages/server/src/lib/model.ts @@ -6,7 +6,7 @@ import { anthropic } from "@ai-sdk/anthropic"; import { cerebras } from "@ai-sdk/cerebras"; import { google } from "@ai-sdk/google"; import { groq } from "@ai-sdk/groq"; -import { openai } from "@ai-sdk/openai"; +import { createOpenAI, openai } from "@ai-sdk/openai"; import { createOpenRouter } from "@openrouter/ai-sdk-provider"; import { @@ -26,6 +26,7 @@ type GoogleModelId = Extract["id"]; type GroqModelId = Extract["id"]; type CerebrasModelId = Extract["id"]; type OpenRouterModelId = Extract["id"]; +type DeepSeekModelId = Extract["id"]; export type ResolvedModel = { model: LanguageModel; @@ -173,6 +174,24 @@ function resolveOpenRouterModel(modelId: OpenRouterModelId): ResolvedModel { }; } +function resolveDeepSeekModel(modelId: DeepSeekModelId): ResolvedModel { + const apiKey = process.env.DEEPSEEK_API_KEY; + if (!apiKey) { + throw new Error("DEEPSEEK_API_KEY is not set"); + } + // DeepSeek only implements Chat Completions; createOpenAI() defaults to Responses API. + const deepseek = createOpenAI({ + apiKey, + baseURL: "https://api.deepseek.com/v1", + name: "deepseek", + }); + return { + model: deepseek.chat(modelId), + provider: "deepseek", + modelId, + }; +} + function resolveSupportedChatModel(model: SupportedChatModel): ResolvedModel { switch (model.provider) { case "anthropic": @@ -187,6 +206,8 @@ function resolveSupportedChatModel(model: SupportedChatModel): ResolvedModel { return resolveCerebrasModel(model.id); case "openrouter": return resolveOpenRouterModel(model.id); + case "deepseek": + return resolveDeepSeekModel(model.id); default: { const _exhaustive: never = model; throw new Error(`Unsupported provider: ${String(_exhaustive)}`); diff --git a/packages/shared/src/models.test.ts b/packages/shared/src/models.test.ts index b3530ae..e6c6667 100644 --- a/packages/shared/src/models.test.ts +++ b/packages/shared/src/models.test.ts @@ -8,8 +8,8 @@ import { } from "./models"; describe("model catalog hints", () => { - test("default model is Cerebras gpt-oss-120b", () => { - expect(DEFAULT_CHAT_MODEL_ID).toBe("gpt-oss-120b"); + test("default model is DeepSeek Flash", () => { + expect(DEFAULT_CHAT_MODEL_ID).toBe("deepseek-flash"); }); test("marks free-tier tool-calling guidance", () => { diff --git a/packages/shared/src/models.ts b/packages/shared/src/models.ts index 7332f3a..785caea 100644 --- a/packages/shared/src/models.ts +++ b/packages/shared/src/models.ts @@ -15,7 +15,8 @@ export type ModelPricing = { | "google" | "groq" | "cerebras" - | "openrouter"; + | "openrouter" + | "deepseek"; type SupportedChatModelDefinition = { id: string; @@ -130,6 +131,27 @@ export type ModelPricing = { outputUsdPerMillionTokens: 0.1, }, }, + { + // DeepSeek V4.1 Flash (1M context, tool calls + vision). Pricing is peak + // cache-miss input + peak output so credit estimates over- rather than + // under-charge; cache hits and off-peak hours are half or less. + // Source: api-docs.deepseek.com/quick_start/pricing + id: "deepseek-flash", + provider: "deepseek", + pricing: { + inputUsdPerMillionTokens: 0.3, + outputUsdPerMillionTokens: 1.2, + }, + }, + { + // DeepSeek V4 Pro — same pricing basis as deepseek-flash above. + id: "deepseek-v4-pro", + provider: "deepseek", + pricing: { + inputUsdPerMillionTokens: 1.32, + outputUsdPerMillionTokens: 3.96, + }, + }, ] as const satisfies readonly SupportedChatModelDefinition[]; export type SupportedChatModel = (typeof SUPPORTED_CHAT_MODELS)[number]; @@ -147,8 +169,12 @@ export type ModelPricing = { { message: "Unsupported chat model" }, ); - /** Default model when the session UI does not expose an explicit picker yet. */ - export const DEFAULT_CHAT_MODEL_ID: SupportedChatModelId = "gpt-oss-120b"; + /** + * Default model. DeepSeek is reachable from mainland China without a proxy, + * unlike the Cerebras / Groq / Anthropic endpoints (which answer 403 "Forbidden" + * behind a country block). Switch with `/models`. + */ + export const DEFAULT_CHAT_MODEL_ID: SupportedChatModelId = "deepseek-flash"; /** BYOK catalog hints for the `/models` picker (free-tier tool-calling guidance). */ export type ModelCatalogHint = "free-recommended" | "free-ok" | "weak-tools"; From 0b27ce9f725b4d3629165d1b1af20ea8aaed6c70 Mon Sep 17 00:00:00 2001 From: moyun Date: Thu, 12 Nov 2026 09:35:23 +0800 Subject: [PATCH 02/11] feat(agent): explicit loop budgets and live progress The multi-step loop was an implicit side effect of the SDK's auto-resubmit: no turn budget, no per-turn tool cap, and no way to explain why a run stopped. - `agent-loop.ts`: turn budget (24), per-turn tool cap (16), and a consecutive-repeat guard that distinguishes "read -> edit -> read" from a true loop. Five named stop reasons. - `agent-phase.ts`: derive waiting/thinking/running/generating from the live message instead of tracking it separately, so the label cannot disagree with the transcript. Run activity is keyed on pending tool calls too, because the client-driven loop sits at `ready` while tools execute. - `tool-timing.ts`: a bounded side-table of tool start times, so a running tool can show elapsed time without mutating SDK-owned message parts. --- packages/cli/src/lib/agent-loop.test.ts | 187 +++++++++++++++++++ packages/cli/src/lib/agent-loop.ts | 225 +++++++++++++++++++++++ packages/cli/src/lib/agent-phase.test.ts | 118 ++++++++++++ packages/cli/src/lib/agent-phase.ts | 120 ++++++++++++ packages/cli/src/lib/tool-timing.test.ts | 58 ++++++ packages/cli/src/lib/tool-timing.ts | 47 +++++ 6 files changed, 755 insertions(+) create mode 100644 packages/cli/src/lib/agent-loop.test.ts create mode 100644 packages/cli/src/lib/agent-loop.ts create mode 100644 packages/cli/src/lib/agent-phase.test.ts create mode 100644 packages/cli/src/lib/agent-phase.ts create mode 100644 packages/cli/src/lib/tool-timing.test.ts create mode 100644 packages/cli/src/lib/tool-timing.ts diff --git a/packages/cli/src/lib/agent-loop.test.ts b/packages/cli/src/lib/agent-loop.test.ts new file mode 100644 index 0000000..1747f82 --- /dev/null +++ b/packages/cli/src/lib/agent-loop.test.ts @@ -0,0 +1,187 @@ +import { describe, expect, test } from "bun:test"; +import { AgentLoop, DEFAULT_AGENT_LIMITS, formatAgentProgress, toolSignature } from "./agent-loop"; + +describe("AgentLoop turn budget", () => { + test("allows maxTurns turns, then stops with a named reason", () => { + const loop = new AgentLoop({ maxTurns: 3 }); + + expect(loop.beginTurn().action).toBe("continue"); + expect(loop.beginTurn().action).toBe("continue"); + expect(loop.beginTurn().action).toBe("continue"); + + const decision = loop.beginTurn(); + expect(decision).toEqual({ + action: "stop", + reason: "max-turns", + detail: "Stopped after 3 turns without a final answer.", + }); + }); + + test("stop is sticky and reports the original reason", () => { + const loop = new AgentLoop({ maxTurns: 1 }); + loop.beginTurn(); + loop.beginTurn(); + + expect(loop.beginTurn()).toEqual(loop.beginTurn()); + expect(loop.stopped).toBe("max-turns"); + }); +}); + +describe("AgentLoop tool budget", () => { + test("caps tool calls inside one turn and resets per turn", () => { + const loop = new AgentLoop({ maxToolCallsPerTurn: 2 }); + + loop.beginTurn(); + expect(loop.recordToolCall("readFile", { path: "a" }).action).toBe("continue"); + expect(loop.recordToolCall("readFile", { path: "b" }).action).toBe("continue"); + + const capped = loop.recordToolCall("readFile", { path: "c" }); + expect(capped).toMatchObject({ action: "stop", reason: "max-tool-calls" }); + }); + + test("a new turn restores the per-turn budget", () => { + const loop = new AgentLoop({ maxToolCallsPerTurn: 1 }); + + loop.beginTurn(); + loop.recordToolCall("readFile", { path: "a" }); + expect(loop.recordToolCall("readFile", { path: "b" }).action).toBe("stop"); + + const fresh = new AgentLoop({ maxToolCallsPerTurn: 1 }); + fresh.beginTurn(); + fresh.recordToolCall("readFile", { path: "a" }); + fresh.beginTurn(); + expect(fresh.recordToolCall("readFile", { path: "b" }).action).toBe("continue"); + }); +}); + +describe("AgentLoop repeat guard", () => { + test("stops when an identical call repeats consecutively beyond the threshold", () => { + const loop = new AgentLoop({ maxConsecutiveRepeats: 3 }); + loop.beginTurn(); + + for (let i = 0; i < 3; i += 1) { + expect(loop.recordToolCall("bash", { command: "ls" }).action).toBe("continue"); + } + + expect(loop.recordToolCall("bash", { command: "ls" })).toMatchObject({ + action: "stop", + reason: "no-progress", + }); + }); + + test("read → edit → read is normal work, not a loop", () => { + const loop = new AgentLoop({ maxConsecutiveRepeats: 1 }); + loop.beginTurn(); + + expect(loop.recordToolCall("readFile", { path: "a" }).action).toBe("continue"); + expect(loop.recordToolCall("editFile", { path: "a" }).action).toBe("continue"); + expect(loop.recordToolCall("readFile", { path: "a" }).action).toBe("continue"); + }); + + test("same tool with different input does not count as a repeat", () => { + const loop = new AgentLoop({ maxConsecutiveRepeats: 1 }); + loop.beginTurn(); + + expect(loop.recordToolCall("readFile", { path: "a" }).action).toBe("continue"); + expect(loop.recordToolCall("readFile", { path: "b" }).action).toBe("continue"); + expect(loop.recordToolCall("readFile", { path: "c" }).action).toBe("continue"); + }); + + test("signature is insensitive to key order", () => { + expect(toolSignature("grep", { pattern: "x", path: "a" })).toBe( + toolSignature("grep", { path: "a", pattern: "x" }), + ); + expect(toolSignature("grep", { pattern: "x" })).not.toBe( + toolSignature("grep", { pattern: "y" }), + ); + }); +}); + +describe("AgentLoop lifecycle", () => { + test("a new user message resets the budget", () => { + const loop = new AgentLoop({ maxTurns: 1 }); + + loop.beginUserMessage("user-1"); + loop.beginTurn(); + expect(loop.beginTurn().action).toBe("stop"); + + loop.beginUserMessage("user-2"); + expect(loop.beginTurn().action).toBe("continue"); + }); + + test("regenerating the same message keeps the budget", () => { + const loop = new AgentLoop({ maxTurns: 1 }); + + loop.beginUserMessage("user-1"); + loop.beginTurn(); + loop.beginTurn(); + loop.beginUserMessage("user-1"); + + expect(loop.stopped).toBe("max-turns"); + }); + + test("reset() clears a stop so an explicit retry can proceed", () => { + const loop = new AgentLoop({ maxTurns: 1 }); + + loop.beginUserMessage("user-1"); + loop.beginTurn(); + loop.beginTurn(); + + loop.reset(); + expect(loop.stopped).toBeNull(); + expect(loop.beginTurn().action).toBe("continue"); + }); + + test("abort() is sticky", () => { + const loop = new AgentLoop(); + loop.beginUserMessage("user-1"); + loop.beginTurn(); + loop.abort(); + + expect(loop.stopped).toBe("aborted"); + expect(loop.beginTurn()).toMatchObject({ action: "stop", reason: "aborted" }); + }); + + test("snapshot reports budget usage", () => { + const loop = new AgentLoop({ maxTurns: 5, maxToolCallsPerTurn: 4 }); + loop.beginUserMessage("user-1"); + loop.beginTurn(); + loop.recordToolCall("readFile", { path: "a" }); + loop.recordToolCall("readFile", { path: "b" }); + + expect(loop.snapshot()).toEqual({ + turn: 1, + maxTurns: 5, + toolCallsThisTurn: 2, + totalToolCalls: 2, + stopReason: null, + }); + }); + + test("defaults are exposed for the TUI status line", () => { + expect(DEFAULT_AGENT_LIMITS.maxTurns).toBeGreaterThan(0); + }); +}); + +describe("formatAgentProgress", () => { + test("renders nothing before a run starts", () => { + expect(formatAgentProgress(null)).toBeNull(); + expect(formatAgentProgress({ turn: 0, maxTurns: 24, toolCallsThisTurn: 0 })).toBeNull(); + }); + + test("renders the turn budget", () => { + expect(formatAgentProgress({ turn: 3, maxTurns: 24, toolCallsThisTurn: 0 })).toBe("turn 4/25"); + }); + + test("appends the tool count once the turn has run tools", () => { + expect(formatAgentProgress({ turn: 3, maxTurns: 24, toolCallsThisTurn: 4 })).toBe( + "turn 4/25 · 4 tools", + ); + }); + + test("uses the singular label for a single tool call", () => { + expect(formatAgentProgress({ turn: 1, maxTurns: 24, toolCallsThisTurn: 1 })).toBe( + "turn 2/25 · 1 tool", + ); + }); +}); diff --git a/packages/cli/src/lib/agent-loop.ts b/packages/cli/src/lib/agent-loop.ts new file mode 100644 index 0000000..47032d3 --- /dev/null +++ b/packages/cli/src/lib/agent-loop.ts @@ -0,0 +1,225 @@ +/** + * Explicit agent-loop controller. + * + * Before this module, MoCode's multi-step loop was implicit: the AI SDK's + * `sendAutomaticallyWhen` re-posted whenever the last assistant message had no + * pending tool calls. That produces a working loop but leaves three questions a + * production agent must be able to answer unanswered: + * + * 1. How much work may one user message spend? → turn + per-turn tool budgets + * 2. When is the agent stuck rather than working? → consecutive-repeat guard + * 3. Why did it stop? → named stop reasons + * + * One *turn* is one model response plus the tool calls it requested. The first + * response after a user message is turn 1. {@link AgentLoop.beginTurn} runs when + * the loop decides to continue, so `maxTurns` bounds *continued* turns: with + * `maxTurns: 24` a single user message can produce at most 25 model responses. + * + * Pure module: no React, no network, no filesystem — unit-testable in isolation. + */ + +/** Why the loop stopped. Every exit is named so the TUI can explain it. */ +export type AgentStopReason = + | "max-turns" + | "max-tool-calls" + | "no-progress" + | "aborted" + | "error"; + +/** Budgets for one user message. */ +export type AgentLoopLimits = { + /** Model responses the loop may continue to. */ + maxTurns: number; + /** Tool calls allowed inside a single turn. */ + maxToolCallsPerTurn: number; + /** Identical consecutive tool calls tolerated before declaring no progress. */ + maxConsecutiveRepeats: number; +}; + +export const DEFAULT_AGENT_LIMITS: AgentLoopLimits = { + maxTurns: 24, + maxToolCallsPerTurn: 16, + maxConsecutiveRepeats: 3, +}; + +export type AgentLoopDecision = + | { action: "continue" } + | { action: "stop"; reason: AgentStopReason; detail: string }; + +export type AgentLoopSnapshot = { + turn: number; + maxTurns: number; + toolCallsThisTurn: number; + totalToolCalls: number; + stopReason: AgentStopReason | null; +}; + +/** Reactive subset rendered by the status line ({components.status-bar} in DESIGN.md). */ +export type AgentProgress = Pick< + AgentLoopSnapshot, + "turn" | "maxTurns" | "toolCallsThisTurn" +>; + +/** + * Status-line label for the loop budget, or `null` when no run has started. + * One dim segment: `turn 3/24`, plus `· 4 tools` when the turn has run tools. + */ +export function formatAgentProgress(progress: AgentProgress | null): string | null { + if (!progress || progress.turn <= 0) return null; + + // `turn` counts *continuations*: the first response is already spent by the time + // the loop decides to continue, and `maxTurns` bounds continuations. Showing the + // raw counter labelled "turn" under-reported both the spend and the budget. + const responses = progress.turn + 1; + const budget = progress.maxTurns + 1; + const base = `turn ${responses}/${budget}`; + if (progress.toolCallsThisTurn <= 0) return base; + const label = progress.toolCallsThisTurn === 1 ? "tool" : "tools"; + return `${base} · ${progress.toolCallsThisTurn} ${label}`; +} + +/** + * Stable serialization used for tool-call signatures. Key order in model-emitted + * JSON is not stable, so a plain `JSON.stringify` would miss true repeats. + */ +function stableKey(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value) ?? "null"; + if (Array.isArray(value)) return `[${value.map(stableKey).join(",")}]`; + + const entries = Object.entries(value as Record) + .filter(([, v]) => v !== undefined) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => `${JSON.stringify(k)}:${stableKey(v)}`); + + return `{${entries.join(",")}}`; +} + +/** Signature of one tool call: same tool + same normalized input. */ +export function toolSignature(toolName: string, input: unknown): string { + return `${toolName}:${stableKey(input)}`; +} + +export class AgentLoop { + private readonly limits: AgentLoopLimits; + private turn = 0; + private toolCallsThisTurn = 0; + private totalToolCalls = 0; + private stopReason: AgentStopReason | null = null; + private stopDetail = ""; + private userMessageId: string | null = null; + private lastSignature: string | null = null; + private consecutiveRepeats = 0; + + constructor(limits: Partial = {}) { + this.limits = { ...DEFAULT_AGENT_LIMITS, ...limits }; + } + + /** + * Start (or continue) the loop for a user message. A different id starts a + * fresh budget; the same id keeps the current one, so regenerating a reply does + * not silently hand the model an extra 24 turns. + */ + beginUserMessage(messageId: string): void { + if (this.userMessageId === messageId) return; + this.userMessageId = messageId; + this.reset(); + } + + /** Clear budgets and stop state, keeping the current user message id. */ + reset(): void { + this.turn = 0; + this.toolCallsThisTurn = 0; + this.totalToolCalls = 0; + this.stopReason = null; + this.stopDetail = ""; + this.lastSignature = null; + this.consecutiveRepeats = 0; + } + + /** Ask permission to start the next model turn. */ + beginTurn(): AgentLoopDecision { + const stopped = this.stoppedDecision(); + if (stopped) return stopped; + + if (this.turn >= this.limits.maxTurns) { + return this.fail( + "max-turns", + `Stopped after ${this.limits.maxTurns} turns without a final answer.`, + ); + } + + this.turn += 1; + this.toolCallsThisTurn = 0; + return { action: "continue" }; + } + + /** Account for one tool call before it executes. */ + recordToolCall(toolName: string, input: unknown): AgentLoopDecision { + const stopped = this.stoppedDecision(); + if (stopped) return stopped; + + if (this.toolCallsThisTurn >= this.limits.maxToolCallsPerTurn) { + return this.fail( + "max-tool-calls", + `Stopped after ${this.limits.maxToolCallsPerTurn} tool calls in one turn.`, + ); + } + + this.toolCallsThisTurn += 1; + this.totalToolCalls += 1; + + const signature = toolSignature(toolName, input); + // Only *consecutive* identical calls count. read → edit → read is normal + // work; read → read → read → read with identical input is a loop. + this.consecutiveRepeats = signature === this.lastSignature ? this.consecutiveRepeats + 1 : 1; + this.lastSignature = signature; + + if (this.consecutiveRepeats > this.limits.maxConsecutiveRepeats) { + return this.fail( + "no-progress", + `Stopped: ${toolName} was called ${this.consecutiveRepeats} times in a row with identical input.`, + ); + } + + return { action: "continue" }; + } + + /** Escape hatch for user interruption (Esc). */ + abort(): void { + this.fail("aborted", "Stopped by user."); + } + + /** Escape hatch for a transport or tool failure that ends the run. */ + failWith(reason: AgentStopReason, detail: string): AgentLoopDecision { + return this.fail(reason, detail); + } + + get stopped(): AgentStopReason | null { + return this.stopReason; + } + + snapshot(): AgentLoopSnapshot { + return { + turn: this.turn, + maxTurns: this.limits.maxTurns, + toolCallsThisTurn: this.toolCallsThisTurn, + totalToolCalls: this.totalToolCalls, + stopReason: this.stopReason, + }; + } + + private fail(reason: AgentStopReason, detail: string): AgentLoopDecision { + // Sticky: once stopped, every later decision repeats the same reason. + if (this.stopReason === null) { + this.stopReason = reason; + this.stopDetail = detail; + } + + return { action: "stop", reason: this.stopReason, detail: this.stopDetail }; + } + + private stoppedDecision(): AgentLoopDecision | null { + if (this.stopReason === null) return null; + return { action: "stop", reason: this.stopReason, detail: this.stopDetail }; + } +} diff --git a/packages/cli/src/lib/agent-phase.test.ts b/packages/cli/src/lib/agent-phase.test.ts new file mode 100644 index 0000000..749af79 --- /dev/null +++ b/packages/cli/src/lib/agent-phase.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, test } from "bun:test"; +import { + deriveAgentPhase, + isRunActive, + formatAgentPhase, + formatElapsed, + type PhaseMessage, +} from "./agent-phase"; + +function message(parts: unknown[]): PhaseMessage { + return { parts }; +} + +const pendingTool = { type: "tool-bash", state: "input-available" }; +const doneTool = { type: "tool-bash", state: "output-available" }; +const reasoning = { type: "reasoning" }; +const text = { type: "text" }; + +describe("isRunActive", () => { + test("status alone does not decide it", () => { + // The loop is client-driven: between a model step and the auto-resubmit the + // status is `ready` while tools are still executing. + expect(isRunActive({ status: "ready", message: message([pendingTool]) })).toBe(true); + }); + + test("idle once the last tool call is resolved", () => { + expect(isRunActive({ status: "ready", message: message([doneTool]) })).toBe(false); + }); + + test("streaming is active even with no parts yet", () => { + expect(isRunActive({ status: "streaming" })).toBe(true); + }); +}); + +describe("deriveAgentPhase", () => { + test("stays 'running' in the gap between a step and the resubmit", () => { + expect(deriveAgentPhase({ status: "ready", message: message([pendingTool]) })).toBe("running"); + }); + + test("idle when the chat is not running", () => { + expect(deriveAgentPhase({ status: "ready", message: message([text]) })).toBeNull(); + expect(deriveAgentPhase({ status: "error" })).toBeNull(); + }); + + test("waiting for the first token", () => { + expect(deriveAgentPhase({ status: "submitted" })).toBe("waiting"); + expect(deriveAgentPhase({ status: "streaming", message: message([]) })).toBe("waiting"); + }); + + test("thinking while only reasoning has streamed", () => { + expect(deriveAgentPhase({ status: "streaming", message: message([reasoning]) })).toBe( + "thinking", + ); + }); + + test("generating once text appears", () => { + expect(deriveAgentPhase({ status: "streaming", message: message([reasoning, text]) })).toBe( + "generating", + ); + }); + + test("running wins over text — the agent is acting, not talking", () => { + expect( + deriveAgentPhase({ status: "streaming", message: message([text, pendingTool]) }), + ).toBe("running"); + }); + + test("a resolved tool no longer counts as running", () => { + expect(deriveAgentPhase({ status: "streaming", message: message([text, doneTool]) })).toBe( + "generating", + ); + }); + + test("dynamic MCP calls count as running too", () => { + expect( + deriveAgentPhase({ + status: "streaming", + message: message([{ type: "dynamic-tool", state: "input-available" }]), + }), + ).toBe("running"); + }); + + test("tool parts with no state yet are treated as pending", () => { + expect( + deriveAgentPhase({ status: "streaming", message: message([{ type: "tool-readFile" }]) }), + ).toBe("running"); + }); +}); + +describe("formatElapsed", () => { + test("shows tenths under a minute", () => { + expect(formatElapsed(2_400)).toBe("2.4s"); + expect(formatElapsed(0)).toBe("0.0s"); + }); + + test("switches to minutes past 60s, like the harness does", () => { + expect(formatElapsed(174_000)).toBe("2m54s"); + }); + + test("never goes negative", () => { + expect(formatElapsed(-500)).toBe("0.0s"); + }); +}); + +describe("formatAgentPhase", () => { + test("idle renders nothing", () => { + expect(formatAgentPhase(null, 1_000)).toBeNull(); + }); + + test("shows the phase with elapsed time", () => { + expect(formatAgentPhase("running", 2_400)).toBe("执行中 2.4s"); + expect(formatAgentPhase("thinking", 1_000)).toBe("思考中 1.0s"); + }); + + test("omits the timer when the elapsed time is unknown", () => { + expect(formatAgentPhase("waiting", null)).toBe("等待首字"); + }); +}); diff --git a/packages/cli/src/lib/agent-phase.ts b/packages/cli/src/lib/agent-phase.ts new file mode 100644 index 0000000..ecc0d43 --- /dev/null +++ b/packages/cli/src/lib/agent-phase.ts @@ -0,0 +1,120 @@ +/** + * What the agent is doing *right now*. + * + * The transcript already records what happened; this answers the question a user + * asks while waiting: "is it thinking, or stuck?" The phase is derived from the + * live message parts rather than tracked separately, so it can never disagree with + * what is on screen. + * + * Pure module: no timers, no I/O. + */ + +export type AgentPhase = "waiting" | "thinking" | "running" | "generating"; + +/** Minimum shape needed from a UI message — keeps this testable without the SDK. */ +export type PhaseMessage = { + parts?: unknown; +}; + +type PartLike = { + type?: unknown; + state?: unknown; +}; + +function partsOf(message: PhaseMessage | null | undefined): PartLike[] { + if (!message || !Array.isArray(message.parts)) return []; + return message.parts as PartLike[]; +} + +/** A tool part that has been requested but not yet resolved. */ +function hasPendingTool(parts: PartLike[]): boolean { + return parts.some((part) => { + const type = typeof part.type === "string" ? part.type : ""; + const isTool = type === "dynamic-tool" || type.startsWith("tool-"); + if (!isTool) return false; + + const state = typeof part.state === "string" ? part.state : ""; + return state !== "output-available" && state !== "output-error"; + }); +} + +function hasPart(parts: PartLike[], type: string): boolean { + return parts.some((part) => part.type === type); +} + +/** + * Whether a run is in progress. + * + * `chat.status` alone is not enough: the loop is client-driven, so between a model + * step and the auto-resubmit the status is `ready` even though the agent is busy + * executing tools. Keying "active" on the status made the phase disappear exactly + * during tool execution — the one moment a user most wants to see it. + */ +export function isRunActive(params: { + /** Chat status from the SDK. */ + status: string; + /** The message being streamed or awaiting tool results. */ + message?: PhaseMessage | null; +}): boolean { + if (params.status === "submitted" || params.status === "streaming") return true; + + return hasPendingTool(partsOf(params.message)); +} + +/** + * Derive the current phase, or `null` when the agent is idle. + * + * Precedence is deliberate: a pending tool means the agent is *acting*, which is + * more useful to show than the text that preceded it. + */ +export function deriveAgentPhase(params: { + /** Chat status from the SDK. */ + status: string; + /** The message being streamed, if any. */ + message?: PhaseMessage | null; +}): AgentPhase | null { + if (!isRunActive(params)) return null; + + const parts = partsOf(params.message); + if (parts.length === 0) return "waiting"; + + if (hasPendingTool(parts)) return "running"; + + const hasText = hasPart(parts, "text"); + const hasReasoning = hasPart(parts, "reasoning"); + + if (hasReasoning && !hasText) return "thinking"; + if (hasText) return "generating"; + + return "waiting"; +} + +/** Human label per phase. Short on purpose — it lives in a one-row status line. */ +const PHASE_LABEL: Record = { + waiting: "等待首字", + thinking: "思考中", + running: "执行中", + generating: "生成中", +}; + +/** Seconds with one decimal, e.g. `2.4s`. */ +export function formatElapsed(ms: number): string { + const seconds = Math.max(0, ms) / 1000; + return seconds < 60 ? `${seconds.toFixed(1)}s` : `${Math.floor(seconds / 60)}m${Math.round(seconds % 60)}s`; +} + +/** + * Status-line segment, e.g. `执行中 2.4s`, or `null` when idle. + * + * `elapsedMs` is omitted from the label when unknown so the line does not show a + * misleading `0.0s` for work that has only just started. + */ +export function formatAgentPhase( + phase: AgentPhase | null, + elapsedMs: number | null, +): string | null { + if (!phase) return null; + + const label = PHASE_LABEL[phase]; + return elapsedMs === null ? label : `${label} ${formatElapsed(elapsedMs)}`; +} diff --git a/packages/cli/src/lib/tool-timing.test.ts b/packages/cli/src/lib/tool-timing.test.ts new file mode 100644 index 0000000..449078e --- /dev/null +++ b/packages/cli/src/lib/tool-timing.test.ts @@ -0,0 +1,58 @@ +import { beforeEach, describe, expect, test } from "bun:test"; +import { + clearToolTiming, + formatToolElapsed, + markToolStarted, + resetToolTimings, + toolElapsedMs, +} from "./tool-timing"; + +beforeEach(() => resetToolTimings()); + +describe("tool timing registry", () => { + test("reports elapsed time from the mark", () => { + markToolStarted("call-1", 1_000); + expect(toolElapsedMs("call-1", 3_400)).toBe(2_400); + }); + + test("unknown calls report null instead of a fake zero", () => { + expect(toolElapsedMs("never-seen", 5_000)).toBeNull(); + }); + + test("the first mark wins, so a retry does not reset the clock", () => { + markToolStarted("call-1", 1_000); + markToolStarted("call-1", 9_000); + expect(toolElapsedMs("call-1", 3_000)).toBe(2_000); + }); + + test("never reports negative time if the clock jumps backwards", () => { + markToolStarted("call-1", 5_000); + expect(toolElapsedMs("call-1", 1_000)).toBe(0); + }); + + test("clearing removes the entry", () => { + markToolStarted("call-1", 1_000); + clearToolTiming("call-1"); + expect(toolElapsedMs("call-1", 2_000)).toBeNull(); + }); + + test("stays bounded across a long session", () => { + for (let index = 0; index < 700; index += 1) { + markToolStarted(`call-${index}`, index); + } + + // Oldest evicted, newest retained — no unbounded growth. + expect(toolElapsedMs("call-0", 1_000)).toBeNull(); + expect(toolElapsedMs("call-699", 1_000)).not.toBeNull(); + }); +}); + +describe("formatToolElapsed", () => { + test("tenths under a minute", () => { + expect(formatToolElapsed(2_300)).toBe("2.3s"); + }); + + test("minutes past 60s", () => { + expect(formatToolElapsed(125_000)).toBe("2m5s"); + }); +}); diff --git a/packages/cli/src/lib/tool-timing.ts b/packages/cli/src/lib/tool-timing.ts new file mode 100644 index 0000000..75a92fc --- /dev/null +++ b/packages/cli/src/lib/tool-timing.ts @@ -0,0 +1,47 @@ +/** + * Per-tool start times, so a running tool can show how long it has been running. + * + * Deliberately a tiny side-table rather than a field on the message part: adding a + * field would mean mutating SDK-owned message data, and the number is presentation + * only. Entries are keyed by `toolCallId`, which is unique per call. + * + * The map is bounded — a long session with hundreds of calls must not leak. + */ + +const startedAt = new Map(); + +/** Cap on retained entries; oldest are evicted first. */ +const MAX_ENTRIES = 512; + +/** Record when a tool call began. Safe to call twice (first write wins). */ +export function markToolStarted(toolCallId: string, now: number = Date.now()): void { + if (startedAt.has(toolCallId)) return; + + if (startedAt.size >= MAX_ENTRIES) { + const oldest = startedAt.keys().next(); + if (!oldest.done) startedAt.delete(oldest.value); + } + + startedAt.set(toolCallId, now); +} + +/** Milliseconds since the call began, or `null` when it was never marked. */ +export function toolElapsedMs(toolCallId: string, now: number = Date.now()): number | null { + const start = startedAt.get(toolCallId); + return start === undefined ? null : Math.max(0, now - start); +} + +export function clearToolTiming(toolCallId: string): void { + startedAt.delete(toolCallId); +} + +/** Drop all entries — used when a new run starts and by tests. */ +export function resetToolTimings(): void { + startedAt.clear(); +} + +/** Compact label for a running tool, e.g. `2.3s`. */ +export function formatToolElapsed(ms: number): string { + const seconds = Math.max(0, ms) / 1000; + return seconds < 60 ? `${seconds.toFixed(1)}s` : `${Math.floor(seconds / 60)}m${Math.round(seconds % 60)}s`; +} From cd99524ba0c89e1558408f76973fc9163b5fccdb Mon Sep 17 00:00:00 2001 From: moyun Date: Thu, 12 Nov 2026 09:35:27 +0800 Subject: [PATCH 03/11] feat(cli): provider failover with consent, retries and stall detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A model was hard-bound to one provider, so one blocked endpoint meant every turn failed. Measured on a blocked network: 2 of 15 catalog models usable. Policy: - failover is **opt-in** (`MOCODE_ALLOW_MODEL_FAILOVER=1`): retrying on another vendor ships the user's code to a company they did not pick - the default chain is `deepseek-flash` only — `deepseek-v4-pro` costs ~4x and is never auto-escalated Transport: - retries with exponential backoff + jitter; a valid key should be enough - `retrySameProvider` distinguishes transient (429/5xx/timeout) from terminal (401/403/404) failures — retrying a 403 only delays the fallback - the deadline bounds **connect + headers** only; reading the stream uses a silent-for-N-seconds stall timer reset on every chunk, because a long generation is not a network fault - a 200 can still carry a provider error as an SSE event; peeking covers that, and an empty stream no longer counts as success `HttpError` keeps the status code so callers can act on it at all. --- packages/cli/src/lib/chat-fetch.test.ts | 384 ++++++++++++++++++ packages/cli/src/lib/chat-fetch.ts | 348 ++++++++++++++++ packages/cli/src/lib/http-errors.ts | 19 +- packages/cli/src/lib/provider-chain.test.ts | 293 +++++++++++++ packages/cli/src/lib/provider-chain.ts | 285 +++++++++++++ .../server/scripts/provider-failover-bench.ts | 216 ++++++++++ 6 files changed, 1541 insertions(+), 4 deletions(-) create mode 100644 packages/cli/src/lib/chat-fetch.test.ts create mode 100644 packages/cli/src/lib/chat-fetch.ts create mode 100644 packages/cli/src/lib/provider-chain.test.ts create mode 100644 packages/cli/src/lib/provider-chain.ts create mode 100644 packages/server/scripts/provider-failover-bench.ts diff --git a/packages/cli/src/lib/chat-fetch.test.ts b/packages/cli/src/lib/chat-fetch.test.ts new file mode 100644 index 0000000..c919098 --- /dev/null +++ b/packages/cli/src/lib/chat-fetch.test.ts @@ -0,0 +1,384 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { createFailoverFetch } from "./chat-fetch"; +import { HttpError } from "./http-errors"; + +const readError = async (response: Response) => { + try { + const data = (await response.json()) as { error?: string }; + return data.error ?? `HTTP ${response.status}`; + } catch { + return `HTTP ${response.status}`; + } +}; + +/** Records every call so tests can assert on the rewritten body. */ +function makeFetch(responses: Array Response)>) { + const calls: Array<{ url: string; body: unknown }> = []; + + const fetchImpl = (async (input: RequestInfo | URL, init?: RequestInit) => { + let body: unknown = init?.body; + if (typeof init?.body === "string") { + try { + body = JSON.parse(init.body); + } catch { + body = init.body; + } + } + calls.push({ url: String(input), body }); + + const next = responses.shift(); + if (!next) throw new Error("unexpected extra fetch call"); + return typeof next === "function" ? next() : next; + }) as typeof globalThis.fetch; + + return { fetchImpl, calls }; +} + +function chatInit(model: string) { + return { + method: "POST", + body: JSON.stringify({ id: "session-1", model, mode: "build", messages: [] }), + } satisfies RequestInit; +} + +/** Build a 200 SSE response shaped like the server's UI message stream. */ +function sseResponse(events: Array>): Response { + const body = events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join(""); + + return new Response(body, { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); +} + +describe("createFailoverFetch", () => { + test("returns the primary response without reporting a downgrade", async () => { + const { fetchImpl, calls } = makeFetch([new Response("ok", { status: 200 })]); + const downgrades: unknown[] = []; + + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + attemptsPerModel: 1, + fetchImpl, + readError, + onDowngrade: (info) => downgrades.push(info), + }); + + const response = await fetchWithFallback("http://localhost:3000/chat", chatInit("deepseek-flash")); + + expect(response.status).toBe(200); + expect(calls).toHaveLength(1); + expect(downgrades).toEqual([]); + }); + + test("retries on 403 and reports the downgrade explicitly", async () => { + const { fetchImpl, calls } = makeFetch([ + new Response(JSON.stringify({ error: "Forbidden" }), { status: 403 }), + new Response("ok", { status: 200 }), + ]); + const downgrades: Array<{ from: string; to: string }> = []; + + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + attemptsPerModel: 1, + fetchImpl, + readError, + onDowngrade: (info) => downgrades.push(info), + }); + + const response = await fetchWithFallback("http://localhost:3000/chat", chatInit("gpt-oss-120b")); + + expect(response.status).toBe(200); + expect(calls).toHaveLength(2); + // Second attempt must ask the server for the fallback model, nothing else changes. + expect(calls[1]?.body).toMatchObject({ model: "deepseek-flash", id: "session-1", mode: "build" }); + expect(downgrades).toEqual([ + { + from: "gpt-oss-120b", + to: "deepseek-flash", + failures: [{ modelId: "gpt-oss-120b", detail: "HTTP 403" }], + }, + ]); + }); + + test("does not retry a 400 — the request itself is wrong", async () => { + const { fetchImpl, calls } = makeFetch([ + new Response(JSON.stringify({ error: "Invalid request body" }), { status: 400 }), + ]); + + const fetchWithFallback = createFailoverFetch({ allowFailover: true, attemptsPerModel: 1, fetchImpl, readError }); + + await expect( + fetchWithFallback("http://localhost:3000/chat", chatInit("deepseek-flash")), + ).rejects.toBeInstanceOf(HttpError); + + expect(calls).toHaveLength(1); + }); + + test("throws the last failure when every candidate is blocked", async () => { + const { fetchImpl, calls } = makeFetch([ + new Response("", { status: 403 }), + new Response("", { status: 403 }), + new Response("", { status: 403 }), + ]); + + const fetchWithFallback = createFailoverFetch({ allowFailover: true, attemptsPerModel: 1, fetchImpl, readError }); + + await expect( + fetchWithFallback("http://localhost:3000/chat", chatInit("gpt-oss-120b")), + ).rejects.toBeInstanceOf(HttpError); + + // Default chain is flash only: Pro costs ~4x and is never auto-escalated. + expect(calls.map((call) => (call.body as { model: string }).model)).toEqual([ + "gpt-oss-120b", + "deepseek-flash", + ]); + }); + + test("passes through requests it cannot fail over", async () => { + const { fetchImpl, calls } = makeFetch([ + new Response("ok", { status: 200 }), + new Response("ok", { status: 200 }), + ]); + const fetchWithFallback = createFailoverFetch({ allowFailover: true, attemptsPerModel: 1, fetchImpl, readError }); + + await fetchWithFallback("http://localhost:3000/chat", { method: "POST", body: "not-json" }); + await fetchWithFallback("http://localhost:3000/chat", { + method: "POST", + body: JSON.stringify({ id: "x" }), + }); + + expect(calls).toHaveLength(2); + }); + + test("fails over when the server reports the provider error inside a 200 stream", async () => { + // This is the real shape: the server answers 200 and streams the upstream + // failure as an `error` event, so `response.ok` is true and a status-only + // check would never retry. + const { fetchImpl, calls } = makeFetch([ + sseResponse([{ type: "start" }, { type: "error", errorText: "Forbidden" }]), + sseResponse([ + { type: "start" }, + { type: "text-delta", delta: "recovered" }, + { type: "finish" }, + ]), + ]); + const downgrades: Array<{ from: string; to: string }> = []; + + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + attemptsPerModel: 1, + fetchImpl, + readError, + onDowngrade: (info) => downgrades.push(info), + }); + + const response = await fetchWithFallback( + "http://localhost:3000/chat", + chatInit("gpt-oss-120b"), + ); + + expect(calls).toHaveLength(2); + expect(calls[1]?.body).toMatchObject({ model: "deepseek-flash" }); + expect(downgrades[0]).toMatchObject({ from: "gpt-oss-120b", to: "deepseek-flash" }); + expect(await response.text()).toContain("recovered"); + }); + + test("a healthy stream is returned byte-for-byte, nothing lost by peeking", async () => { + const events = [ + { type: "start" }, + { type: "text-delta", delta: "hello" }, + { type: "text-delta", delta: " world" }, + { type: "finish" }, + ]; + const expected = events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join(""); + + const { fetchImpl, calls } = makeFetch([sseResponse(events)]); + const fetchWithFallback = createFailoverFetch({ allowFailover: true, attemptsPerModel: 1, fetchImpl, readError }); + + const response = await fetchWithFallback( + "http://localhost:3000/chat", + chatInit("deepseek-flash"), + ); + + expect(calls).toHaveLength(1); + expect(await response.text()).toBe(expected); + }); + + test("user abort is classified as AbortError, not a retryable HttpError", async () => { + const user = new AbortController(); + user.abort(); + + const fetchImpl = (async (_input: RequestInfo | URL, init?: RequestInit) => { + // Simulate a racing 403 that arrives as the user hits Esc. + if (init?.signal?.aborted) { + throw new HttpError(403, "Forbidden"); + } + return new Response("ok", { status: 200 }); + }) as typeof globalThis.fetch; + + const fetchWithFallback = createFailoverFetch({ allowFailover: true, attemptsPerModel: 1, fetchImpl, readError }); + + await expect( + fetchWithFallback("http://localhost:3000/chat", { + ...chatInit("deepseek-flash"), + signal: user.signal, + }), + ).rejects.toMatchObject({ name: "AbortError" }); + }); + + test("an SSE body that closes before any output fails over", async () => { + const { fetchImpl, calls } = makeFetch([ + sseResponse([{ type: "start" }]), + sseResponse([{ type: "start" }, { type: "text-delta", delta: "recovered" }, { type: "finish" }]), + ]); + + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + attemptsPerModel: 1, + fetchImpl, + readError, + }); + + const response = await fetchWithFallback( + "http://localhost:3000/chat", + chatInit("gpt-oss-120b"), + ); + + expect(calls).toHaveLength(2); + expect(await response.text()).toContain("recovered"); + }); + + test("without consent the turn is never sent to another provider", async () => { + const { fetchImpl, calls } = makeFetch([new Response("", { status: 403 })]); + + const fetchWithFallback = createFailoverFetch({ fetchImpl, readError }); + + await expect( + fetchWithFallback("http://localhost:3000/chat", chatInit("gpt-oss-120b")), + ).rejects.toBeInstanceOf(HttpError); + + // Exactly one attempt: the user did not authorise another vendor. + expect(calls).toHaveLength(1); + }); + + test("a healthy long stream is not killed by the per-attempt deadline", async () => { + // Regression: the deadline used to cover the whole stream, so any turn longer + // than the timeout died mid-reply with "The operation timed out". + const encoder = new TextEncoder(); + const event = (payload: Record) => + encoder.encode(`data: ${JSON.stringify(payload)}\n\n`); + + const fetchImpl = (async () => + new Response( + new ReadableStream({ + async start(controller) { + controller.enqueue(event({ type: "start" })); + controller.enqueue(event({ type: "text-delta", delta: "first" })); + await new Promise((resolve) => setTimeout(resolve, 150)); + controller.enqueue(event({ type: "text-delta", delta: "late" })); + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + )) as typeof globalThis.fetch; + + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + attemptsPerModel: 1, + fetchImpl, + readError, + attemptTimeoutMs: 50, + }); + + const response = await fetchWithFallback( + "http://localhost:3000/chat", + chatInit("deepseek-flash"), + ); + const text = await response.text(); + + expect(text).toContain("first"); + // This chunk arrives well after the 50ms deadline; it must still be delivered. + expect(text).toContain("late"); + }); +}); + +describe("deadline policy", () => { + const original = process.env.MOCODE_CONNECT_TIMEOUT_MS; + const originalStall = process.env.MOCODE_STALL_TIMEOUT_MS; + + afterAll(() => { + if (original === undefined) delete process.env.MOCODE_CONNECT_TIMEOUT_MS; + else process.env.MOCODE_CONNECT_TIMEOUT_MS = original; + if (originalStall === undefined) delete process.env.MOCODE_STALL_TIMEOUT_MS; + else process.env.MOCODE_STALL_TIMEOUT_MS = originalStall; + }); + + test("connect budget defaults to 20s", async () => { + const { connectTimeoutMs } = await import("./chat-fetch"); + delete process.env.MOCODE_CONNECT_TIMEOUT_MS; + expect(connectTimeoutMs()).toBe(20_000); + }); + + test("stall budget defaults to 120s — it bounds silence, not generation", async () => { + const { stallTimeoutMs } = await import("./chat-fetch"); + delete process.env.MOCODE_STALL_TIMEOUT_MS; + expect(stallTimeoutMs()).toBe(120_000); + }); + + test("both are overridable", async () => { + const { connectTimeoutMs, stallTimeoutMs } = await import("./chat-fetch"); + process.env.MOCODE_CONNECT_TIMEOUT_MS = "3000"; + process.env.MOCODE_STALL_TIMEOUT_MS = "5000"; + expect(connectTimeoutMs()).toBe(3_000); + expect(stallTimeoutMs()).toBe(5_000); + }); + + test("nonsense values fall back instead of producing a zero budget", async () => { + const { connectTimeoutMs, stallTimeoutMs } = await import("./chat-fetch"); + process.env.MOCODE_CONNECT_TIMEOUT_MS = "-1"; + process.env.MOCODE_STALL_TIMEOUT_MS = "abc"; + expect(connectTimeoutMs()).toBe(20_000); + expect(stallTimeoutMs()).toBe(120_000); + }); +}); + +describe("a slow but alive stream is never killed", () => { + test("chunks trickling in with gaps under the stall budget complete normally", async () => { + const encoder = new TextEncoder(); + const event = (payload: Record) => + encoder.encode(`data: ${JSON.stringify(payload)}\n\n`); + + // Total duration ~450ms, well past the 120ms stall budget — but no single gap is. + const fetchImpl = (async () => + new Response( + new ReadableStream({ + async start(controller) { + controller.enqueue(event({ type: "start" })); + for (const piece of ["a", "b", "c", "d", "e"]) { + await new Promise((resolve) => setTimeout(resolve, 80)); + controller.enqueue(event({ type: "text-delta", delta: piece })); + } + controller.close(); + }, + }), + { status: 200, headers: { "content-type": "text/event-stream" } }, + )) as typeof globalThis.fetch; + + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + attemptsPerModel: 1, + fetchImpl, + readError, + stallTimeoutMs: 120, + }); + + const response = await fetchWithFallback( + "http://localhost:3000/chat", + chatInit("deepseek-flash"), + ); + const text = await response.text(); + + expect(text).toContain("text-delta"); + }); +}); diff --git a/packages/cli/src/lib/chat-fetch.ts b/packages/cli/src/lib/chat-fetch.ts new file mode 100644 index 0000000..93cce6d --- /dev/null +++ b/packages/cli/src/lib/chat-fetch.ts @@ -0,0 +1,348 @@ +/** + * Failover fetch for the SaaS chat transport. + * + * The server hard-binds one provider per model, so a region-blocked provider makes + * `POST /chat` answer 403 and the whole turn dies. This wrapper retries the request + * against the next model in the chain, rewriting only the `model` field of the body. + * + * Retry happens **before** any bytes stream: only a non-2xx response is retried, so a + * stream that already started (and was already billed) is never replayed. + * + * The downgrade is reported to the caller, never applied silently. + */ +import type { SupportedChatModelId } from "@mocode/shared"; +import { HttpError } from "./http-errors"; +import { runWithFailover, type FailoverDecision } from "./provider-chain"; + +export type DowngradeInfo = { + /** Model the user asked for. */ + from: string; + /** Model that actually answered. */ + to: string; + failures: Array<{ modelId: string; detail: string }>; +}; + +export type FailoverFetchOptions = { + /** Injectable for tests. */ + fetchImpl?: typeof globalThis.fetch; + /** Reads the `{ error }` envelope off a failed response. */ + readError: (response: Response) => Promise; + fallbacks?: readonly SupportedChatModelId[]; + /** Budget for connect + headers. Does NOT bound generation. */ + connectTimeoutMs?: number; + /** @deprecated alias for {@link connectTimeoutMs}. */ + attemptTimeoutMs?: number; + /** Silence budget while reading; reset on every chunk. */ + stallTimeoutMs?: number; + /** Attempts per model; defaults to `MOCODE_PROVIDER_RETRIES` or 3. */ + attemptsPerModel?: number; + /** Injectable for tests so they do not wait on real backoff. */ + sleep?: (ms: number) => Promise; + /** Explicit opt-in to trying another provider; see `FailoverOptions.consent`. */ + allowFailover?: boolean; + onDowngrade?: (info: DowngradeInfo) => void; + onAttemptFailed?: (event: { modelId: string; decision: FailoverDecision }) => void; +}; + +/** + * Budget for *reaching the stream* — DNS, TCP, TLS, HTTP headers. + * + * This bounds the network, not the model. It is cleared the moment headers arrive, + * because everything after that is generation and generation is allowed to be slow. + */ +export function connectTimeoutMs(): number { + const configured = Number(process.env.MOCODE_CONNECT_TIMEOUT_MS ?? process.env.MOCODE_ATTEMPT_TIMEOUT_MS); + return Number.isFinite(configured) && configured > 0 ? configured : 20_000; +} + +/** + * Silence budget while reading the stream. + * + * Reset on **every chunk**. A provider that keeps emitting bytes is healthy no + * matter how long the whole turn takes; only a provider that goes quiet is a + * stall. This is the primitive Claude Code / Codex-style clients use, and it is + * why a long code generation never trips a timeout. + */ +export function stallTimeoutMs(): number { + const configured = Number(process.env.MOCODE_STALL_TIMEOUT_MS); + return Number.isFinite(configured) && configured > 0 ? configured : 120_000; +} + +/** + * How many chunks to inspect before declaring the stream healthy. + * + * The server answers `200` and puts the upstream failure *inside* the SSE body, so + * `response.ok` says nothing about whether the provider worked. Peeking is the only + * way to fail over without changing the server's streaming contract. + */ +const PEEK_MAX_CHUNKS = 16; + +/** + * Events that prove the provider produced something. + * + * Deliberately excludes `text-start` / `tool-input-start`: those arrive *before* + * any content, so treating them as success would suppress failover for a provider + * that opens a block and then fails. + */ +const HEALTHY_EVENT_TYPES = new Set([ + "text-delta", + "reasoning-delta", + "tool-input-available", + "tool-call", + "finish", + "finish-step", +]); + +type StreamInspection = { error: string | null; response: Response }; + +/** Extract `data: {...}` payloads from a partial SSE buffer. */ +function completeDataLines(buffer: string): string[] { + return buffer + .split("\n") + .filter((line) => line.startsWith("data:")) + .map((line) => line.slice("data:".length).trim()) + .filter((line) => line.length > 0 && line !== "[DONE]"); +} + +/** + * Read just enough of the SSE body to tell "the provider worked" from + * "the provider rejected us", then hand back an equivalent response. + * + * Buffered bytes are replayed into the returned stream, so nothing is lost. + */ +type InspectionOptions = { + /** Silence budget; reset on every chunk. */ + stallMs: number; + /** Cancels the underlying response when the stall budget is exceeded. */ + abort: () => void; +}; + +async function inspectStream( + response: Response, + inspection: InspectionOptions, +): Promise { + if (!response.body) return { error: null, response }; + + const reader = response.body.getReader(); + let stallTimer = setTimeout(inspection.abort, inspection.stallMs); + const resetStall = () => { + clearTimeout(stallTimer); + stallTimer = setTimeout(inspection.abort, inspection.stallMs); + }; + const buffered: Uint8Array[] = []; + const decoder = new TextDecoder(); + let text = ""; + let finished = false; + let sawOutput = false; + + try { + for (let read = 0; read < PEEK_MAX_CHUNKS; read += 1) { + const { done, value } = await reader.read(); + // Any bytes at all mean the provider is alive, however slow the turn is. + resetStall(); + if (done) { + finished = true; + break; + } + + buffered.push(value); + text += decoder.decode(value, { stream: true }); + + for (const payload of completeDataLines(text)) { + let parsed: { type?: unknown; errorText?: unknown; error?: unknown }; + try { + parsed = JSON.parse(payload); + } catch { + continue; + } + + if (parsed.type === "error") { + await reader.cancel().catch(() => {}); + const detail = + typeof parsed.errorText === "string" + ? parsed.errorText + : typeof parsed.error === "string" + ? parsed.error + : "upstream provider error"; + return { error: detail, response }; + } + + if (typeof parsed.type === "string" && HEALTHY_EVENT_TYPES.has(parsed.type)) { + sawOutput = true; + read = PEEK_MAX_CHUNKS; // healthy — stop peeking + break; + } + } + + // Only the newest line can be incomplete; drop it before re-checking next round. + text = text.slice(text.lastIndexOf("\n") + 1); + } + + } finally { + clearTimeout(stallTimer); + } + + // EOF before any real output is not success — but only for SSE bodies. A plain + // JSON/text response is not a stream and must pass through untouched. + const isSse = response.headers.get("content-type")?.includes("text/event-stream") ?? false; + if (finished && isSse && !sawOutput) { + return { error: "stream ended before any output", response }; + } + + const headers = new Headers(response.headers); + headers.delete("content-length"); + headers.delete("content-encoding"); + + const stream = new ReadableStream({ + start(controller) { + for (const chunk of buffered) controller.enqueue(chunk); + + if (finished) { + controller.close(); + return; + } + + void (async () => { + try { + while (true) { + const { done, value } = await reader.read(); + if (done) { + controller.close(); + return; + } + controller.enqueue(value); + } + } catch (cause) { + controller.error(cause); + } + })(); + }, + cancel(reason) { + return reader.cancel(reason); + }, + }); + + return { + error: null, + response: new Response(stream, { + status: response.status, + statusText: response.statusText, + headers, + }), + }; +} + +/** Combine the caller's abort signal with our own cancellable deadline. */ +function attemptSignal( + userSignal: AbortSignal | null | undefined, + deadline: AbortSignal, +): AbortSignal { + return userSignal ? AbortSignal.any([userSignal, deadline]) : deadline; +} + +/** Error shape `classifyFailover` treats as a retryable timeout. */ +function timeoutError(): Error { + const error = new Error("attempt timed out before the stream started"); + error.name = "TimeoutError"; + return error; +} + +/** + * Wrap `fetch` so a retryable failure transparently retries on the next model. + * + * Bodies that are not a JSON object with a `model` field pass straight through — + * this wrapper only knows how to fail over chat requests. + */ +export function createFailoverFetch( + options: FailoverFetchOptions, +): typeof globalThis.fetch { + const fetchImpl = options.fetchImpl ?? globalThis.fetch; + const connectMs = options.connectTimeoutMs ?? options.attemptTimeoutMs ?? connectTimeoutMs(); + const stallMs = options.stallTimeoutMs ?? stallTimeoutMs(); + + return (async ( + input: Parameters[0], + init?: Parameters[1], + ) => { + const body = typeof init?.body === "string" ? init.body : null; + if (!body) return fetchImpl(input, init); + + let payload: { model?: unknown }; + try { + payload = JSON.parse(body) as { model?: unknown }; + } catch { + return fetchImpl(input, init); + } + + const primary = typeof payload.model === "string" ? payload.model : null; + if (!primary) return fetchImpl(input, init); + + const outcome = await runWithFailover({ + primary, + fallbacks: options.fallbacks, + consent: options.allowFailover === true, + attemptsPerModel: options.attemptsPerModel, + sleep: options.sleep, + attempt: async (modelId) => { + const attemptBody = + modelId === primary ? body : JSON.stringify({ ...payload, model: modelId }); + + // Phase 1: connect + headers. Bounded, because a dead endpoint should fail + // fast instead of hanging the turn. + const controller = new AbortController(); + const connectDeadline = setTimeout(() => controller.abort(), connectMs); + + try { + const response = await fetchImpl(input, { + ...init, + body: attemptBody, + signal: attemptSignal(init?.signal, controller.signal), + }); + + // Headers arrived: the network is fine. From here on only *silence* is a + // failure, and `inspectStream` resets its timer on every chunk. + clearTimeout(connectDeadline); + + if (!response.ok) { + throw new HttpError(response.status, await options.readError(response)); + } + + // 200 does not mean the provider worked: the server streams upstream + // failures as an `error` event inside the body. Fail over on those too. + const inspected = await inspectStream(response, { + stallMs, + abort: () => controller.abort(), + }); + if (inspected.error) { + throw new HttpError(502, inspected.error); + } + + return inspected.response; + } catch (error) { + // Our deadline must be retryable; a user abort must never be retried — + // even when the underlying throw looks like a retryable HttpError. + if (init?.signal?.aborted) { + const abort = new Error("The operation was aborted"); + abort.name = "AbortError"; + throw abort; + } + if (controller.signal.aborted) throw timeoutError(); + throw error; + } finally { + clearTimeout(connectDeadline); + } + }, + onAttemptFailed: options.onAttemptFailed, + }); + + if (outcome.downgradedFrom) { + options.onDowngrade?.({ + from: outcome.downgradedFrom, + to: outcome.modelId, + failures: outcome.failures, + }); + } + + return outcome.value; + }) as typeof globalThis.fetch; +} diff --git a/packages/cli/src/lib/http-errors.ts b/packages/cli/src/lib/http-errors.ts index 67dc3d5..ed91500 100644 --- a/packages/cli/src/lib/http-errors.ts +++ b/packages/cli/src/lib/http-errors.ts @@ -1,9 +1,20 @@ type ErrorResponse = { - json: () => Promise; - status: number; - statusText: string; + json: () => Promise; + status: number; + statusText: string; }; +/** HTTP failure that keeps the status code so callers can act on it. */ +export class HttpError extends Error { + constructor( + readonly statusCode: number, + message: string, + ) { + super(message); + this.name = "HttpError"; + } +} + /** Extract a human-readable message from the server's `{ error }` JSON envelope. */ export async function getErrorMessage(response: ErrorResponse) { try { @@ -16,4 +27,4 @@ export async function getErrorMessage(response: ErrorResponse) { } return response.statusText || `Request failed with status ${response.status}`; - }; \ No newline at end of file + } diff --git a/packages/cli/src/lib/provider-chain.test.ts b/packages/cli/src/lib/provider-chain.test.ts new file mode 100644 index 0000000..9dec710 --- /dev/null +++ b/packages/cli/src/lib/provider-chain.test.ts @@ -0,0 +1,293 @@ +import { describe, expect, test } from "bun:test"; +import { + DEFAULT_FALLBACK_CHAIN, + retryDelayMs, + buildModelChain, + classifyFailover, + runWithFailover, +} from "./provider-chain"; + +/** Error carrying an HTTP status, shaped like the providers' failures. */ +function httpError(status: number): Error & { statusCode: number } { + const error = new Error(`upstream failed with ${status}`) as Error & { statusCode: number }; + error.statusCode = status; + return error; +} + +function namedError(name: string, message = name): Error { + const error = new Error(message); + error.name = name; + return error; +} + +describe("buildModelChain", () => { + test("puts the primary first and appends fallbacks", () => { + expect(buildModelChain("claude-opus-4-6")).toEqual(["claude-opus-4-6", "deepseek-flash"]); + }); + + test("drops the primary when it also appears in the fallback chain", () => { + expect(buildModelChain("deepseek-flash")).toEqual(["deepseek-flash"]); + }); + + test("never retries the same model twice", () => { + const chain = buildModelChain("a", ["a", "b", "a", "c"]); + expect(chain).toEqual(["a", "b", "c"]); + expect(new Set(chain).size).toBe(chain.length); + }); + + test("empty primary is ignored rather than attempted", () => { + expect(buildModelChain("", ["b"])).toEqual(["b"]); + }); +}); + +describe("classifyFailover", () => { + test("retries region blocks and auth failures — another provider may work", () => { + for (const status of [401, 403, 429, 502, 503]) { + const decision = classifyFailover(httpError(status)); + expect(decision.retryable).toBe(true); + expect(decision.kind).toBe("http"); + expect(decision.detail).toBe(`HTTP ${status}`); + } + }); + + test("does NOT retry a 400 — that is our own malformed request", () => { + expect(classifyFailover(httpError(400)).retryable).toBe(false); + }); + + test("does NOT retry a user abort — Esc must stay cancelled", () => { + const decision = classifyFailover(namedError("AbortError")); + expect(decision).toMatchObject({ retryable: false, retrySameProvider: false, kind: "abort" }); + }); + + test("retries a timeout — a hung provider should not end the turn", () => { + expect(classifyFailover(namedError("TimeoutError")).retryable).toBe(true); + }); + + test("retries connection-level failures", () => { + expect(classifyFailover(new Error("fetch failed")).kind).toBe("network"); + expect(classifyFailover(new Error("connect ECONNREFUSED 127.0.0.1:443")).retryable).toBe(true); + }); + + test("unknown errors are not retried", () => { + expect(classifyFailover(new Error("boom")).retryable).toBe(false); + }); + + test("reads status from a fetch-style response object", () => { + expect(classifyFailover({ response: { status: 403 } }).detail).toBe("HTTP 403"); + }); +}); + +describe("runWithFailover", () => { + test("returns the primary result without a downgrade when it works", async () => { + const outcome = await runWithFailover({ + primary: "deepseek-flash", + consent: true, + attempt: async (modelId) => `answer from ${modelId}`, + }); + + expect(outcome.value).toBe("answer from deepseek-flash"); + expect(outcome.modelId).toBe("deepseek-flash"); + expect(outcome.downgradedFrom).toBeNull(); + expect(outcome.failures).toEqual([]); + }); + + test("falls through a blocked provider and reports the downgrade", async () => { + const outcome = await runWithFailover({ + primary: "gpt-oss-120b", + consent: true, + attempt: async (modelId) => { + if (modelId === "gpt-oss-120b") throw httpError(403); + return `answer from ${modelId}`; + }, + }); + + expect(outcome.value).toBe("answer from deepseek-flash"); + expect(outcome.downgradedFrom).toBe("gpt-oss-120b"); + expect(outcome.failures).toEqual([{ modelId: "gpt-oss-120b", detail: "HTTP 403" }]); + }); + + test("tries every candidate before giving up, then rethrows the last error", async () => { + const tried: string[] = []; + + await expect( + runWithFailover({ + primary: "a", + consent: true, + fallbacks: ["b", "c"], + attemptsPerModel: 1, + attempt: async (modelId) => { + tried.push(modelId); + throw httpError(503); + }, + }), + ).rejects.toThrow("upstream failed with 503"); + + expect(tried).toEqual(["a", "b", "c"]); + }); + + test("stops immediately on a non-retryable failure", async () => { + const tried: string[] = []; + + await expect( + runWithFailover({ + primary: "a", + consent: true, + fallbacks: ["b"], + attempt: async (modelId) => { + tried.push(modelId); + throw httpError(400); + }, + }), + ).rejects.toThrow("upstream failed with 400"); + + expect(tried).toEqual(["a"]); + }); + + test("does not burn the chain when the user aborts", async () => { + const tried: string[] = []; + + await expect( + runWithFailover({ + primary: "a", + consent: true, + fallbacks: ["b"], + attempt: async (modelId) => { + tried.push(modelId); + throw namedError("AbortError"); + }, + }), + ).rejects.toThrow(); + + expect(tried).toEqual(["a"]); + }); + + test("reports every failed attempt to the caller", async () => { + const events: string[] = []; + + await runWithFailover({ + primary: "a", + consent: true, + fallbacks: ["b", "c"], + attemptsPerModel: 1, + attempt: async (modelId) => { + if (modelId !== "c") throw httpError(403); + return "ok"; + }, + onAttemptFailed: ({ modelId, decision }) => events.push(`${modelId}:${decision.detail}`), + }); + + expect(events).toEqual(["a:HTTP 403", "b:HTTP 403"]); + }); + + test("default chain starts with the reachable provider", () => { + expect(DEFAULT_FALLBACK_CHAIN[0]).toBe("deepseek-flash"); + }); + + test("the 4x-cost model is NOT a default fallback", () => { + // Silently escalating someone's spend is not this code's call. + expect(DEFAULT_FALLBACK_CHAIN).not.toContain("deepseek-v4-pro"); + }); + + test("failover requires explicit consent", async () => { + const tried: string[] = []; + + await expect( + runWithFailover({ + primary: "gpt-oss-120b", + consent: false, + attemptsPerModel: 1, + attempt: async (modelId) => { + tried.push(modelId); + throw httpError(403); + }, + }), + ).rejects.toThrow(); + + expect(tried).toEqual(["gpt-oss-120b"]); + }); +}); + +describe("retries keep a valid key sufficient", () => { + const noWait = async () => {}; + + test("a transient failure is retried and the turn completes", async () => { + let calls = 0; + + const outcome = await runWithFailover({ + primary: "deepseek-flash", + consent: false, + attemptsPerModel: 3, + sleep: noWait, + attempt: async (modelId) => { + calls += 1; + if (calls < 3) throw httpError(429); + return `ok from ${modelId}`; + }, + }); + + expect(outcome.value).toBe("ok from deepseek-flash"); + expect(calls).toBe(3); + }); + + test("exhausting retries surfaces the last error", async () => { + let calls = 0; + + await expect( + runWithFailover({ + primary: "deepseek-flash", + consent: false, + attemptsPerModel: 2, + sleep: noWait, + attempt: async () => { + calls += 1; + throw httpError(503); + }, + }), + ).rejects.toThrow("upstream failed with 503"); + + expect(calls).toBe(2); + }); + + test("a non-retryable error is not retried", async () => { + let calls = 0; + + await expect( + runWithFailover({ + primary: "deepseek-flash", + consent: false, + attemptsPerModel: 3, + sleep: noWait, + attempt: async () => { + calls += 1; + throw httpError(400); + }, + }), + ).rejects.toThrow(); + + expect(calls).toBe(1); + }); + + test("A timeout is retried — a stalled stream is not the user's fault", async () => { + let calls = 0; + + await runWithFailover({ + primary: "deepseek-flash", + consent: false, + attemptsPerModel: 2, + sleep: noWait, + attempt: async () => { + calls += 1; + if (calls === 1) throw namedError("TimeoutError"); + return "recovered"; + }, + }); + + expect(calls).toBe(2); + }); + + test("backoff grows and stays bounded", () => { + expect(retryDelayMs(1, 500)).toBeLessThan(1_001); + expect(retryDelayMs(2, 500)).toBeLessThan(1_501); + expect(retryDelayMs(10, 500)).toBeLessThanOrEqual(8_500); + }); +}); diff --git a/packages/cli/src/lib/provider-chain.ts b/packages/cli/src/lib/provider-chain.ts new file mode 100644 index 0000000..ad3af7f --- /dev/null +++ b/packages/cli/src/lib/provider-chain.ts @@ -0,0 +1,285 @@ +/** + * Model failover — retry a turn on the next model when the current provider fails. + * + * Why: `resolveChatModel` hard-binds one provider per model. On a network where that + * provider is region-blocked, 13 of 15 catalog models were unusable (see + * `packages/server/scripts/provider-failover-bench.ts`). A single unreachable + * provider meant 100% of turns failed with a bare `Forbidden`. + * + * This module is pure orchestration — no I/O, no React. Callers supply an `attempt` + * thunk; the module decides whether a failure is worth retrying, walks the chain, + * and reports which model actually answered so the downgrade can be shown to the user. + * + * A downgrade is never silent: `downgradedFrom` is non-null whenever a fallback + * answered, and callers must surface it. + */ +import type { SupportedChatModelId } from "@mocode/shared"; + +/** + * Ordered fallback chain. + * + * `deepseek-v4-pro` is deliberately NOT here: it costs roughly four times + * `deepseek-flash`, and silently escalating a user's spend because the cheap model + * was rate-limited is not a decision this code gets to make. Opt in explicitly via + * `MOCODE_FALLBACK_CHAIN` when the extra cost is acceptable. + */ +export const DEFAULT_FALLBACK_CHAIN: readonly SupportedChatModelId[] = ["deepseek-flash"]; + +/** + * Whether the operator has authorised cross-provider failover. + * + * Off unless explicitly enabled: retrying on another vendor sends the same code to + * a different company, which needs a decision, not a default. + */ +export function failoverConsentFromEnv(): boolean { + return process.env.MOCODE_ALLOW_MODEL_FAILOVER === "1"; +} + +/** Fallback chain, overridable by the operator. */ +export function fallbackChainFromEnv(): readonly string[] { + const configured = process.env.MOCODE_FALLBACK_CHAIN?.split(",") + .map((id) => id.trim()) + .filter(Boolean); + + return configured && configured.length > 0 ? configured : DEFAULT_FALLBACK_CHAIN; +} + +/** HTTP statuses that mean "this provider cannot serve us — try another". */ +const RETRYABLE_STATUS = new Set([401, 403, 404, 408, 429, 500, 502, 503, 504]); + +/** Subset that is worth retrying against the *same* provider. */ +const SAME_PROVIDER_RETRY_STATUS = new Set([408, 429, 500, 502, 503, 504]); + +export type FailoverDecision = { + /** Worth trying *somewhere* — another provider or a later attempt. */ + retryable: boolean; + /** + * Worth retrying against the **same** provider. + * + * A 403/401/404 will answer identically in 500ms; retrying it only delays the + * fallback and burns the user's patience. 429/5xx/timeouts are transient and do + * deserve a backed-off retry. + */ + retrySameProvider: boolean; + /** Short machine-readable bucket, used in logs and tests. */ + kind: "http" | "timeout" | "abort" | "network" | "unknown"; + detail: string; +}; + +/** + * Build the attempt order for a primary model: primary first, then the fallbacks, + * de-duplicated. A fallback equal to the primary is dropped rather than retried + * against the same broken provider. + */ +export function buildModelChain( + primary: string, + fallbacks: readonly string[] = DEFAULT_FALLBACK_CHAIN, +): string[] { + const seen = new Set(); + const chain: string[] = []; + + for (const id of [primary, ...fallbacks]) { + if (!id || seen.has(id)) continue; + seen.add(id); + chain.push(id); + } + + return chain; +} + +/** Read a numeric HTTP status off an SDK/`fetch` error, if there is one. */ +function statusOf(error: unknown): number | null { + if (typeof error !== "object" || error === null) return null; + + const candidate = error as { + statusCode?: unknown; + status?: unknown; + response?: { status?: unknown }; + }; + + for (const value of [candidate.statusCode, candidate.status, candidate.response?.status]) { + if (typeof value === "number") return value; + } + + return null; +} + +/** + * Decide whether a failed attempt is worth retrying on another model. + * + * Deliberately conservative: a 400 means *we* built a bad request, and a user abort + * (Esc) must never be retried — retrying either would mask the real problem. + */ +export function classifyFailover(error: unknown): FailoverDecision { + const status = statusOf(error); + if (status !== null) { + return { + retryable: RETRYABLE_STATUS.has(status), + // Auth/URL rejections are not transient; rate limits and server faults are. + retrySameProvider: SAME_PROVIDER_RETRY_STATUS.has(status), + kind: "http", + detail: `HTTP ${status}`, + }; + } + + const name = error instanceof Error ? error.name : ""; + + if (name === "AbortError") { + return { retryable: false, retrySameProvider: false, kind: "abort", detail: "aborted by caller" }; + } + + if (name === "TimeoutError") { + return { retryable: true, retrySameProvider: true, kind: "timeout", detail: "attempt timed out" }; + } + + const message = error instanceof Error ? error.message : String(error); + + if (/fetch failed|ECONNREFUSED|ENOTFOUND|ETIMEDOUT|socket hang up|network/i.test(message)) { + return { retryable: true, retrySameProvider: true, kind: "network", detail: message.slice(0, 80) }; + } + + return { retryable: false, retrySameProvider: false, kind: "unknown", detail: message.slice(0, 80) }; +} + +/** What the caller must surface: which model answered, and what it replaced. */ +export type FailoverOutcome = { + value: T; + /** Model that actually produced the result. */ + modelId: string; + /** Non-null when a fallback answered — the UI must show this. */ + downgradedFrom: string | null; + /** Failures that happened before the successful attempt. */ + failures: Array<{ modelId: string; detail: string }>; +}; + +/** Attempts per model before giving up on it. Retries are what make a valid key + * sufficient: a transient 429 or stalled connection should not end the turn. */ +export function attemptsPerModelFromEnv(): number { + const configured = Number(process.env.MOCODE_PROVIDER_RETRIES); + return Number.isFinite(configured) && configured >= 1 ? Math.floor(configured) : 3; +} + +/** Exponential backoff with jitter: 500ms, 1s, 2s … capped at 8s. */ +export function retryDelayMs(attempt: number, baseMs = 500): number { + const exponential = Math.min(baseMs * 2 ** (attempt - 1), 8_000); + return exponential + Math.floor(Math.random() * baseMs); +} + +export type FailoverOptions = { + primary: string; + fallbacks?: readonly string[]; + /** Attempts per model. Defaults to `MOCODE_PROVIDER_RETRIES` or 3. */ + attemptsPerModel?: number; + /** Base backoff delay. */ + retryBaseMs?: number; + /** Injectable for tests so they do not actually wait. */ + sleep?: (ms: number) => Promise; + /** + * Explicit authorisation to send the turn to a *different* provider. + * + * Required, with no default: switching provider means the user's code leaves for + * a vendor they did not pick. Without consent only the selected model is tried + * and its failure surfaces unchanged. + */ + consent: boolean; + /** One attempt against one model. Throw to signal failure. */ + attempt: (modelId: string) => Promise; + /** Called for every failure so callers can log it. */ + onAttemptFailed?: (event: { modelId: string; decision: FailoverDecision }) => void; +}; + +/** + * Run `attempt` down the chain until one succeeds. + * + * Re-throws the *last* error when every candidate fails, so the original cause is + * not swallowed. Non-retryable failures (bad request, user abort) throw immediately + * without burning fallbacks. + */ +export async function runWithFailover({ + primary, + fallbacks = fallbackChainFromEnv(), + consent, + attemptsPerModel = attemptsPerModelFromEnv(), + retryBaseMs = 500, + sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + attempt, + onAttemptFailed, +}: FailoverOptions): Promise> { + if (!consent) { + // No authorisation to switch providers — but retrying the *selected* provider + // is not a privacy decision, and a transient failure must not end the turn. + const value = await attemptWithRetries(primary, attempt, { + attemptsPerModel, + retryBaseMs, + sleep, + onAttemptFailed, + }); + return { value, modelId: primary, downgradedFrom: null, failures: [] }; + } + + const chain = buildModelChain(primary, fallbacks); + const failures: FailoverOutcome["failures"] = []; + let lastError: unknown; + + for (const modelId of chain) { + try { + const value = await attemptWithRetries(modelId, attempt, { + attemptsPerModel, + retryBaseMs, + sleep, + onAttemptFailed, + }); + + return { + value, + modelId, + downgradedFrom: modelId === primary ? null : primary, + failures, + }; + } catch (error) { + const decision = classifyFailover(error); + failures.push({ modelId, detail: decision.detail }); + if (!decision.retryable) throw error; + lastError = error; + } + } + + throw lastError ?? new Error("No model candidates available"); +} + +type RetryContext = { + attemptsPerModel: number; + retryBaseMs: number; + sleep: (ms: number) => Promise; + onAttemptFailed?: FailoverOptions["onAttemptFailed"]; +}; + +/** One model, up to `attemptsPerModel` tries with backoff. Non-retryable throws at once. */ +async function attemptWithRetries( + modelId: string, + attempt: (modelId: string) => Promise, + context: RetryContext, +): Promise { + let lastError: unknown; + + for (let attemptNo = 1; attemptNo <= context.attemptsPerModel; attemptNo += 1) { + try { + return await attempt(modelId); + } catch (error) { + const decision = classifyFailover(error); + context.onAttemptFailed?.({ modelId, decision }); + + if (!decision.retryable) throw error; + lastError = error; + + // A rejection that will repeat verbatim is not worth waiting for. + if (!decision.retrySameProvider) throw error; + + if (attemptNo < context.attemptsPerModel) { + await context.sleep(retryDelayMs(attemptNo, context.retryBaseMs)); + } + } + } + + throw lastError ?? new Error(`All ${context.attemptsPerModel} attempts failed for ${modelId}`); +} diff --git a/packages/server/scripts/provider-failover-bench.ts b/packages/server/scripts/provider-failover-bench.ts new file mode 100644 index 0000000..51dfbd1 --- /dev/null +++ b/packages/server/scripts/provider-failover-bench.ts @@ -0,0 +1,216 @@ +/** + * Provider availability + failover bench. + * + * Why this exists: `resolveChatModel` hard-binds one provider per model. When that + * provider is unreachable the whole agent turn dies with a bare `403 Forbidden`, + * which is what actually happened on a mainland-China network. + * + * The fault source is real, not mocked: Cerebras / Groq / Anthropic answer 403 + * behind a country block, and the OpenAI / Google endpoints do not connect at all. + * That makes them a free, reproducible fault injector — every failure below is a + * genuine upstream failure, which is the whole point. + * + * It measures two policies against the same probe results: + * baseline — one model, one provider, no fallback + * failover — if the primary fails, walk FALLBACK_CHAIN + * + * Run it from the repo root so `.env` is picked up: + * bun run packages/server/scripts/provider-failover-bench.ts + * + * Note: this makes real API calls. The reachable providers cost a few tokens each. + */ +import { generateText, tool } from "ai"; +import { z } from "zod"; +import { SUPPORTED_CHAT_MODELS } from "@mocode/shared"; +import { resolveChatModel } from "../src/lib/model"; + +/** Per-model budget. Must exceed normal latency but stay under user patience. */ +const TIMEOUT_MS = 12_000; + +/** + * The failover policy under test. Ordered, cheapest-capable-first. + * Today no such chain exists in the code — this script measures what adding it + * would buy, using the probe results we already paid for (no extra requests). + */ +const FALLBACK_CHAIN = ["deepseek-flash", "deepseek-v4-pro"] as const; + +/** Minimal tool so the probe also proves the provider can emit tool calls. */ +const pingTool = tool({ + description: "List the entries of a directory.", + inputSchema: z.object({ path: z.string() }), + execute: async ({ path }) => ({ path, entries: ["a.ts"] }), +}); + +type Probe = { + modelId: string; + provider: string; + ok: boolean; + ms: number; + detail: string; +}; + +/** Collapse SDK errors into something a table can show. */ +function classify(error: unknown): string { + if (error instanceof Error && (error.name === "TimeoutError" || error.name === "AbortError")) { + return "timeout"; + } + + const status = (error as { statusCode?: number }).statusCode; + if (typeof status === "number") return `HTTP ${status}`; + + const message = error instanceof Error ? error.message : String(error); + if (/is not set/.test(message)) return "no-key"; + return message.replace(/\s+/g, " ").slice(0, 58); +} + +async function probe(modelId: string, provider: string): Promise { + const started = Date.now(); + + try { + const resolved = resolveChatModel(modelId); + const result = await generateText({ + model: resolved.model, + messages: [{ role: "user", content: "List the current directory using the ping tool." }], + tools: { ping: pingTool }, + maxOutputTokens: 64, + abortSignal: AbortSignal.timeout(TIMEOUT_MS), + }); + + // Answering without calling the tool is not a working agent turn — a model + // that chats instead of acting would otherwise score as healthy. + const calledPing = result.toolCalls.some((call) => call.toolName === "ping"); + + return { + modelId, + provider, + ok: calledPing, + ms: Date.now() - started, + detail: calledPing ? "tool_calls=1" : `no ping call (tool_calls=${result.toolCalls.length})`, + }; + } catch (error) { + return { modelId, provider, ok: false, ms: Date.now() - started, detail: classify(error) }; + } +} + +function pct(part: number, total: number): string { + return `${part}/${total} (${Math.round((part / total) * 100)}%)`; +} + +const rows: Probe[] = []; + +for (const model of SUPPORTED_CHAT_MODELS) { + process.stdout.write(` probing ${model.id} … `); + const row = await probe(model.id, model.provider); + rows.push(row); + console.log(row.ok ? `ok ${(row.ms / 1000).toFixed(1)}s` : `FAIL ${row.detail}`); +} + +const byId = new Map(rows.map((row) => [row.modelId, row])); + +console.log("\n=== 逐模型探测(真实请求;失败即为上游真实故障) ==="); +console.log("model".padEnd(30), "provider".padEnd(12), "result"); +for (const row of rows) { + const result = row.ok + ? `ok ${(row.ms / 1000).toFixed(1)}s ${row.detail}` + : `FAIL ${(row.ms / 1000).toFixed(1)}s ${row.detail}`; + console.log(row.modelId.padEnd(30), row.provider.padEnd(12), result); +} + +const baselineOk = rows.filter((row) => row.ok).length; +const failoverOk = rows.filter( + (row) => row.ok || FALLBACK_CHAIN.some((id) => byId.get(id)?.ok), +).length; + +console.log("\n=== 结论 ==="); +console.log("无回退(单 provider 硬绑):", pct(baselineOk, rows.length)); +console.log("带回退(失败后走", FALLBACK_CHAIN.join(" → ") + "):", pct(failoverOk, rows.length)); + +const failures = rows.filter((row) => !row.ok); +const fastFail = failures.filter((row) => row.ms < 5_000); +const slowFail = failures.filter((row) => row.ms >= 5_000); + +console.log("\n=== 失败耗时分布(决定回退策略要不要设超时) ==="); +console.log("快速失败 <5s(如 403):", fastFail.length, "→", fastFail.map((r) => r.detail).join(", ") || "-"); +console.log("慢失败 ≥5s(连接挂起):", slowFail.length, "→", slowFail.map((r) => r.detail).join(", ") || "-"); + +if (slowFail.length > 0) { + console.log( + `\n注意:有 ${slowFail.length} 个模型是"挂到超时"才失败的。若回退链不设 per-attempt 超时,` + + `一次请求会被最慢的那个 provider 拖到 ${TIMEOUT_MS / 1000}s 之后。`, + ); +} + +// --------------------------------------------------------------------------- +// Phase 2 — end-to-end failover, real requests. +// +// Phase 1 above only *simulates* the policy by re-reading probe results. This +// phase actually walks the chain with real calls, so the wall-clock number +// includes the failed attempts a user would wait through. +// +// The policy below mirrors `packages/cli/src/lib/provider-chain.ts` (same +// retryable-status set). It is duplicated here on purpose: the server package may +// not import the CLI package, and a script must not be the thing that defines +// shipped behaviour. +// --------------------------------------------------------------------------- + +const RETRYABLE = new Set([401, 403, 404, 408, 429, 500, 502, 503, 504]); + +/** Three representative primaries: fast-fail, region-block, already-working. */ +const E2E_SAMPLE = ["claude-sonnet-4-6", "gpt-oss-120b", "deepseek-flash"] as const; + +function retryable(error: unknown): boolean { + const status = (error as { statusCode?: number }).statusCode; + if (typeof status === "number") return RETRYABLE.has(status); + const name = error instanceof Error ? error.name : ""; + if (name === "AbortError") return false; // user cancel must never retry + return name === "TimeoutError"; +} + +async function e2eFailover(primary: string) { + const chain = [...new Set([primary, ...FALLBACK_CHAIN])]; + const started = Date.now(); + const failures: string[] = []; + + for (const modelId of chain) { + try { + const resolved = resolveChatModel(modelId); + await generateText({ + model: resolved.model, + messages: [{ role: "user", content: "List the current directory using the ping tool." }], + tools: { ping: pingTool }, + maxOutputTokens: 64, + abortSignal: AbortSignal.timeout(TIMEOUT_MS), + }); + + return { + primary, + answeredBy: modelId, + downgraded: modelId !== primary, + totalMs: Date.now() - started, + failures, + }; + } catch (error) { + failures.push(`${modelId}: ${classify(error)}`); + if (!retryable(error)) break; + } + } + + return { + primary, + answeredBy: null, + downgraded: false, + totalMs: Date.now() - started, + failures, + }; +} + +console.log("\n=== 端到端回退(真实请求,含失败的等待时间) ==="); +for (const primary of E2E_SAMPLE) { + const result = await e2eFailover(primary); + const verdict = result.answeredBy + ? `→ ${result.answeredBy}${result.downgraded ? " (downgraded)" : ""} ${(result.totalMs / 1000).toFixed(1)}s` + : `FAILED ${(result.totalMs / 1000).toFixed(1)}s`; + console.log(primary.padEnd(30), verdict); + for (const failure of result.failures) console.log("".padEnd(30), ` ✗ ${failure}`); +} + From c14ea048d3703ce3d0a6f94e07890ceb3fbc60c4 Mon Sep 17 00:00:00 2001 From: moyun Date: Thu, 12 Nov 2026 09:35:32 +0800 Subject: [PATCH 04/11] feat(cli): permission model, OS sandbox and audit trail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Manual acceptance caught the agent reading /etc/passwd through `bash head -3 secret.txt`: path checks guard the file tools, and a path check cannot contain a shell. Layers, outside in: - **OS sandbox** — seatbelt (`deny default` + explicit read/write sets, with credential trees denied) or bwrap (read-only root, writable project, sensitive paths masked). Applied at spawn; capability is probed at runtime and `available` is tracked separately from `enforced`. - **executor gate** — a single authorized entry point; every call, allowed or refused, appends one JSONL audit line - **path containment** — lexical + realpath, which also rejects dangling symlinks (walking past one would accept a write that escapes) - **hard denies** — refused even after the user approves Details that matter: - refusal codes (`PATH_ESCAPE` / `HARD_DENY` / `MODE_DENIED` / `UNKNOWN_TOOL`) are what audits group by; prose drifts - audit records the command but masks `sk-*`, `Bearer`, `*_API_KEY=` and `X-API-Key:`; file bodies are dropped, never truncated - logs rotate at 5MB × 5 so disk use is bounded - `2>/dev/null` no longer matches the device-redirect rule: false positives train users to approve blindly, which destroys the control - an unconfined shell requires approval for **every** command Includes `acceptance.ts`, `sandbox-doctor.ts` and `sandbox-bisect.ts`, which is how the SIGABRT / false-permission-path issues were found. --- packages/cli/scripts/acceptance.ts | 365 ++++++++++++++++++ packages/cli/scripts/sandbox-bisect.ts | 105 +++++ packages/cli/scripts/sandbox-doctor.ts | 168 ++++++++ .../cli/src/lib/audit-log.rotation.test.ts | 121 ++++++ packages/cli/src/lib/audit-log.test.ts | 151 ++++++++ packages/cli/src/lib/audit-log.ts | 247 ++++++++++++ .../cli/src/lib/local-tools.audit.test.ts | 128 ++++++ .../cli/src/lib/local-tools.readline.test.ts | 56 +++ .../cli/src/lib/local-tools.sandbox.test.ts | 132 +++++++ packages/cli/src/lib/permission-rules.test.ts | 125 ++++++ packages/cli/src/lib/permission-rules.ts | 189 +++++++++ .../cli/src/lib/sandbox-capability.test.ts | 110 ++++++ packages/cli/src/lib/sandbox-capability.ts | 174 +++++++++ packages/cli/src/lib/sandbox-path.test.ts | 92 +++++ packages/cli/src/lib/sandbox-path.ts | 85 ++++ packages/cli/src/lib/sandbox-profile.test.ts | 124 ++++++ packages/cli/src/lib/sandbox-profile.ts | 247 ++++++++++++ 17 files changed, 2619 insertions(+) create mode 100644 packages/cli/scripts/acceptance.ts create mode 100644 packages/cli/scripts/sandbox-bisect.ts create mode 100644 packages/cli/scripts/sandbox-doctor.ts create mode 100644 packages/cli/src/lib/audit-log.rotation.test.ts create mode 100644 packages/cli/src/lib/audit-log.test.ts create mode 100644 packages/cli/src/lib/audit-log.ts create mode 100644 packages/cli/src/lib/local-tools.audit.test.ts create mode 100644 packages/cli/src/lib/local-tools.readline.test.ts create mode 100644 packages/cli/src/lib/local-tools.sandbox.test.ts create mode 100644 packages/cli/src/lib/permission-rules.test.ts create mode 100644 packages/cli/src/lib/permission-rules.ts create mode 100644 packages/cli/src/lib/sandbox-capability.test.ts create mode 100644 packages/cli/src/lib/sandbox-capability.ts create mode 100644 packages/cli/src/lib/sandbox-path.test.ts create mode 100644 packages/cli/src/lib/sandbox-path.ts create mode 100644 packages/cli/src/lib/sandbox-profile.test.ts create mode 100644 packages/cli/src/lib/sandbox-profile.ts diff --git a/packages/cli/scripts/acceptance.ts b/packages/cli/scripts/acceptance.ts new file mode 100644 index 0000000..a6daf63 --- /dev/null +++ b/packages/cli/scripts/acceptance.ts @@ -0,0 +1,365 @@ +/** + * Manual acceptance runner. + * + * One command that exercises the four shipped capabilities the way a user would + * hit them, and prints PASS/FAIL per check. It replaces "trust me, the tests + * pass" with something you can actually watch. + * + * What it CANNOT cover: colours, toasts, and transcript behaviour in the TUI. + * Those need a real terminal — see `--` section at the bottom of the output. + * + * bun run packages/cli/scripts/acceptance.ts + * bun run packages/cli/scripts/acceptance.ts --with-eval # also runs 2 real tasks + * bun run packages/cli/scripts/acceptance.ts --with-network # also probes real providers + */ +import { readFileSync, rmSync } from "node:fs"; +import { isShellConfined, sandboxCapability } from "../src/lib/sandbox-capability"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Mode } from "@mocode/shared"; +import { executeLocalTool } from "../src/lib/local-tools"; +import { createFailoverFetch } from "../src/lib/chat-fetch"; +import { buildModelChain, classifyFailover, runWithFailover } from "../src/lib/provider-chain"; + +const args = new Set(process.argv.slice(2)); +const withEval = args.has("--with-eval"); +const withNetwork = args.has("--with-network"); + +type Check = { section: string; name: string; ok: boolean; detail: string }; +const checks: Check[] = []; + +function record(section: string, name: string, ok: boolean, detail: string) { + checks.push({ section, name, ok, detail }); + console.log(` ${ok ? "✅" : "❌"} ${name}`); + if (detail) console.log(` ${detail}`); +} + +/** Expect the call to reject; report the message for eyeballing. */ +async function expectRefusal( + section: string, + name: string, + run: () => Promise, + needle: string, +) { + try { + await run(); + record(section, name, false, "NOT REFUSED — this is a real hole"); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + record(section, name, message.includes(needle), message.slice(0, 90)); + } +} + +function sh(command: string[]): { code: number; out: string } { + const result = Bun.spawnSync(command, { stdout: "pipe", stderr: "pipe" }); + return { + code: result.exitCode ?? 1, + out: `${result.stdout.toString()}${result.stderr.toString()}`.trim(), + }; +} + +// --------------------------------------------------------------------------- +console.log("\n=== P1 · Provider 失败回退 ===\n"); + +{ + // Policy invariants — the part that must never regress, regardless of network. + const chain = buildModelChain("gpt-oss-120b"); + record( + "P1", + "回退链把主模型排在首位、且不重复尝试同一家", + chain[0] === "gpt-oss-120b" && new Set(chain).size === chain.length, + chain.join(" → "), + ); + + const blocked = classifyFailover(Object.assign(new Error("x"), { statusCode: 403 })); + record("P1", "403 判定为可重试(换一家可能就好了)", blocked.retryable, blocked.detail); + + const badRequest = classifyFailover(Object.assign(new Error("x"), { statusCode: 400 })); + record( + "P1", + "400 判定为不可重试(是我们请求体写错了)", + badRequest.retryable === false, + badRequest.detail, + ); + + const aborted = classifyFailover(Object.assign(new Error("x"), { name: "AbortError" })); + record("P1", "用户按 Esc 取消后不重试", aborted.retryable === false, aborted.detail); + + // Real orchestration with a simulated provider (no network, no cost). + const outcome = await runWithFailover({ + primary: "gpt-oss-120b", + consent: true, + attempt: async (modelId) => { + if (modelId === "gpt-oss-120b") throw Object.assign(new Error("blocked"), { statusCode: 403 }); + return `answered by ${modelId}`; + }, + }); + record( + "P1", + "降级被显式上报(downgradedFrom 非空)", + outcome.downgradedFrom === "gpt-oss-120b" && outcome.modelId === "deepseek-flash", + `${outcome.downgradedFrom} → ${outcome.modelId}`, + ); +} + +{ + // The shape that broke in production: server answers 200, the provider error + // arrives as an SSE event. Status-code-only retry logic never fires here. + const sse = (events: Array>) => + new Response(events.map((event) => `data: ${JSON.stringify(event)}\n\n`).join(""), { + status: 200, + headers: { "content-type": "text/event-stream" }, + }); + + let calls = 0; + const fetchWithFallback = createFailoverFetch({ + allowFailover: true, + fetchImpl: (async () => { + calls += 1; + return calls === 1 + ? sse([{ type: "start" }, { type: "error", errorText: "Forbidden" }]) + : sse([{ type: "start" }, { type: "text-delta", delta: "ok" }, { type: "finish" }]); + }) as unknown as typeof globalThis.fetch, + readError: async (response) => `HTTP ${response.status}`, + }); + + const response = await fetchWithFallback("http://localhost:3000/chat", { + method: "POST", + body: JSON.stringify({ id: "s", model: "gpt-oss-120b", mode: "build", messages: [] }), + }); + const text = await response.text(); + + record( + "P1", + "200 响应体里的 error 事件也会触发回退", + calls === 2 && text.includes("ok"), + `共 ${calls} 次请求,最终流内容包含成功响应`, + ); +} + +if (withNetwork) { console.log("\n [--with-network] 真实探测三家 Provider(约 40 秒)…"); + const result = sh(["bun", "run", "packages/server/scripts/provider-failover-bench.ts"]); + const e2e = result.out.split("端到端回退")[1] ?? ""; + record( + "P1", + "真实请求下回退到 DeepSeek 并成功", + e2e.includes("deepseek-flash") && e2e.includes("(downgraded)"), + "见下方 provider-failover-bench 输出", + ); + console.log(result.out.split("\n").slice(-12).join("\n")); +} else { + console.log("\n (网络真实探测需加 --with-network)"); +} + +// --------------------------------------------------------------------------- +console.log("\n=== P2 · 权限与沙箱 ===\n"); + +{ + // Informational: which sandbox backend this host can actually apply. + const capability = sandboxCapability(); + const verdict = capability.available + ? `✅ 可用(${capability.backend},${capability.enforced ? "已在 spawn 处套用" : "未套用"})` + : `⚠️ 不可用(${capability.backend})→ 每条 bash 命令都需人工批准`; + + console.log(` ${verdict}`); + console.log(` ${capability.detail}`); +} + +await expectRefusal( + "P2", + "读 /etc/passwd 被拒", + () => executeLocalTool("readFile", { path: "/etc/passwd" }, Mode.BUILD), + "outside the project", +); + +await expectRefusal( + "P2", + "父目录穿越 ../../etc/passwd 被拒", + () => executeLocalTool("readFile", { path: "../../../../etc/passwd" }, Mode.BUILD), + "outside the project", +); + +await expectRefusal( + "P2", + "写到项目外被拒", + () => executeLocalTool("writeFile", { path: "/tmp/mocode-acceptance.txt", content: "x" }, Mode.BUILD), + "outside the project", +); + +await expectRefusal( + "P2", + "硬封禁命令(dd 写设备)即使批准也拒绝", + () => executeLocalTool("bash", { command: "dd if=/dev/zero of=/dev/null count=1" }, Mode.BUILD), + "cannot be approved", +); + +await expectRefusal( + "P2", + "Plan 模式写文件被拒", + () => executeLocalTool("writeFile", { path: "x.txt", content: "y" }, Mode.PLAN), + "not available in PLAN", +); + +{ + // Real leak probe: put a canary in the CLI process and ask the shell for it. + const canary = "sk-acceptance-canary"; + const previousKey = process.env.DEEPSEEK_API_KEY; + process.env.DEEPSEEK_API_KEY = canary; + try { + const result = (await executeLocalTool( + "bash", + { command: 'echo -n "${DEEPSEEK_API_KEY:-absent}"' }, + Mode.BUILD, + )) as { stdout: string; stderr: string; exitCode: number }; + + const clean = result.stdout === "absent" && !result.stdout.includes(canary); + record( + "P2", + "子 shell 看不到 CLI 的密钥", + clean, + clean + ? `echo $DEEPSEEK_API_KEY → "${result.stdout}"` + : `stdout=${JSON.stringify(result.stdout)} exit=${result.exitCode} stderr=${JSON.stringify( + (result.stderr ?? "").slice(0, 200), + )} — 若 exit≠0 说明 shell 没跑起来,与密钥无关`, + ); + } finally { + if (previousKey === undefined) delete process.env.DEEPSEEK_API_KEY; + else process.env.DEEPSEEK_API_KEY = previousKey; + } +} + +{ + // Audit trail: a refusal that leaves no record is indistinguishable from a bug. + const auditPath = join(tmpdir(), `mocode-acceptance-audit-${Date.now()}.jsonl`); + process.env.MOCODE_AUDIT_PATH = auditPath; + + try { + await expectRefusal( + "P2", + "越界读取在审计日志里留下 PATH_ESCAPE", + () => executeLocalTool("readFile", { path: "/etc/shadow" }, Mode.BUILD), + "outside the project", + ); + + const logged = readFileSync(auditPath, "utf8"); + record( + "P2", + "审计日志记录了拒绝原因", + logged.includes('"code":"PATH_ESCAPE"') && logged.includes('"outcome":"denied"'), + logged.trim().split("\n").at(-1)?.slice(0, 120) ?? "", + ); + } catch { + record("P2", "审计日志记录了拒绝原因", false, "日志文件未生成"); + } finally { + delete process.env.MOCODE_AUDIT_PATH; + try { + rmSync(auditPath, { force: true }); + } catch { + /* ignore */ + } + } +} + +{ + // The original exploit, replayed end-to-end: the agent read /etc/passwd through + // bash and `head`, bypassing the file tools entirely. + if (!isShellConfined()) { + console.log(" ⏭ 跳过「bash 读 /etc/passwd」:本机无 OS 沙箱,该路径靠审批阻挡"); + } else { + const shellRead = (await executeLocalTool( + "bash", + { command: "head -1 /etc/passwd 2>&1" }, + Mode.BUILD, + )) as { stdout: string; stderr: string; exitCode: number }; + + const output = `${shellRead.stdout}${shellRead.stderr}`; + const leaked = /^[^:\n]+:[^:\n]*:[0-9]+:/m.test(output); + + record( + "P2", + "bash 也读不到 /etc/passwd(原始逃逸路径已封)", + !leaked, + leaked ? `泄漏了内容:${output.slice(0, 80)}` : output.trim().slice(0, 90), + ); + } +} + +// --------------------------------------------------------------------------- +console.log("\n=== P3 · 上下文压缩 ===\n"); + +{ + const result = sh(["bun", "run", "packages/server/scripts/context-compaction-bench.ts"]); + const triggered = result.out.includes("压缩触发: 是"); + const reduced = /→ (\d+) \(-(\d+)%\)/.exec(result.out); + const percent = reduced?.[2] ?? "?"; + + record( + "P3", + "50 轮会话触发压缩", + triggered, + result.out.split("\n").find((line) => line.startsWith("prompt tokens")) ?? "", + ); + record("P3", "压缩后 prompt 显著下降(≥70%)", Number(percent) >= 70, `降幅 ${percent}%`); + record( + "P3", + "折叠摘要说明了内容已不在上下文", + result.out.includes("re-read a file"), + "摘要末句提示模型需要时重新读文件", + ); +} + +// --------------------------------------------------------------------------- +console.log("\n=== P4 · 任务评估 ===\n"); + +if (withEval) { + console.log(" [--with-eval] 跑 2 条真实任务(约 15 秒,产生少量 token 费用)…"); + for (const id of ["fix-off-by-one", "rename-across-files"]) { + const result = sh(["bun", "run", "packages/cli/evals/run.ts", "--only", id]); + record("P4", `真实任务 ${id} 通过`, result.code === 0, result.out.split("\n")[2] ?? ""); + } +} else { + const result = sh(["bun", "test", "packages/cli/evals/cases.test.ts"]); + record( + "P4", + "元测试:每条用例在初始状态必须失败", + result.code === 0, + result.out.split("\n").find((line) => line.includes("pass")) ?? "", + ); + console.log(" (真实任务需加 --with-eval)"); +} + +// --------------------------------------------------------------------------- +const failed = checks.filter((check) => !check.ok); + +console.log("\n=== 汇总 ==="); +console.log(`通过 ${checks.length - failed.length}/${checks.length}`); + +if (failed.length > 0) { + console.log("\n失败项:"); + for (const check of failed) console.log(` - [${check.section}] ${check.name}: ${check.detail}`); +} + +console.log(` +=== 需要你在真实终端里确认的部分(脚本无法覆盖)=== + +1) 状态行预算可视化 + bun run dev → 发一条会连续调工具的任务 + 期望:状态行出现 "turn 2/24 · 3 tools",数字随工具调用递增 + +2) 降级提示(P1 的肉眼验证) + /models 选 gpt-oss-120b → 发消息 + 期望:Toast「模型 gpt-oss-120b 不可用,已回退到 deepseek-flash」 + 状态行出现红色 "fallback → deepseek-flash" + 回复内容正常,不是 Forbidden + +3) 压缩可观测(P3) + 连续读几个大文件,跑过 25 轮 + 期望:该轮 finish metadata 带 compaction 字段(服务端日志) + +4) 任务评估全量 + bun run packages/cli/evals/run.ts + 期望:10/10,报告写入 packages/cli/evals/results/latest.json +`); + +process.exit(failed.length === 0 ? 0 : 1); diff --git a/packages/cli/scripts/sandbox-bisect.ts b/packages/cli/scripts/sandbox-bisect.ts new file mode 100644 index 0000000..b639482 --- /dev/null +++ b/packages/cli/scripts/sandbox-bisect.ts @@ -0,0 +1,105 @@ +/** + * Sandbox profile bisect. + * + * `SIGABRT` from `sandbox-exec` gives no stderr, so guessing which rule is + * malformed is pointless. This walks from the smallest profile to the full + * generated one, adding one operation at a time, and reports the first case that + * aborts — which names the offending rule. + * + * bun run packages/cli/scripts/sandbox-bisect.ts + */ +import { buildSeatbeltProfile, parsePathList, runtimePaths } from "../src/lib/sandbox-profile"; + +const cwd = process.cwd(); +const options = { + cwd, + home: process.env.HOME, + tmpDir: process.env.TMPDIR, + runtimePaths: runtimePaths(), + extraReadPaths: parsePathList(process.env.MOCODE_SANDBOX_EXTRA_READ), + extraWritePaths: parsePathList(process.env.MOCODE_SANDBOX_EXTRA_WRITE), +}; + +const full = buildSeatbeltProfile(options); + +/** Each case adds exactly one rule to the previous one. */ +const cases: Array<{ name: string; profile: string }> = [ + { name: "1 (allow default) 基线", profile: "(version 1)(allow default)" }, + ...(() => { + const rules = [ + "(deny default)", + "(allow file-read*)", + "(allow process*)", + "(allow signal)", + "(allow sysctl-read)", + "(allow mach-lookup)", + "(allow file-read-metadata)", + "(allow file-map-executable)", + "(allow network*)", + ]; + + return rules.map((rule, index) => ({ + name: `2.${index + 1} + ${rule}`, + profile: `(version 1)\n${rules.slice(0, index + 1).join("\n")}`, + })); + })(), + { name: "3 完整生成的 profile(-p)", profile: full }, +]; + +function run(name: string, profile: string) { + const result = Bun.spawnSync(["/usr/bin/sandbox-exec", "-p", profile, "/bin/echo", "ok"], { + stdout: "pipe", + stderr: "pipe", + }); + + const exit = result.exitCode; + const signal = result.signalCode; + const stdout = result.stdout.toString().trim(); + const stderr = result.stderr.toString().trim(); + + const verdict = exit === 0 && stdout === "ok" ? "OK " : "FAIL"; + console.log(`${verdict} ${name.padEnd(34)} exit=${exit} signal=${signal ?? "-"}`); + + if (verdict === "FAIL") { + if (stderr) console.log(` stderr: ${stderr.slice(0, 300)}`); + if (stdout) console.log(` stdout: ${JSON.stringify(stdout.slice(0, 120))}`); + } + + return verdict === "OK "; +} + +console.log(`cwd: ${cwd}`); +console.log(`TMPDIR: ${process.env.TMPDIR ?? "(unset)"}\n`); + +let lastOk = ""; +let firstFail = ""; + +for (const testCase of cases) { + const ok = run(testCase.name, testCase.profile); + if (ok) lastOk = testCase.name; + else if (!firstFail) firstFail = testCase.name; +} + +// File mode prints parse errors that -p swallows. +console.log("\n--- 完整 profile 用 -f(文件模式会打印语法错误)---"); +const profilePath = `${process.env.TMPDIR ?? "/tmp"}/mocode-bisect.sb`; +await Bun.write(profilePath, full); +const fileRun = Bun.spawnSync(["/usr/bin/sandbox-exec", "-f", profilePath, "/bin/echo", "ok"], { + stdout: "pipe", + stderr: "pipe", +}); +console.log( + ` -f 结果: exit=${fileRun.exitCode} signal=${fileRun.signalCode ?? "-"} stderr=${JSON.stringify( + fileRun.stderr.toString().trim().slice(0, 300), + )}`, +); + +console.log(`\n=== 结论 ===`); +console.log(`最后一条通过的规则组合: ${lastOk || "(无)"}`); +console.log(`第一条失败的: ${firstFail || "(无)"}`); +console.log(` +判读: +- 卡在 2.x 的某一步 → 那一步的规则名就是元凶,删掉或换写法。 +- 1 就失败 → 这台机器的 sandbox-exec 对文字 profile 有问题,只认 -f 文件。 +- 3 失败而 2.x 全过 → 差值就是完整 profile 里的路径规则(subpath 列表)。 +`); diff --git a/packages/cli/scripts/sandbox-doctor.ts b/packages/cli/scripts/sandbox-doctor.ts new file mode 100644 index 0000000..b684cb0 --- /dev/null +++ b/packages/cli/scripts/sandbox-doctor.ts @@ -0,0 +1,168 @@ +/** + * Sandbox doctor. + * + * `SIGABRT` with empty stderr has two candidates, and they need different fixes: + * the profile is malformed, or the child environment we hand `sandbox-exec` is too + * bare. The probes below vary exactly one of those at a time. + * + * bun run packages/cli/scripts/sandbox-doctor.ts + */ +import { buildSandboxEnv } from "../src/lib/permission-rules"; +import { sandboxBackend, sandboxCapability } from "../src/lib/sandbox-capability"; +import { buildSeatbeltProfile, parsePathList, runtimePaths, spawnPlan } from "../src/lib/sandbox-profile"; + +const cwd = process.cwd(); +const backend = sandboxBackend(); + +const profileOptions = { + cwd, + home: process.env.HOME, + tmpDir: process.env.TMPDIR, + runtimePaths: runtimePaths(), + extraReadPaths: parsePathList(process.env.MOCODE_SANDBOX_EXTRA_READ), + extraWritePaths: parsePathList(process.env.MOCODE_SANDBOX_EXTRA_WRITE), +}; + +if (process.argv.includes("--dump-profile")) { + console.log(buildSeatbeltProfile(profileOptions)); + process.exit(0); +} + +console.log("=== capability ==="); +console.log(JSON.stringify(sandboxCapability(), null, 2)); +console.log(`backend used at spawn: ${backend}`); +console.log(`platform: ${process.platform}`); +console.log(`cwd: ${cwd}`); +console.log(`TMPDIR: ${process.env.TMPDIR ?? "(unset)"}`); + +// A canary in the parent process proves the env allowlist drops it. +process.env.MOCODE_DOCTOR_CANARY = "should-not-be-visible"; +const sandboxEnv = buildSandboxEnv(); + +const FULL_PROFILE = buildSeatbeltProfile(profileOptions); +const TRIVIAL_PROFILE = "(version 1)(allow default)"; + +type Variant = { + label: string; + argv: string[]; + env: Record; + expect: string; +}; + +function seatbelt(profile: string, command: string): string[] { + return ["/usr/bin/sandbox-exec", "-p", profile, "bash", "-c", command]; +} + +/** Linux has no sandbox-exec: build the bubblewrap argv from the real plan. */ +function bwrap(command: string): string[] { + const plan = spawnPlan("bwrap", command, profileOptions); + return [plan.file, ...plan.args]; +} + +const isMac = process.platform === "darwin"; + +const macVariants: Variant[] = [ + { + label: "1. 极简 profile + 继承 env (= 能力探测的跑法,已知能过)", + argv: seatbelt(TRIVIAL_PROFILE, "echo hi"), + env: process.env, + expect: "hi", + }, + { + label: "2. 极简 profile + 白名单 env (隔离 env 的影响)", + argv: seatbelt(TRIVIAL_PROFILE, "echo hi"), + env: sandboxEnv, + expect: "hi", + }, + { + label: "3. 完整 profile + 继承 env (隔离 profile 的影响)", + argv: seatbelt(FULL_PROFILE, "echo hi"), + env: process.env, + expect: "hi", + }, + { + label: "4. 完整 profile + 白名单 env (= 真实 spawn,当前失败)", + argv: seatbelt(FULL_PROFILE, "echo hi"), + env: sandboxEnv, + expect: "hi", + }, + { + label: "5. 真实 spawnPlan 路径", + argv: (() => { + const plan = spawnPlan(backend, "echo hi", profileOptions); + return [plan.file, ...plan.args]; + })(), + env: sandboxEnv, + expect: "hi", + }, + { + label: "6. 完整 profile:读 /etc/passwd 应被拒", + argv: seatbelt(FULL_PROFILE, "head -1 /etc/passwd"), + env: process.env, + expect: "denied (no content)", + }, + { + label: "7. 完整 profile:canary 变量应不可见", + argv: seatbelt(FULL_PROFILE, 'echo -n "${MOCODE_DOCTOR_CANARY:-absent}"'), + env: sandboxEnv, + expect: "absent", + }, +]; + +const linuxVariants: Variant[] = [ + { + label: "1. 真实 bwrap 计划 + 继承 env", + argv: bwrap("echo hi"), + env: process.env, + expect: "hi", + }, + { + label: "2. 真实 bwrap 计划 + 白名单 env", + argv: bwrap("echo hi"), + env: sandboxEnv, + expect: "hi", + }, + { + label: "3. 读 /etc/shadow 应被 mask", + argv: bwrap("head -1 /etc/shadow"), + env: process.env, + expect: "denied (no content)", + }, + { + label: "4. canary 变量应不可见", + argv: bwrap('echo -n "${MOCODE_DOCTOR_CANARY:-absent}"'), + env: sandboxEnv, + expect: "absent", + }, +]; + +const variants: Variant[] = isMac ? macVariants : linuxVariants; + +for (const variant of variants) { + const result = Bun.spawnSync(variant.argv, { + cwd, + env: variant.env as Record, + stdout: "pipe", + stderr: "pipe", + }); + + const stdout = result.stdout.toString(); + const stderr = result.stderr.toString(); + + console.log(`\n### ${variant.label}`); + console.log(` expect : ${variant.expect}`); + console.log(` exit : ${result.exitCode} signal: ${result.signalCode ?? "-"}`); + console.log(` stdout : ${JSON.stringify(stdout.slice(0, 200))}`); + console.log(` stderr : ${JSON.stringify(stderr.slice(0, 400))}`); +} + +console.log(` +=== 怎么读 === + +2 崩了(而 1 正常) → 是**白名单 env** 的问题。sandbox-exec 需要某个被我们剔除的 + 环境变量;修法是给它一份完整的 env,再单独约束子进程。 +3 崩了(而 2 正常) → 是**profile** 的问题。跑 sandbox-bisect.ts 定位到具体规则。 +2 和 3 都正常,4 崩 → 两者相互作用(一般仍是 env:profile 里引用了仅 parent 才有的变量)。 +6 有内容 → 沙箱没真正生效,按安全 bug 处理。 +7 打印 canary → env 白名单失效。 +`); diff --git a/packages/cli/src/lib/audit-log.rotation.test.ts b/packages/cli/src/lib/audit-log.rotation.test.ts new file mode 100644 index 0000000..4e9de45 --- /dev/null +++ b/packages/cli/src/lib/audit-log.rotation.test.ts @@ -0,0 +1,121 @@ +/** + * Audit log rotation. + * + * Without rotation a long-lived agent grows `audit.jsonl` without bound — the + * audit trail becomes the outage. These tests pin the two properties that matter: + * the file stays under the limit, and the archive count stays fixed. + */ +import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { auditRotation, recordAudit, rotateIfNeeded } from "./audit-log"; + +const dir = mkdtempSync(join(tmpdir(), "mocode-audit-rotate-")); +const logPath = join(dir, "audit.jsonl"); + +beforeAll(() => { + process.env.MOCODE_AUDIT_PATH = logPath; + // Tiny limits so a handful of entries trigger rotation. + process.env.MOCODE_AUDIT_MAX_BYTES = "400"; + process.env.MOCODE_AUDIT_MAX_FILES = "2"; +}); + +afterEach(() => { + for (let index = 0; index <= 4; index += 1) { + rmSync(`${logPath}.${index}`, { force: true }); + } + rmSync(logPath, { force: true }); +}); + +afterAll(() => { + delete process.env.MOCODE_AUDIT_PATH; + delete process.env.MOCODE_AUDIT_MAX_BYTES; + delete process.env.MOCODE_AUDIT_MAX_FILES; + rmSync(dir, { recursive: true, force: true }); +}); + +function write(count: number) { + for (let index = 0; index < count; index += 1) { + recordAudit({ tool: "bash", mode: "BUILD", outcome: "ok", detail: `n=${index}` }); + } +} + +describe("policy", () => { + test("defaults are read from the environment when set", () => { + expect(auditRotation()).toEqual({ maxBytes: 400, maxFiles: 2 }); + }); + + test("invalid values fall back to the defaults, never to zero", () => { + process.env.MOCODE_AUDIT_MAX_BYTES = "nonsense"; + process.env.MOCODE_AUDIT_MAX_FILES = "-3"; + try { + const rotation = auditRotation(); + expect(rotation.maxBytes).toBeGreaterThan(0); + expect(rotation.maxFiles).toBeGreaterThan(0); + } finally { + process.env.MOCODE_AUDIT_MAX_BYTES = "400"; + process.env.MOCODE_AUDIT_MAX_FILES = "2"; + } + }); +}); + +describe("rotation", () => { + test("does nothing while the file is under the limit", () => { + writeFileSync(logPath, "x".repeat(50)); + + expect(rotateIfNeeded(logPath)).toBe(false); + expect(existsSync(`${logPath}.1`)).toBe(false); + }); + + test("moves the file aside once it reaches the limit", () => { + writeFileSync(logPath, "x".repeat(600)); + + expect(rotateIfNeeded(logPath)).toBe(true); + expect(existsSync(`${logPath}.1`)).toBe(true); + expect(existsSync(logPath)).toBe(false); + }); + + test("returns false when there is no file yet", () => { + expect(rotateIfNeeded(logPath)).toBe(false); + }); + + test("a burst of writes stays bounded and keeps the newest data", () => { + write(40); + + // Active file must be under the limit — that is the whole point. + const activeSize = existsSync(logPath) ? readFileSync(logPath, "utf8").length : 0; + expect(activeSize).toBeLessThan(400 + 200); // limit + one line of slack + + // The most recent entry survives somewhere in the chain. + const all = [ + logPath, + `${logPath}.1`, + `${logPath}.2`, + ] + .filter((path) => existsSync(path)) + .map((path) => readFileSync(path, "utf8")) + .join(""); + + expect(all).toContain("n=39"); + }); + + test("never keeps more archives than maxFiles", () => { + write(60); + + expect(existsSync(`${logPath}.3`)).toBe(false); + expect(existsSync(`${logPath}.2`)).toBe(true); + }); + + test("every archive line is still valid JSON", () => { + write(30); + + for (const path of [logPath, `${logPath}.1`, `${logPath}.2`]) { + if (!existsSync(path)) continue; + + for (const line of readFileSync(path, "utf8").trim().split("\n").filter(Boolean)) { + expect(() => JSON.parse(line)).not.toThrow(); + } + } + }); +}); diff --git a/packages/cli/src/lib/audit-log.test.ts b/packages/cli/src/lib/audit-log.test.ts new file mode 100644 index 0000000..97c8c0a --- /dev/null +++ b/packages/cli/src/lib/audit-log.test.ts @@ -0,0 +1,151 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { defaultAuditPath, recordAudit, summarizeInput } from "./audit-log"; + +const dir = mkdtempSync(join(tmpdir(), "mocode-audit-")); +const logPath = join(dir, "audit.jsonl"); + +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +function lines(): Array> { + return readFileSync(logPath, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); +} + +describe("summarizeInput", () => { + test("keeps short scalars that identify the call", () => { + expect(summarizeInput({ path: "src/a.ts", line_start: 3 })).toBe("path=src/a.ts line_start=3"); + }); + + test("never records file contents", () => { + const body = "x".repeat(5_000); + expect(summarizeInput({ path: "a.ts", content: body })).toBe("path=a.ts"); + }); + + test("records the command — an audit entry without it cannot be reviewed", () => { + expect(summarizeInput({ command: "cat ~/.ssh/id_rsa", description: "list" })).toBe( + "command=cat ~/.ssh/id_rsa description=list", + ); + }); + + test("masks credentials that appear inside a command", () => { + expect(summarizeInput({ command: "export ANTHROPIC_API_KEY=abcd1234efgh" })).not.toContain( + "abcd1234efgh", + ); + expect(summarizeInput({ command: 'curl -H "Authorization: Bearer sk-live-abcdef123456"' })).not.toContain( + "sk-live-abcdef123456", + ); + expect(summarizeInput({ command: "sk-abcdefghijklmnop tool" })).not.toContain("abcdefghijklmnop"); + }); + + test("masks X-API-Key and other credential headers", () => { + expect( + summarizeInput({ command: 'curl -H "X-API-Key: sk-live-abcdef123456" https://x' }), + ).not.toContain("sk-live-abcdef123456"); + + expect(summarizeInput({ command: "curl -H 'X-Api-Key: plainsecretvalue' https://x" })) + .not.toContain("plainsecretvalue"); + }); + + test("never records edit payloads", () => { + expect( + summarizeInput({ path: "a.ts", oldString: "secret", newString: "other" }), + ).toBe("path=a.ts"); + }); + + test("drops multi-line values even when short", () => { + expect(summarizeInput({ path: "a.ts", description: "a\nb" })).toBe("path=a.ts"); + }); + + test("is bounded regardless of how many keys arrive", () => { + const input = Object.fromEntries( + Array.from({ length: 50 }, (_, i) => [`path${i}`, "v".repeat(50)]), + ); + // Unknown keys are ignored; only the allowlisted `path` field is kept. + expect(summarizeInput({ ...input, path: "src/a.ts" }).length).toBeLessThanOrEqual(240); + expect(summarizeInput({ ...input, path: "src/a.ts" })).toContain("path=src/a.ts"); + }); + + test("non-object input yields nothing", () => { + expect(summarizeInput(null)).toBe(""); + expect(summarizeInput("raw")).toBe(""); + }); +}); + +describe("recordAudit", () => { + test("appends one JSON line per decision", () => { + recordAudit({ tool: "readFile", mode: "BUILD", outcome: "ok", detail: "path=a.ts" }, logPath); + recordAudit( + { tool: "readFile", mode: "BUILD", outcome: "denied", code: "PATH_ESCAPE", detail: "path=/etc" }, + logPath, + ); + + const entries = lines(); + expect(entries).toHaveLength(2); + expect(entries[0]).toMatchObject({ tool: "readFile", outcome: "ok" }); + expect(entries[1]).toMatchObject({ outcome: "denied", code: "PATH_ESCAPE" }); + }); + + test("stamps every entry with an ISO timestamp", () => { + recordAudit({ tool: "bash", mode: "BUILD", outcome: "denied", code: "HARD_DENY" }, logPath); + + const entry = lines().at(-1); + expect(typeof entry?.timestamp).toBe("string"); + expect(Number.isNaN(Date.parse(String(entry?.timestamp)))).toBe(false); + }); + + test("creates the directory when it does not exist", () => { + const nested = join(dir, "deep", "nested", "audit.jsonl"); + expect(recordAudit({ tool: "grep", mode: "BUILD", outcome: "ok" }, nested)).toBe(true); + }); + + test("fails soft instead of throwing when the path is unusable", () => { + // A directory cannot be appended to as a file. + expect(recordAudit({ tool: "bash", mode: "BUILD", outcome: "ok" }, dir)).toBe(false); + }); + + test("honours MOCODE_AUDIT_PATH", () => { + process.env.MOCODE_AUDIT_PATH = logPath; + try { + expect(defaultAuditPath()).toBe(logPath); + } finally { + delete process.env.MOCODE_AUDIT_PATH; + } + }); +}); + +describe("shell commands are auditable", () => { + test("the command itself is recorded — an entry without it is useless", () => { + expect(summarizeInput({ command: "head -1 /etc/passwd" })).toBe("command=head -1 /etc/passwd"); + }); + + test("multi-line commands are collapsed, not dropped", () => { + const summary = summarizeInput({ command: "ls -la\nfile secret.txt" }); + expect(summary).toBe("command=ls -la file secret.txt"); + }); + + test("a command containing a heredoc is capped, not logged in full", () => { + const body = "cat <<'X'\n" + "y".repeat(2_000) + "\nX"; + const summary = summarizeInput({ command: body }); + expect(summary.length).toBeLessThanOrEqual(240); + expect(summary.startsWith("command=")).toBe(true); + }); + + test("file bodies are still dropped", () => { + expect(summarizeInput({ path: "a.ts", content: "x".repeat(500) })).toBe("path=a.ts"); + }); + + test("command and description both survive", () => { + const summary = summarizeInput({ + command: "ls -la secret.txt", + description: "Inspect link", + }); + expect(summary).toContain("command=ls -la secret.txt"); + expect(summary).toContain("description=Inspect link"); + }); +}); diff --git a/packages/cli/src/lib/audit-log.ts b/packages/cli/src/lib/audit-log.ts new file mode 100644 index 0000000..f2ebbbc --- /dev/null +++ b/packages/cli/src/lib/audit-log.ts @@ -0,0 +1,247 @@ +/** + * Append-only audit log for tool authorization decisions. + * + * Enterprise requirement, and the reason the executor — not the model — is the + * authority: every call must leave a record of *what was attempted* and *what was + * decided*, whether it ran or was refused. A refusal that nobody can audit is + * indistinguishable from a bug. + * + * Design constraints: + * + * 1. **Never log file contents.** Only short, single-line scalar arguments are + * recorded, capped hard. An audit log that captures source code is a data leak + * with extra steps. + * 2. **Never break a tool call.** Auditing is best-effort: a full disk or a + * read-only home must not turn a working tool into a failing one. + * 3. **One line per decision, JSONL** — append-only and greppable, so `jq` and log + * shippers work without a parser. + */ +import { appendFileSync, chmodSync, mkdirSync, renameSync, rmSync, statSync } from "node:fs"; +import { homedir } from "node:os"; +import { dirname, join } from "node:path"; + +/** Owner-only directory mode for the audit trail. */ +const AUDIT_DIR_MODE = 0o700; +/** Owner-only file mode for the active log and rotated archives. */ +const AUDIT_FILE_MODE = 0o600; + +/** Best-effort chmod — never fails the tool call that triggered the write. */ +function ownerOnly(path: string, mode: number): void { + try { + chmodSync(path, mode); + } catch { + // Platform/umask may refuse; auditing remains best-effort. + } +} + +/** Rotate once the active file reaches this size. */ +const DEFAULT_MAX_BYTES = 5 * 1024 * 1024; +/** Archives kept alongside the active file (`audit.jsonl.1` … `.N`). */ +const DEFAULT_MAX_FILES = 5; + +export type AuditRotation = { + maxBytes: number; + maxFiles: number; +}; + +/** Read positive numbers from the environment, falling back to the defaults. */ +function readPositive(value: string | undefined, fallback: number): number { + const parsed = Number(value); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +/** Rotation policy, overridable for tests and for long-lived agents. */ +export function auditRotation(): AuditRotation { + return { + maxBytes: readPositive(process.env.MOCODE_AUDIT_MAX_BYTES, DEFAULT_MAX_BYTES), + maxFiles: readPositive(process.env.MOCODE_AUDIT_MAX_FILES, DEFAULT_MAX_FILES), + }; +} + +function rotatedName(path: string, index: number): string { + return `${path}.${index}`; +} + +/** + * Move the active file aside once it exceeds `maxBytes`. + * + * Archives are numbered newest-first (`.1` is the most recent) and the oldest is + * dropped, so a chatty agent costs a bounded amount of disk and never grows the + * log indefinitely. Returns `true` when a rotation happened. + * + * Best-effort like the rest of this module: a failed rotation leaves the file in + * place and must not fail the tool call that triggered it. + */ +export function rotateIfNeeded( + path: string = defaultAuditPath(), + rotation: AuditRotation = auditRotation(), +): boolean { + let size: number; + + try { + size = statSync(path).size; + } catch { + // No file yet — nothing to rotate. + return false; + } + + if (size < rotation.maxBytes) return false; + + try { + // Drop the oldest archive before shifting, so the chain never exceeds maxFiles. + rmSync(rotatedName(path, rotation.maxFiles), { force: true }); + + for (let index = rotation.maxFiles - 1; index >= 1; index -= 1) { + try { + const from = rotatedName(path, index); + const to = rotatedName(path, index + 1); + renameSync(from, to); + ownerOnly(to, AUDIT_FILE_MODE); + } catch { + // That archive does not exist yet — fine. + } + } + + const archive = rotatedName(path, 1); + renameSync(path, archive); + ownerOnly(archive, AUDIT_FILE_MODE); + return true; + } catch { + return false; + } +} + +export type AuditOutcome = "ok" | "denied" | "error"; + +export type AuditEvent = { + /** ISO-8601, UTC. */ + timestamp: string; + tool: string; + mode: string; + outcome: AuditOutcome; + /** Machine-readable refusal code when `outcome` is `denied`. */ + code?: string; + /** Short summary of the call — never full argument payloads. */ + /** For shell calls: whether an OS sandbox confined the child process. */ + shellConfined?: boolean; + detail?: string; +}; + +/** Where decisions are appended. `MOCODE_AUDIT_PATH` exists for tests and CI. */ +export function defaultAuditPath(): string { + return process.env.MOCODE_AUDIT_PATH ?? join(homedir(), ".mocode", "audit.jsonl"); +} + +/** Longest single value kept in the log. */ +const MAX_VALUE_LENGTH = 120; +/** Longest total summary kept in the log. */ +const MAX_DETAIL_LENGTH = 240; +/** + * Commands get their own, larger budget. + * + * Auditing a shell call without the command is pointless, and shell commands are + * routinely multi-line. They are collapsed and truncated instead of dropped — the + * cap still bounds how much of a heredoc can leak. + */ +const MAX_COMMAND_LENGTH = 160; + +/** + * Secrets that may appear *inside* a command. + * + * The audit needs the command to be reviewable, but a command is user-controlled + * text and can carry credentials. Masking known shapes beats both extremes: + * logging raw (leaks) and dropping the command (useless). + */ +const SECRET_PATTERNS: ReadonlyArray<[RegExp, string]> = [ + [/(\bsk-[A-Za-z0-9_-]{8,})/g, "sk-***"], + [/(\bBearer\s+)[A-Za-z0-9._~+/-]{8,}/gi, "$1***"], + [/\b((?:[A-Z0-9_]*(?:API_KEY|TOKEN|SECRET|PASSWORD)[A-Z0-9_]*)=)(\S+)/g, "$1***"], + [/(Authorization:\s*)\S+/gi, "$1***"], + // Any header-ish `Name: value` where the name looks like a credential. + [/([\w-]*(?:API[-_]?Key|ApiKey|Token|Secret|Password)[\w-]*\s*:\s*)[^\s"']+/gi, "$1***"], +]; + +/** Mask credential-shaped substrings before a command reaches the log. */ +export function redactSecrets(value: string): string { + let redacted = value; + for (const [pattern, replacement] of SECRET_PATTERNS) { + redacted = redacted.replace(pattern, replacement); + } + return redacted; +} + +/** + * Explicitly approved string fields. Everything else — file bodies, credentials, + * bash commands, edit payloads, free-form prompts — is omitted. + */ +const SAFE_STRING_KEYS = new Set([ + "path", + "pattern", + "include", + "ref", + "description", + "subagent_type", + // A shell audit entry without its command cannot be reviewed. It is collapsed + // and capped rather than dropped — see MAX_COMMAND_LENGTH. + "command", +]); + +/** + * Reduce tool input to a short, single-line summary of approved fields only. + * + * Strings longer than {@link MAX_VALUE_LENGTH} or containing newlines are dropped + * entirely rather than truncated — a truncated file body is still file body. + */ +export function summarizeInput(input: unknown): string { + if (input === null || typeof input !== "object") return ""; + + const parts: string[] = []; + + for (const [key, value] of Object.entries(input as Record)) { + if (typeof value === "number" || typeof value === "boolean") { + parts.push(`${key}=${value}`); + continue; + } + + if (typeof value !== "string" || !SAFE_STRING_KEYS.has(key)) continue; + + if (key === "command") { + const collapsed = redactSecrets(value).replace(/\s+/g, " ").trim(); + if (collapsed) parts.push(`${key}=${collapsed.slice(0, MAX_COMMAND_LENGTH)}`); + continue; + } + + if (value.length <= MAX_VALUE_LENGTH && !value.includes("\n")) { + parts.push(`${key}=${value}`); + } + } + + return parts.join(" ").slice(0, MAX_DETAIL_LENGTH); +} + +/** + * Append one decision. Returns `true` when written, `false` when the write was + * skipped or failed — never throws. + * + * Rotation is checked immediately before the append, so the log is bounded from + * the first write without a background worker. + */ +export function recordAudit( + event: Omit, + path: string = defaultAuditPath(), +): boolean { + try { + const dir = dirname(path); + mkdirSync(dir, { recursive: true, mode: AUDIT_DIR_MODE }); + ownerOnly(dir, AUDIT_DIR_MODE); + rotateIfNeeded(path); + const line = JSON.stringify({ timestamp: new Date().toISOString(), ...event }); + appendFileSync(path, `${line}\n`, { encoding: "utf8", mode: AUDIT_FILE_MODE }); + // Re-assert owner-only so a pre-existing world-readable file does not stay open. + ownerOnly(path, AUDIT_FILE_MODE); + return true; + } catch { + // Auditing is best-effort: losing a log line must never fail a tool call. + return false; + } +} diff --git a/packages/cli/src/lib/local-tools.audit.test.ts b/packages/cli/src/lib/local-tools.audit.test.ts new file mode 100644 index 0000000..6d84ba2 --- /dev/null +++ b/packages/cli/src/lib/local-tools.audit.test.ts @@ -0,0 +1,128 @@ +/** + * Audit + structured refusal codes. + * + * Enterprise rule: the executor is the authority, so every decision must be + * recordable. These tests assert on the *log*, not on the error message — a + * refusal nobody can audit is indistinguishable from a bug. + */ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Mode } from "@mocode/shared"; +import { executeLocalTool } from "./local-tools"; +import { ToolPermissionError } from "./permission-rules"; + +const dir = mkdtempSync(join(tmpdir(), "mocode-tool-audit-")); +const logPath = join(dir, "audit.jsonl"); + +beforeAll(() => { + process.env.MOCODE_AUDIT_PATH = logPath; +}); + +afterAll(() => { + delete process.env.MOCODE_AUDIT_PATH; + rmSync(dir, { recursive: true, force: true }); +}); + +function entries(): Array> { + return readFileSync(logPath, "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as Record); +} + +/** Run a call that is expected to be refused and return the thrown error. */ +async function refused( + tool: string, + input: unknown, + mode: (typeof Mode)[keyof typeof Mode] = Mode.BUILD, +): Promise { + try { + await executeLocalTool(tool, input, mode); + } catch (error) { + expect(error).toBeInstanceOf(ToolPermissionError); + return error as ToolPermissionError; + } + + throw new Error(`expected ${tool} to be refused`); +} + +describe("structured refusal codes", () => { + test("path escape carries PATH_ESCAPE", async () => { + const error = await refused("readFile", { path: "/etc/passwd" }); + expect(error.code).toBe("PATH_ESCAPE"); + }); + + test("hard deny carries HARD_DENY", async () => { + const error = await refused("bash", { command: "mkfs.ext4 /dev/null" }); + expect(error.code).toBe("HARD_DENY"); + }); + + test("plan mode carries MODE_DENIED", async () => { + const error = await refused("writeFile", { path: "a.txt", content: "x" }, Mode.PLAN); + expect(error.code).toBe("MODE_DENIED"); + }); + + test("unknown tool carries UNKNOWN_TOOL", async () => { + const error = await refused("rmMinusRf", { path: "x" }); + expect(error.code).toBe("UNKNOWN_TOOL"); + }); + + test("the message stays readable for the model", async () => { + const error = await refused("readFile", { path: "/etc/passwd" }); + expect(error.message).toContain("outside the project directory"); + }); +}); + +describe("audit trail", () => { + test("a successful call is logged as ok", async () => { + await executeLocalTool("listDirectory", { path: "." }, Mode.BUILD); + const entry = entries().at(-1); + + expect(entry).toMatchObject({ tool: "listDirectory", mode: "BUILD", outcome: "ok" }); + }); + + test("a refusal is logged with its code — not silently dropped", async () => { + await refused("readFile", { path: "../../etc/passwd" }); + const entry = entries().at(-1); + + expect(entry).toMatchObject({ + tool: "readFile", + outcome: "denied", + code: "PATH_ESCAPE", + }); + }); + + test("a non-permission failure is logged as error without a code", async () => { + await expect( + executeLocalTool("readFile", { path: "definitely-missing.txt" }, Mode.BUILD), + ).rejects.toThrow(); + + const entry = entries().at(-1); + expect(entry).toMatchObject({ tool: "readFile", outcome: "error" }); + expect(entry?.code).toBeUndefined(); + }); + + test("file contents never reach the log", async () => { + const secret = "TOP-SECRET-PAYLOAD-" + "y".repeat(500); + + // Refused on purpose: the write must not happen, yet the attempt is logged — + // proving the summary is built from the input without echoing it. + await refused("writeFile", { path: "/etc/audit-probe.txt", content: secret }); + + const raw = readFileSync(logPath, "utf8"); + expect(raw).not.toContain("TOP-SECRET-PAYLOAD"); + expect(raw).toContain("writeFile"); + }); + + test("every decision produces exactly one line", async () => { + const before = entries().length; + + await executeLocalTool("listDirectory", { path: "." }, Mode.BUILD); + await refused("readFile", { path: "/etc/passwd" }); + + expect(entries().length).toBe(before + 2); + }); +}); diff --git a/packages/cli/src/lib/local-tools.readline.test.ts b/packages/cli/src/lib/local-tools.readline.test.ts new file mode 100644 index 0000000..43d0b22 --- /dev/null +++ b/packages/cli/src/lib/local-tools.readline.test.ts @@ -0,0 +1,56 @@ +/** + * `readLinesFromFile` length accounting. + * + * A range whose first selected line is empty used to undercount by one: the code + * treated `selectedLength === 0` as "nothing selected yet", so the *second* line + * skipped its separator. Length is not a proxy for selection. + */ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Mode } from "@mocode/shared"; +import { executeLocalTool } from "./local-tools"; + +const dir = mkdtempSync(join(tmpdir(), "mocode-readline-")); +const originalCwd = process.cwd(); + +// First line empty, then a body long enough to force truncation metadata. +const secondLine = "y".repeat(12_000); +const filePath = "big.txt"; +writeFileSync(join(dir, filePath), `\n${secondLine}\n`); + +beforeAll(() => { + process.chdir(dir); +}); + +afterAll(() => { + process.chdir(originalCwd); + rmSync(dir, { recursive: true, force: true }); +}); + +describe("readLinesFromFile", () => { + test("counts the separator even when the first selected line is empty", async () => { + const result = (await executeLocalTool( + "readFile", + { path: filePath, line_start: 1, line_end: 2 }, + Mode.BUILD, + )) as { content: string; truncated?: boolean; totalLength?: number }; + + expect(result.truncated).toBe(true); + // "" + "\n" + secondLine + expect(result.totalLength).toBe(secondLine.length + 1); + }); + + test("keeps the truncation budget aligned with the counted length", async () => { + const result = (await executeLocalTool( + "readFile", + { path: filePath, line_start: 1, line_end: 2 }, + Mode.BUILD, + )) as { content: string; truncated?: boolean }; + + // 10_000 char budget: the empty first line must not silently buy an extra cell. + expect(result.truncated).toBe(true); + expect(result.content.length).toBeLessThanOrEqual(10_000); + }); +}); diff --git a/packages/cli/src/lib/local-tools.sandbox.test.ts b/packages/cli/src/lib/local-tools.sandbox.test.ts new file mode 100644 index 0000000..9ddb06b --- /dev/null +++ b/packages/cli/src/lib/local-tools.sandbox.test.ts @@ -0,0 +1,132 @@ +/** + * Sandbox attack suite. + * + * Each case is an escape attempt that a model could plausibly emit. They assert on + * the real `executeLocalTool` entry point — the same function the agent calls — + * not on a helper, because the point is that the fence holds where it is used. + * + * Every command used here is harmless if a guard ever regresses: they either fail + * to resolve, or write to `/dev/null`. + */ +import { describe, expect, test } from "bun:test"; +import { Mode } from "@mocode/shared"; +import { executeLocalTool } from "./local-tools"; +import { isShellConfined } from "./sandbox-capability"; + +describe("path escapes are refused", () => { + test("absolute path outside the project", async () => { + await expect( + executeLocalTool("readFile", { path: "/etc/passwd" }, Mode.BUILD), + ).rejects.toThrow("Path is outside the project directory"); + }); + + test("parent traversal", async () => { + await expect( + executeLocalTool("readFile", { path: "../../../../../../etc/passwd" }, Mode.BUILD), + ).rejects.toThrow("Path is outside the project directory"); + }); + + test("writing outside the project", async () => { + await expect( + executeLocalTool("writeFile", { path: "/tmp/mocode-escape-probe.txt", content: "x" }, Mode.BUILD), + ).rejects.toThrow("Path is outside the project directory"); + }); + + test("listing outside the project", async () => { + await expect( + executeLocalTool("listDirectory", { path: "/etc" }, Mode.BUILD), + ).rejects.toThrow("Path is outside the project directory"); + }); +}); + +describe("hard-denied commands are refused even after approval", () => { + test("raw device write is refused at execution time", async () => { + // Harmless if the guard regresses (writes to /dev/null), but it matches the rule. + await expect( + executeLocalTool("bash", { command: "dd if=/dev/zero of=/dev/null count=1" }, Mode.BUILD), + ).rejects.toThrow("cannot be approved"); + }); + + test("filesystem formatting is refused at execution time", async () => { + await expect( + executeLocalTool("bash", { command: "mkfs.ext4 /dev/null" }, Mode.BUILD), + ).rejects.toThrow("cannot be approved"); + }); +}); + +describe("the shell cannot read secret trees when confined", () => { + // Host-dependent by nature: on a machine without seatbelt/bwrap the executor is + // unconfined, and the control that applies there is per-command approval — not + // this assertion. Declaring the precondition keeps the suite honest instead of + // failing on CI hosts that cannot sandbox. + test.skipIf(!isShellConfined())("reading /etc/passwd is refused by the OS sandbox", async () => { + const result = (await executeLocalTool( + "bash", + { command: "head -1 /etc/passwd 2>&1; echo EXIT:$?" }, + Mode.BUILD, + )) as { stdout: string }; + + // Confined hosts deny the read; unconfined hosts still fail closed via approval, + // but this suite runs the executor directly — assert the sandbox denial when present. + expect(result.stdout).not.toMatch(/^[^:\n]+:[^:\n]*:[0-9]+:/); + expect(result.stdout).toMatch(/Operation not permitted|Permission denied|EXIT:[1-9]/); + }); +}); + +describe("the shell does not inherit secrets", () => { + test("an API key in the CLI process is not visible to the agent's shell", async () => { + const canary = "sk-mocode-canary-should-not-leak"; + process.env.DEEPSEEK_API_KEY = canary; + + try { + const result = (await executeLocalTool( + "bash", + { command: 'echo -n "${DEEPSEEK_API_KEY:-absent}"' }, + Mode.BUILD, + )) as { stdout: string }; + + expect(result.stdout).toBe("absent"); + expect(result.stdout).not.toContain(canary); + } finally { + delete process.env.DEEPSEEK_API_KEY; + } + }); + + test("the shell still works — PATH is present", async () => { + const result = (await executeLocalTool("bash", { command: "echo -n $PATH" }, Mode.BUILD)) as { + stdout: string; + }; + + expect(result.stdout.length).toBeGreaterThan(0); + }); + + test("TERM is forced to dumb so escape codes never reach the TUI", async () => { + const result = (await executeLocalTool("bash", { command: "echo -n $TERM" }, Mode.BUILD)) as { + stdout: string; + }; + + expect(result.stdout).toBe("dumb"); + }); +}); + +describe("plan mode is still read-only at the executor", () => { + test("write tools are refused even if the server contract leaked through", async () => { + await expect( + executeLocalTool("writeFile", { path: "leak.txt", content: "x" }, Mode.PLAN), + ).rejects.toThrow("not available in PLAN mode"); + }); + + test("bash is refused", async () => { + await expect(executeLocalTool("bash", { command: "echo hi" }, Mode.PLAN)).rejects.toThrow( + "not available in PLAN mode", + ); + }); +}); + +describe("shell termination is reported with its real cause", () => { + test("a timeout says timeout — not 'the sandbox failed'", async () => { + await expect( + executeLocalTool("bash", { command: "sleep 5", timeout: 150 }, Mode.BUILD), + ).rejects.toThrow(/timed out/); + }); +}); diff --git a/packages/cli/src/lib/permission-rules.test.ts b/packages/cli/src/lib/permission-rules.test.ts new file mode 100644 index 0000000..774a10d --- /dev/null +++ b/packages/cli/src/lib/permission-rules.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, test } from "bun:test"; +import { + buildSandboxEnv, + evaluateToolPermission, + findHardDeny, + isHardDenied, +} from "./permission-rules"; + +describe("hard denies — not approvable in any mode", () => { + test("catches filesystem-destroying commands", () => { + const cases = [ + "rm -rf /", + "rm -rf /;", + "rm -rf /&&", + "rm -rf ~", + "rm -rf $HOME", + "mkfs.ext4 /dev/sda1", + "dd if=/dev/zero of=/dev/sda", + "cat /dev/zero > /dev/sda", + ":(){ :|:& };:", + ]; + + for (const command of cases) { + expect(findHardDeny(command)).not.toBeNull(); + } + }); + + test("leaves legitimate work alone", () => { + const safe = [ + "rm -rf ./dist", + "rm -rf node_modules", + "npm test", + "git status", + "chmod -R 755 ./scripts", + "curl -s https://example.com", + ]; + + for (const command of safe) { + expect(isHardDenied(command)).toBe(false); + } + }); + + test("reports why the command was refused", () => { + expect(findHardDeny("mkfs.ext4 /dev/sda1")).toEqual({ + id: "mkfs", + reason: "formatting a filesystem", + }); + }); +}); + +describe("evaluateToolPermission", () => { + test("read-only tools are allowed everywhere, including plan mode", () => { + for (const mode of ["plan", "default", "acceptEdits", "bypass"] as const) { + expect(evaluateToolPermission("readFile", mode).action).toBe("allow"); + expect(evaluateToolPermission("grep", mode).action).toBe("allow"); + expect(evaluateToolPermission("gitDiff", mode).action).toBe("allow"); + } + }); + + test("plan mode denies every write path", () => { + expect(evaluateToolPermission("writeFile", "plan")).toEqual({ + action: "deny", + reason: "plan mode is read-only", + }); + expect(evaluateToolPermission("editFile", "plan").action).toBe("deny"); + expect(evaluateToolPermission("bash", "plan").action).toBe("deny"); + }); + + test("default mode asks before writing", () => { + expect(evaluateToolPermission("writeFile", "default")).toEqual({ + action: "ask", + reason: "write requires approval", + }); + }); + + test("acceptEdits and bypass allow writing", () => { + expect(evaluateToolPermission("editFile", "acceptEdits").action).toBe("allow"); + expect(evaluateToolPermission("editFile", "bypass").action).toBe("allow"); + }); + + test("unknown tools ask rather than guess", () => { + expect(evaluateToolPermission("mcp__fs__write", "default").action).toBe("ask"); + }); +}); + +describe("buildSandboxEnv", () => { + const source = { + PATH: "/usr/bin", + HOME: "/Users/dev", + LANG: "zh_CN.UTF-8", + TERM: "xterm-256color", + DEEPSEEK_API_KEY: "sk-should-not-leak", + CLERK_SECRET_KEY: "sk_test_should_not_leak", + DATABASE_URL: "postgres://user:pass@host/db", + }; + + test("keeps the keys a shell legitimately needs", () => { + const env = buildSandboxEnv(source); + + expect(env.PATH).toBe("/usr/bin"); + expect(env.HOME).toBe("/Users/dev"); + expect(env.LANG).toBe("zh_CN.UTF-8"); + }); + + test("drops every secret", () => { + const env = buildSandboxEnv(source); + const serialized = JSON.stringify(env); + + expect(env.DEEPSEEK_API_KEY).toBeUndefined(); + expect(env.CLERK_SECRET_KEY).toBeUndefined(); + expect(env.DATABASE_URL).toBeUndefined(); + expect(serialized).not.toContain("sk-should-not-leak"); + expect(serialized).not.toContain("postgres://"); + }); + + test("forces TERM=dumb so escape codes never reach the TUI", () => { + expect(buildSandboxEnv(source).TERM).toBe("dumb"); + }); + + test("skips unset keys instead of writing undefined", () => { + const env = buildSandboxEnv({ PATH: "/bin" }); + expect(env.HOME).toBeUndefined(); + expect(Object.values(env).every((value) => typeof value === "string")).toBe(true); + }); +}); diff --git a/packages/cli/src/lib/permission-rules.ts b/packages/cli/src/lib/permission-rules.ts new file mode 100644 index 0000000..31425c4 --- /dev/null +++ b/packages/cli/src/lib/permission-rules.ts @@ -0,0 +1,189 @@ +/** + * Permission policy for agent tools. + * + * Two independent concerns live here: + * + * 1. **Hard denies** — commands that are refused even when the user clicks + * "approve". Approval is for *destructive but legitimate* work; it is not a + * licence to format a disk. The earlier blocklist could only *ask*, so a user + * clicking through the dialog could still run `mkfs`. + * + * 2. **Environment isolation** — bash used to inherit the full `process.env`, + * which meant the agent could read `DEEPSEEK_API_KEY`, `CLERK_SECRET_KEY` and + * every other secret the CLI was started with. The sandbox env is an explicit + * allowlist. + * + * Deliberately a policy module: no I/O, no React, no process spawning. + */ + +/** Coarse permission mode, mirroring the Plan/Build split the TUI already exposes. */ +export type PermissionMode = "plan" | "default" | "acceptEdits" | "bypass"; + +export type PermissionAction = "allow" | "deny" | "ask"; + +/** Machine-readable refusal reasons — audited and aggregatable. */ +export type PermissionCode = + | "PATH_ESCAPE" + | "HARD_DENY" + | "MODE_DENIED" + | "UNKNOWN_TOOL"; + +/** + * Refusal raised by the executor. + * + * The message stays human-readable (the model is told why), while `code` is what + * audit logs and dashboards group by. Prose messages drift; codes do not. + */ +export class ToolPermissionError extends Error { + constructor( + readonly code: PermissionCode, + message: string, + ) { + super(message); + this.name = "ToolPermissionError"; + } +} + +export type ToolPermissionDecision = { + action: PermissionAction; + reason: string; +}; + +/** Tools that only read state; safe in every mode, including Plan. */ +const READ_ONLY_TOOLS = new Set([ + "readFile", + "listDirectory", + "glob", + "grep", + "gitStatus", + "gitDiff", + "task", +]); + +/** Tools that mutate the project tree. */ +const WRITE_TOOLS = new Set(["writeFile", "editFile"]); + +/** + * Commands that are never executed, regardless of mode or approval. + * Kept narrow on purpose — a broad list would block legitimate cleanup work. + */ +const HARD_DENY_RULES: ReadonlyArray<{ id: string; pattern: RegExp; reason: string }> = [ + { + id: "rm-root", + // `rm -rf /;` and `rm -rf /&&` are just as final as `rm -rf /`. + pattern: /\brm\s+[^\n]*\s\/(?:\s|;|&|\||$)/, + reason: "deleting the filesystem root", + }, + { + id: "rm-home", + pattern: /\brm\s+[^\n]*\s(?:~|\$HOME)\/?\s?(?:$|;|\|)/, + reason: "deleting the home directory", + }, + { + id: "mkfs", + pattern: /\bmkfs(?:\.\w+)?\b/, + reason: "formatting a filesystem", + }, + { + id: "dd-device", + pattern: /\bdd\s+[^\n]*\bof=\/dev\//, + reason: "writing raw bytes to a device node", + }, + { + id: "dev-redirect", + pattern: />\s*\/dev\/(?:sd|disk|nvme|hd)/, + reason: "redirecting output into a block device", + }, + { + id: "fork-bomb", + pattern: /:\(\)\s*\{[^}]*\};?\s*:/, + reason: "fork bomb", + }, +]; + +/** Returns the matching hard-deny rule, or null when the command is merely risky. */ +export function findHardDeny(command: string): { id: string; reason: string } | null { + for (const rule of HARD_DENY_RULES) { + if (rule.pattern.test(command)) return { id: rule.id, reason: rule.reason }; + } + return null; +} + +export function isHardDenied(command: string): boolean { + return findHardDeny(command) !== null; +} + +/** + * Decide whether a tool may run in the current mode. + * + * Bash is intentionally `allow` here: the finer-grained "is this specific command + * dangerous" question is answered by `bash-approval.ts`, which drives the TUI + * dialog. This function only answers "may this *class* of tool run at all". + */ +export function evaluateToolPermission( + tool: string, + mode: PermissionMode, +): ToolPermissionDecision { + if (READ_ONLY_TOOLS.has(tool)) { + return { action: "allow", reason: "read-only" }; + } + + if (WRITE_TOOLS.has(tool)) { + switch (mode) { + case "plan": + return { action: "deny", reason: "plan mode is read-only" }; + case "acceptEdits": + return { action: "allow", reason: "acceptEdits mode" }; + case "bypass": + return { action: "allow", reason: "bypass mode" }; + case "default": + return { action: "ask", reason: "write requires approval" }; + } + } + + if (tool === "bash") { + return mode === "plan" + ? { action: "deny", reason: "plan mode is read-only" } + : { action: "allow", reason: "command checked against the blocklist separately" }; + } + + // MCP and unknown tools: ask rather than guess. + return { action: "ask", reason: "unrecognised tool" }; +} + +/** + * Environment keys a spawned shell may see. Everything else — API keys especially — + * stays out of the child process. + */ +export const SANDBOX_ENV_KEYS = [ + "PATH", + "HOME", + "LANG", + "LC_ALL", + "LC_CTYPE", + "SHELL", + "TMPDIR", + "TMP", + "USER", + "LOGNAME", + "PWD", +] as const; + +/** + * Build the child environment from an explicit allowlist. + * + * `TERM` is forced to `dumb` so escape sequences never leak into the TUI. + */ +export function buildSandboxEnv( + source: Record = process.env, +): Record { + const env: Record = {}; + + for (const key of SANDBOX_ENV_KEYS) { + const value = source[key]; + if (typeof value === "string") env[key] = value; + } + + env.TERM = "dumb"; + return env; +} diff --git a/packages/cli/src/lib/sandbox-capability.test.ts b/packages/cli/src/lib/sandbox-capability.test.ts new file mode 100644 index 0000000..ed08d0e --- /dev/null +++ b/packages/cli/src/lib/sandbox-capability.test.ts @@ -0,0 +1,110 @@ +import { describe, expect, test } from "bun:test"; +import { + detectSandbox, + probeBwrap, + probeSeatbelt, + sandboxCapability, + resetSandboxCapability, + type ProbeRunner, +} from "./sandbox-capability"; + +const applies: ProbeRunner = () => ({ exitCode: 0, output: "" }); +const refuses: ProbeRunner = () => ({ + exitCode: 1, + output: "sandbox-exec: sandbox_apply: Operation not permitted", +}); + +describe("seatbelt probe", () => { + test("available when the profile actually applies", () => { + expect(probeSeatbelt(applies)).toMatchObject({ backend: "seatbelt", available: true }); + }); + + test("unavailable when the binary exists but cannot apply", () => { + // The real-world case: /usr/bin/sandbox-exec is present, nesting is refused. + const capability = probeSeatbelt(refuses); + expect(capability.available).toBe(false); + expect(capability.detail).toContain("Operation not permitted"); + }); +}); + +describe("bwrap probe", () => { + test("available when a bind mount succeeds", () => { + expect(probeBwrap(applies)).toMatchObject({ backend: "bwrap", available: true }); + }); + + test("unavailable when bwrap is missing", () => { + const capability = probeBwrap(() => ({ exitCode: 127, output: "bwrap: command not found" })); + expect(capability.available).toBe(false); + }); +}); + +describe("detectSandbox", () => { + test("picks the backend by platform and never claims capability it cannot prove", () => { + const capability = detectSandbox(refuses); + expect(capability.available).toBe(false); + expect(["seatbelt", "bwrap", "none"]).toContain(capability.backend); + }); +}); + +describe("sandboxCapability memoisation", () => { + test("env opt-out reports unconfined without probing", () => { + resetSandboxCapability(); + process.env.MOCODE_ALLOW_UNCONFINED_SHELL = "1"; + try { + expect(sandboxCapability().available).toBe(false); + expect(sandboxCapability().detail).toContain("unconfined"); + } finally { + delete process.env.MOCODE_ALLOW_UNCONFINED_SHELL; + resetSandboxCapability(); + } + }); + + test("the answer is stable for the process lifetime", () => { + resetSandboxCapability(); + const first = sandboxCapability(); + const second = sandboxCapability(); + expect(second).toBe(first); + resetSandboxCapability(); + }); +}); + +describe("capability vs enforcement", () => { + test("a successful probe now reports enforcement, because the spawn site wraps it", () => { + const capability = probeSeatbelt(applies); + expect(capability.available).toBe(true); + expect(capability.enforced).toBe(true); + }); + + test("a failed probe reports neither", () => { + const capability = probeSeatbelt(refuses); + expect(capability.available).toBe(false); + expect(capability.enforced).toBe(false); + }); + + test("isShellConfined mirrors the live probe on this host", async () => { + const { detectSandbox, isShellConfined } = await import("./sandbox-capability"); + resetSandboxCapability(); + const capability = detectSandbox(); + expect(isShellConfined()).toBe(capability.available && capability.enforced); + }); + + test("sandboxBackend mirrors the live probe on this host", async () => { + const { detectSandbox, sandboxBackend } = await import("./sandbox-capability"); + resetSandboxCapability(); + const capability = detectSandbox(); + expect(sandboxBackend()).toBe( + capability.available && capability.enforced ? capability.backend : "none", + ); + }); + + test("MOCODE_DISABLE_SANDBOX forces none", async () => { + const { sandboxBackend } = await import("./sandbox-capability"); + process.env.MOCODE_DISABLE_SANDBOX = "1"; + try { + expect(sandboxBackend()).toBe("none"); + } finally { + delete process.env.MOCODE_DISABLE_SANDBOX; + resetSandboxCapability(); + } + }); +}); diff --git a/packages/cli/src/lib/sandbox-capability.ts b/packages/cli/src/lib/sandbox-capability.ts new file mode 100644 index 0000000..7cf0a90 --- /dev/null +++ b/packages/cli/src/lib/sandbox-capability.ts @@ -0,0 +1,174 @@ +/** + * Shell sandbox capability probe. + * + * Why this exists: path containment (`sandbox-path.ts`) only guards the file + * tools. A shell cannot be contained by path checks — `head /etc/passwd`, + * `cat link-to-etc`, `cd ..` all escape anything enforced in-process. The only + * real boundary is an OS-level sandbox. + * + * So the question is not "should bash be sandboxed" but "is this machine even + * capable of it". This module answers that once, at first use, and the rest of the + * system fails closed when the answer is no. + * + * Probes are injectable so tests do not depend on the host kernel. + */ + +import { BWRAP_BIN, SEATBELT_BIN } from "./sandbox-profile"; + +export type SandboxBackend = "seatbelt" | "bwrap" | "none"; + +export type SandboxCapability = { + backend: SandboxBackend; + /** The host *can* apply a sandbox (a probe actually ran successfully). */ + available: boolean; + /** + * The executor *does* apply it to spawned shells. + * + * Capability is not enforcement — reporting `available` as if it were confined + * is exactly the class of bug this whole module exists to prevent. Until the + * spawn site wraps the profile, every backend reports `enforced: false` and the + * approval policy stays fail-closed. + */ + enforced: boolean; + /** Why it is unavailable — surfaced in audit and diagnostics. */ + detail: string; +}; + +/** Runs a command and reports the exit code; split out so tests can fake it. */ +export type ProbeRunner = (command: string[]) => { exitCode: number; output: string }; + +const defaultRunner: ProbeRunner = (command) => { + try { + const result = Bun.spawnSync(command, { stdout: "pipe", stderr: "pipe" }); + return { + exitCode: result.exitCode ?? 1, + output: `${result.stdout.toString()}${result.stderr.toString()}`.trim(), + }; + } catch (error) { + return { exitCode: 1, output: error instanceof Error ? error.message : "probe failed" }; + } +}; + +/** + * Deny-default probe that mirrors the production seatbelt shape. + * + * `(allow default)` is too weak: it succeeds on hosts where the real + * deny-default profile SIGABRTs. Probe with deny-default + the broad read + * allow that modern macOS needs, so "available" means spawn will work. + */ +const SEATBELT_PROBE_PROFILE = [ + "(version 1)", + "(deny default)", + "(allow process*)", + "(allow sysctl-read)", + "(allow mach-lookup)", + '(allow file-read* (subpath "/"))', + "(allow file-map-executable)", +].join(""); + +/** + * macOS: `sandbox-exec` may exist yet fail to apply (nested sandbox, hardened + * runtime, SIP). Existence is not capability — we must run a deny-default profile. + */ +export function probeSeatbelt(run: ProbeRunner = defaultRunner): SandboxCapability { + const probe = run(["sandbox-exec", "-p", SEATBELT_PROBE_PROFILE, "/usr/bin/true"]); + + if (probe.exitCode === 0) { + return { + backend: "seatbelt", + available: true, + enforced: true, + detail: "sandbox-exec applies; profile applied at spawn", + }; + } + + return { + backend: "seatbelt", + available: false, + enforced: false, + detail: probe.output.slice(0, 120) || "sandbox-exec present but could not apply", + }; +} + +/** Linux: bubblewrap is the portable answer; a failed bind mount means no capability. */ +export function probeBwrap(run: ProbeRunner = defaultRunner): SandboxCapability { + const probe = run([BWRAP_BIN, "--ro-bind", "/", "/", "--dev", "/dev", "true"]); + + if (probe.exitCode === 0) { + return { backend: "bwrap", available: true, enforced: true, detail: "bwrap applies; profile applied at spawn" }; + } + + return { + backend: "bwrap", + available: false, + enforced: false, + detail: probe.output.slice(0, 120) || "bwrap unavailable", + }; +} + +/** Detect the platform's backend without caching. */ +export function detectSandbox(run: ProbeRunner = defaultRunner): SandboxCapability { + if (process.platform === "darwin") return probeSeatbelt(run); + if (process.platform === "linux") return probeBwrap(run); + + return { + backend: "none", + available: false, + enforced: false, + detail: `no sandbox backend on ${process.platform}`, + }; +} + +let cached: SandboxCapability | null = null; + +/** + * Cached capability for the process. + * + * An explicit opt-out exists because failing closed is heavy: when no sandbox is + * available every bash command needs approval. `MOCODE_ALLOW_UNCONFINED_SHELL=1` + * accepts that risk knowingly rather than silently. + */ +export function sandboxCapability(): SandboxCapability { + if (cached) return cached; + + if (process.env.MOCODE_ALLOW_UNCONFINED_SHELL === "1") { + cached = { backend: "none", available: false, enforced: false, detail: "unconfined shell allowed by env" }; + return cached; + } + + cached = detectSandbox(); + return cached; +} + +/** + * True only when a spawned shell is *actually* contained. + * + * Deliberately `available && enforced`: a host that can sandbox but whose spawn + * site does not wrap the profile is still unconfined. + */ +export function isShellConfined(): boolean { + // Must be derived from what the spawn site will actually do, not from what the + // host *can* do: `MOCODE_DISABLE_SANDBOX=1` turns confinement off without + // changing the probe result, and an unconfined shell must never be reported as + // confined — the approval policy keys off this. + return sandboxBackend() !== "none"; +} + +/** + * Backend the spawn site should use right now, or `none`. + * + * `MOCODE_DISABLE_SANDBOX=1` exists for environments where seatbelt/bwrap breaks + * legitimate work; it turns confinement off *and* leaves the approval policy + * fail-closed, because `isShellConfined()` then reports false. + */ +export function sandboxBackend(): SandboxBackend { + if (process.env.MOCODE_DISABLE_SANDBOX === "1") return "none"; + + const capability = sandboxCapability(); + return capability.available && capability.enforced ? capability.backend : "none"; +} + +/** Test seam: forget the memoised probe. */ +export function resetSandboxCapability(): void { + cached = null; +} diff --git a/packages/cli/src/lib/sandbox-path.test.ts b/packages/cli/src/lib/sandbox-path.test.ts new file mode 100644 index 0000000..f9b7a38 --- /dev/null +++ b/packages/cli/src/lib/sandbox-path.test.ts @@ -0,0 +1,92 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveInsideCwd } from "./sandbox-path"; + +const root = mkdtempSync(join(tmpdir(), "mocode-sandbox-")); +const outside = mkdtempSync(join(tmpdir(), "mocode-outside-")); + +writeFileSync(join(root, "inside.txt"), "ok"); +writeFileSync(join(outside, "secret.txt"), "top secret"); +mkdirSync(join(root, "nested"), { recursive: true }); + +// The escape hatch: a link that lives inside the project but points outside it. +symlinkSync(outside, join(root, "escape")); +// A link to a single outside file. +symlinkSync(join(outside, "secret.txt"), join(root, "escape-file")); +// A link whose target does not exist: walking past it would resolve the parent +// (inside cwd) and accept a path that writes through the link. +symlinkSync(join(outside, "missing-target"), join(root, "dangling")); + +afterAll(() => { + rmSync(root, { recursive: true, force: true }); + rmSync(outside, { recursive: true, force: true }); +}); + +describe("resolveInsideCwd — lexical escapes", () => { + test("accepts a plain relative path", () => { + expect(resolveInsideCwd(root, "inside.txt")).toBe(join(root, "inside.txt")); + }); + + test("accepts the project root itself", () => { + expect(resolveInsideCwd(root, ".")).toBe(root); + }); + + test("accepts a nested path that does not exist yet (writeFile case)", () => { + expect(resolveInsideCwd(root, "a/b/c/new.ts")).toBe(join(root, "a/b/c/new.ts")); + }); + + test("rejects parent traversal", () => { + expect(() => resolveInsideCwd(root, "../../etc/passwd")).toThrow( + "Path is outside the project directory", + ); + }); + + test("accepts a contained filename that merely begins with two dots", () => { + writeFileSync(join(root, "..config"), "ok"); + expect(resolveInsideCwd(root, "..config")).toBe(join(root, "..config")); + }); + + test("rejects traversal that lands back inside via a sibling prefix", () => { + // `..` must not be treated as a prefix of a legitimate directory name. + expect(() => resolveInsideCwd(root, "../mocode-sandbox-evil/x")).toThrow( + "Path is outside the project directory", + ); + }); + + test("rejects an absolute path outside the project", () => { + expect(() => resolveInsideCwd(root, "/etc/passwd")).toThrow( + "Path is outside the project directory", + ); + }); + + test("does not expand ~ into the home directory", () => { + // `~` is not special to the tool layer; it must stay a literal name inside cwd. + expect(resolveInsideCwd(root, "~/.ssh/id_rsa")).toBe(join(root, "~/.ssh/id_rsa")); + }); +}); + +describe("resolveInsideCwd — symlink escapes", () => { + test("rejects a directory symlink pointing outside the project", () => { + expect(() => resolveInsideCwd(root, "escape/secret.txt")).toThrow( + "Path is outside the project directory", + ); + }); + + test("rejects a file symlink pointing outside the project", () => { + expect(() => resolveInsideCwd(root, "escape-file")).toThrow( + "Path is outside the project directory", + ); + }); + + test("rejects a dangling symlink instead of walking past it", () => { + expect(() => resolveInsideCwd(root, "dangling")).toThrow( + "Path is outside the project directory", + ); + }); + + test("still accepts a normal nested directory", () => { + expect(resolveInsideCwd(root, "nested/file.ts")).toBe(join(root, "nested/file.ts")); + }); +}); diff --git a/packages/cli/src/lib/sandbox-path.ts b/packages/cli/src/lib/sandbox-path.ts new file mode 100644 index 0000000..2cd03ee --- /dev/null +++ b/packages/cli/src/lib/sandbox-path.ts @@ -0,0 +1,85 @@ +/** + * Path containment for agent tools. + * + * The previous check was purely lexical: `resolve()` then reject anything starting + * with `..`. That stops `../../etc/passwd` but not a symlink, because the symlink + * itself lives inside the project while pointing outside it: + * + * ln -s /etc ./docs + * readFile("docs/passwd") ← lexically inside, actually /etc/passwd + * + * This module checks both: the lexical path *and* the real path of the deepest + * existing ancestor (so it works for files that do not exist yet, e.g. writeFile + * creating a new directory tree). + */ +import { lstatSync, realpathSync } from "node:fs"; +import { dirname, isAbsolute, relative, resolve, sep } from "node:path"; + +function outsideError(): Error { + return new Error("Path is outside the project directory"); +} + +/** Throw unless `candidate` is `root` or lives beneath it. */ +function assertInside(root: string, candidate: string): void { + const rel = relative(root, candidate); + if (rel === "") return; + // Reject only the parent segment itself (`..`) or a parent walk (`../…`). + // Filenames that merely begin with two dots (e.g. `..config`) are contained. + if (rel === ".." || rel.startsWith(`..${sep}`) || isAbsolute(rel)) throw outsideError(); +} + +/** Real path, or the input when the file does not exist. */ +function realpathOrSelf(path: string): string { + try { + return realpathSync(path); + } catch { + return path; + } +} + +/** + * Real path of the deepest ancestor that exists. Resolving the ancestor (rather + * than the leaf) keeps this usable for paths that are about to be created. + */ +function realAncestor(path: string): string { + let current = path; + + while (true) { + try { + return realpathSync(current); + } catch { + // A dangling symlink is NOT a "missing component". Walking past it would + // resolve the parent (inside cwd) and accept a path that actually writes + // through the link to a target outside the project. + if (isDanglingSymlink(current)) throw outsideError(); + + const parent = dirname(current); + if (parent === current) return current; + current = parent; + } + } +} + +/** True when `path` exists as a symlink whose target cannot be resolved. */ +function isDanglingSymlink(path: string): boolean { + try { + return lstatSync(path).isSymbolicLink(); + } catch { + return false; + } +} + +/** + * Resolve `target` under `cwd`, rejecting both lexical escapes and symlink escapes. + * + * @throws when the path leaves the project directory by either route. + */ +export function resolveInsideCwd(cwd: string, target: string): string { + const resolved = resolve(cwd, target); + assertInside(cwd, resolved); + + // Symlink check: compare real paths so `docs -> /etc` is caught. + assertInside(realpathOrSelf(cwd), realAncestor(resolved)); + + return resolved; +} diff --git a/packages/cli/src/lib/sandbox-profile.test.ts b/packages/cli/src/lib/sandbox-profile.test.ts new file mode 100644 index 0000000..af1d6e5 --- /dev/null +++ b/packages/cli/src/lib/sandbox-profile.test.ts @@ -0,0 +1,124 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + buildBwrapArgs, + buildSeatbeltProfile, + parsePathList, + runtimePaths, + spawnPlan, +} from "./sandbox-profile"; + +const base = { cwd: "/Users/dev/project", home: "/Users/dev", tmpDir: "/tmp" }; + +describe("seatbelt profile", () => { + const profile = buildSeatbeltProfile(base); + + test("denies by default — that is the whole guarantee", () => { + expect(profile).toContain("(version 1)"); + expect(profile.split("\n")[1]).toBe("(deny default)"); + }); + + test("allows the project tree", () => { + expect(profile).toContain('(subpath "/Users/dev/project")'); + }); + + test("allows broad reads under / (required on modern macOS) then denies secrets", () => { + // Narrow subpath allowlists SIGABRT on dyld/cryptex hosts; deny comes after. + expect(profile).toContain('(allow file-read* (subpath "/"))'); + expect(profile).toContain('(subpath "/etc/passwd")'); + expect(profile).toContain('(subpath "/Users/dev/.ssh")'); + const denyLine = profile.split("\n").find((line) => line.startsWith("(deny file-read*")); + expect(denyLine).toBeDefined(); + }); + + test("keeps the writable set tight", () => { + const writeLine = profile.split("\n").find((line) => line.startsWith("(allow file-write*")); + expect(writeLine).toContain('(subpath "/Users/dev/project")'); + expect(writeLine).not.toContain('(subpath "/Users/dev")'); + }); + + test("escapes quotes so a hostile directory name cannot break the profile", () => { + const tricky = buildSeatbeltProfile({ cwd: '/tmp/we"ird\\(dir)' }); + expect(tricky).toContain('/tmp/we\\"ird\\\\(dir)'); + }); + + test("lists the project once in the write allow", () => { + const duplicated = buildSeatbeltProfile({ + ...base, + extraWritePaths: ["/Users/dev/project", "/tmp/extra"], + }); + const occurrences = duplicated.split('(subpath "/Users/dev/project")').length - 1; + expect(occurrences).toBe(1); + }); +}); + +describe("bwrap args", () => { + // bwrap refuses to start when a bind target is missing, so masks are only emitted + // for paths that exist. Build a throwaway home that has them. + const fakeHome = mkdtempSync(join(tmpdir(), "mocode-bwrap-home-")); + mkdirSync(join(fakeHome, ".ssh"), { recursive: true }); + writeFileSync(join(fakeHome, ".npmrc"), "//registry/:_authToken=x"); + + const args = buildBwrapArgs({ ...base, home: fakeHome }); + + test("mounts a read-only root and a writable project", () => { + expect(args.join(" ")).toContain("--ro-bind / /"); + expect(args.join(" ")).toContain("--bind /Users/dev/project /Users/dev/project"); + }); + + test("masks sensitive home paths", () => { + const joined = args.join(" "); + // Directories are emptied with a tmpfs; binding a *file* over a directory makes + // bubblewrap refuse to start. + expect(joined).not.toContain(`--ro-bind /dev/null ${fakeHome}/.ssh`); + // Files are bound over /dev/null. + expect(joined).toContain(`--ro-bind /dev/null ${fakeHome}/.npmrc`); + // Directories are emptied instead. + expect(joined).toContain(`--tmpfs ${fakeHome}/.ssh`); + }); + + test("always mounts a private tmpfs for TMPDIR", () => { + expect(args.join(" ")).toContain("--tmpfs /tmp"); + }); + + test("does not share the parent's pid namespace", () => { + expect(args).not.toContain("--unshare-pid"); + }); + + afterAll(() => rmSync(fakeHome, { recursive: true, force: true })); +}); + + +describe("spawnPlan", () => { + test("seatbelt wraps bash in sandbox-exec", () => { + const plan = spawnPlan("seatbelt", "echo hi", base); + expect(plan).toMatchObject({ file: "/usr/bin/sandbox-exec", confined: true, backend: "seatbelt" }); + expect(plan.args.slice(-3)).toEqual(["bash", "-c", "echo hi"]); + expect(plan.args[0]).toBe("-p"); + }); + + test("bwrap prefixes the command", () => { + const plan = spawnPlan("bwrap", "echo hi", base); + expect(plan).toMatchObject({ file: "/usr/bin/bwrap", confined: true }); + expect(plan.args.slice(-3)).toEqual(["bash", "-c", "echo hi"]); + }); + + test("none is honest: returns a working shell and says it is unconfined", () => { + const plan = spawnPlan("none", "echo hi", base); + expect(plan).toMatchObject({ file: "bash", confined: false, backend: "none" }); + expect(plan.args).toEqual(["-c", "echo hi"]); + }); +}); + +describe("helpers", () => { + test("parsePathList splits, trims and drops empties", () => { + expect(parsePathList("/a:/b : /c::")).toEqual(["/a", "/b", "/c"]); + expect(parsePathList(undefined)).toEqual([]); + }); + + test("runtimePaths keeps the interpreter directory readable", () => { + expect(runtimePaths("/Users/dev/.bun/bin/bun")).toEqual(["/Users/dev/.bun/bin"]); + }); +}); diff --git a/packages/cli/src/lib/sandbox-profile.ts b/packages/cli/src/lib/sandbox-profile.ts new file mode 100644 index 0000000..9283d2b --- /dev/null +++ b/packages/cli/src/lib/sandbox-profile.ts @@ -0,0 +1,247 @@ +/** + * OS-level sandbox profiles and spawn plans. + * + * The path checks in `sandbox-path.ts` cannot contain a shell — this module is what + * actually does. Given a backend, it produces either an SBPL profile (macOS) or a + * bubblewrap argument list (Linux), plus the exact argv to spawn. + * + * Everything here is a pure function on purpose: a sandbox whose profile is wrong + * fails silently in the worst possible direction, so the profile is asserted in + * tests rather than eyeballed in production. + * + * Policy: **deny by default, then allow what a shell needs, then deny secrets.** + * On modern macOS a narrow read-allowlist SIGABRTs (dyld/cryptex paths), so reads + * are allowed under `/` and sensitive trees are denied afterward. Writes stay + * limited to the project and tmp. + */ +import { existsSync } from "node:fs"; +import { dirname } from "node:path"; + +/** Sandbox binaries are addressed absolutely: a bare name resolves against the + * allowlisted env we hand the child, and a miss looks like "the sandbox broke". */ +export const SEATBELT_BIN = "/usr/bin/sandbox-exec"; +export const BWRAP_BIN = "/usr/bin/bwrap"; + +export type SandboxBackend = "seatbelt" | "bwrap" | "none"; + +export type SandboxProfileOptions = { + /** Project root: the only tree that is writable. */ + cwd: string; + /** Used for tmp paths and the extra allowlist defaults. */ + home?: string; + tmpDir?: string; + /** Additional read-only trees (colon-separated `MOCODE_SANDBOX_EXTRA_READ`). */ + extraReadPaths?: string[]; + /** Additional writable trees (`MOCODE_SANDBOX_EXTRA_WRITE`). */ + extraWritePaths?: string[]; + /** Runtime install dirs that must stay readable (bun/node themselves). */ + runtimePaths?: string[]; +}; + +/** Escape a path for SBPL string literals. */ +function sbplString(value: string): string { + return `"${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`; +} + +function uniq(values: string[]): string[] { + return [...new Set(values.filter((value) => value.length > 0))]; +} + +/** Split a colon-separated env var into paths. */ +export function parsePathList(value: string | undefined): string[] { + if (!value) return []; + return value + .split(":") + .map((entry) => entry.trim()) + .filter(Boolean); +} + +/** + * Sensitive paths denied after the broad filesystem read allow. + * + * Mirrors the bwrap mask list so both backends refuse the same secret trees. + */ +function deniedReadPaths(home: string | undefined): string[] { + const system = [ + "/etc/passwd", + "/etc/shadow", + "/etc/sudoers", + "/private/etc/passwd", + "/private/etc/shadow", + "/private/etc/sudoers", + "/var/root", + "/root", + ]; + const user = home + ? [ + `${home}/.ssh`, + `${home}/.aws`, + `${home}/.gnupg`, + `${home}/.config/gh`, + `${home}/.config/gcloud`, + `${home}/.kube`, + `${home}/.docker`, + // Package-manager config routinely carries registry tokens. + `${home}/.npmrc`, + `${home}/.yarnrc`, + `${home}/.pypirc`, + `${home}/.netrc`, + ] + : []; + return uniq([...system, ...user]); +} + +/** + * Build the macOS seatbelt profile. + * + * `(deny default)` first. Broad `(allow file-read* (subpath "/"))` is required on + * current macOS — a narrow subpath allowlist aborts the child before it starts. + * Sensitive paths are denied afterward (later SBPL rules win for the same op). + * Writes stay limited to the project / tmp. + */ +export function buildSeatbeltProfile(options: SandboxProfileOptions): string { + const { cwd } = options; + const writePaths = uniq([ + cwd, + ...(options.tmpDir ? [options.tmpDir] : []), + ...(options.extraWritePaths ?? []), + "/dev/null", + "/dev/tty", + ]); + const denied = deniedReadPaths(options.home); + + const writeRules = writePaths.map((path) => `(subpath ${sbplString(path)})`).join(" "); + const denyRules = denied.map((path) => `(subpath ${sbplString(path)})`).join(" "); + + return [ + "(version 1)", + "(deny default)", + // Process control is required by any shell. + "(allow process*)", + "(allow signal)", + "(allow sysctl-read)", + // Lookups for system services; keep to the operation names that have been + // stable across macOS releases. An unknown operation makes `sandbox_apply` + // fail as a whole, which shows up as "the command produced no output". + "(allow mach-lookup)", + "(allow file-read-metadata)", + // Broad read: narrow allowlists SIGABRT on dyld/cryptex hosts. + '(allow file-read* (subpath "/"))', + // Executables must be mappable, otherwise the dynamic loader fails. + "(allow file-map-executable)", + `(allow file-write* ${writeRules})`, + "(allow network*)", + // Later rules win — block the secret trees after the broad allow. + `(deny file-read* ${denyRules})`, + ].join("\n"); +} + +/** + * Masked paths are bound to `/dev/null` so the sandbox cannot read them even + * though bwrap's default is a full read-only filesystem view. + */ +const BWRAP_MASKED_FILES = [ + "/etc/shadow", + "/etc/sudoers", + ".netrc", + ".npmrc", + ".yarnrc", + ".pypirc", +] as const; + +/** Directories need an empty dir, not a file bind: binding a file over a + * directory makes bubblewrap refuse to start. */ +const BWRAP_MASKED_DIRS = [".ssh", ".aws", ".gnupg", ".config/gh", ".config/gcloud", ".kube", ".docker"] as const; + +/** + * Build the bubblewrap argument list. + * + * bwrap cannot express "deny by default" without rebuilding the whole root, so the + * approach is: read-only root, writable project, and sensitive paths masked. + */ +export function buildBwrapArgs(options: SandboxProfileOptions): string[] { + const { cwd } = options; + const args = [ + "--die-with-parent", + "--ro-bind", + "/", + "/", + "--bind", + cwd, + cwd, + "--dev", + "/dev", + "--proc", + "/proc", + "--chdir", + cwd, + ]; + + for (const extra of options.extraWritePaths ?? []) { + args.push("--bind", extra, extra); + } + + for (const masked of BWRAP_MASKED_FILES) { + const path = masked.startsWith("/") ? masked : options.home ? `${options.home}/${masked}` : null; + if (path && existsSync(path)) args.push("--ro-bind", "/dev/null", path); + } + + // Absent dirs are skipped: bwrap fails the whole sandbox if a target is missing. + for (const masked of BWRAP_MASKED_DIRS) { + const path = masked.startsWith("/") ? masked : options.home ? `${options.home}/${masked}` : null; + if (path && existsSync(path)) args.push("--tmpfs", path); + } + + // Private scratch space: without this, TMPDIR points at a read-only host path. + args.push("--tmpfs", "/tmp"); + + return args; +} + +export type SpawnPlan = { + file: string; + args: string[]; + /** Recorded in the audit log so "confined" is provable after the fact. */ + confined: boolean; + backend: SandboxBackend; +}; + +/** + * Decide how to spawn a shell command. + * + * `none` is the honest fallback: the caller still gets a working shell, and the + * approval policy stays fail-closed because `confined` is false. + */ +export function spawnPlan( + backend: SandboxBackend, + command: string, + options: SandboxProfileOptions, +): SpawnPlan { + if (backend === "seatbelt") { + return { + // Absolute path on purpose: `Bun.spawn` resolves a bare name against the env + // it is given, and we hand it an allowlisted env. A resolution miss looks + // exactly like "the sandbox broke" (exit null, empty output). + file: SEATBELT_BIN, + args: ["-p", buildSeatbeltProfile(options), "bash", "-c", command], + confined: true, + backend, + }; + } + + if (backend === "bwrap") { + return { + file: BWRAP_BIN, + args: [...buildBwrapArgs(options), "bash", "-c", command], + confined: true, + backend, + }; + } + + return { file: "bash", args: ["-c", command], confined: false, backend: "none" }; +} + +/** Runtime dirs that must stay readable so `bun`/`node` can start inside the sandbox. */ +export function runtimePaths(execPath: string = process.execPath): string[] { + return uniq([dirname(execPath)]); +} From 21653b5b798287941f7d1be138f34cc22ce5fc2d Mon Sep 17 00:00:00 2001 From: moyun Date: Thu, 12 Nov 2026 09:35:35 +0800 Subject: [PATCH 05/11] feat(server): bound the context with deterministic compaction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The server replays the whole session every turn, so the prompt grew linearly with session length — 139k tokens by turn 50 in the bench. - fold everything older than `keepRecent` into one digest message once the budget is exceeded; the digest is deterministic (tool counts, files touched), not an LLM summary: no extra call, no extra latency, no extra failure mode - cut on a user boundary so a tool call is never separated from its result; if no safe boundary exists, skip compaction rather than cut through the active turn - compact only the *model input* — persisted history and the client's own transcript stay complete - report `compaction: { tokensBefore, tokensAfter }` in the finish metadata 50 turns: 139,010 -> 14,011 tokens (-90%), flat after turn ~24. Shared so BYOK can use the same policy. --- docs/context-budget.md | 90 +++++++ .../scripts/context-compaction-bench.ts | 76 ++++++ packages/server/src/routes/chat.ts | 21 +- .../shared/src/context-compaction.test.ts | 213 ++++++++++++++++ packages/shared/src/context-compaction.ts | 229 ++++++++++++++++++ packages/shared/src/index.ts | 11 + 6 files changed, 638 insertions(+), 2 deletions(-) create mode 100644 docs/context-budget.md create mode 100644 packages/server/scripts/context-compaction-bench.ts create mode 100644 packages/shared/src/context-compaction.test.ts create mode 100644 packages/shared/src/context-compaction.ts diff --git a/docs/context-budget.md b/docs/context-budget.md new file mode 100644 index 0000000..f64c20e --- /dev/null +++ b/docs/context-budget.md @@ -0,0 +1,90 @@ +# Context budget & compaction + +The server replays the stored session on every turn, so the prompt grows linearly +with session length. Left alone, a long session either overflows the model window +or pays for history nobody re-reads. + +Compaction puts a **ceiling** on that growth: past the budget, old turns collapse +into one digest message. + +## Policy + +```ts +{ maxTokens: 60_000, keepRecent: 10, minMessages: 16 } +``` + +- `maxTokens` is a **cost** budget, not a context-window budget. DeepSeek Flash + accepts 1M tokens; 60k is where continued growth stops being worth paying for. +- Below `minMessages` compaction is skipped — a short conversation is mostly + intent, and folding it would destroy the task. + +## What the digest contains + +``` +[compacted] 90 earlier messages folded away (45 user turns). +Tools already used: readFile×23, bash×22. +Files touched: src/module-0.ts, src/module-1.ts, … +Those turns are no longer in context; re-read a file if you need its exact contents. +``` + +It is **deterministic, not an LLM summary**. That is a deliberate trade: + +| | deterministic digest | LLM summary | +|---|---|---| +| cost | 0 | one extra model call per compaction | +| latency | 0 | +1 round trip | +| failure mode | none | the summarizer can itself fail or hallucinate | +| fidelity | what tools ran, which files were touched | prose semantics | + +What the agent actually needs from old turns is *what it already did* — tool names, +file paths — not the prose around it. The last line matters more than it looks: it +tells the model the content is **gone**, so it re-reads instead of inventing. + +## Measured + +```bash +bun run packages/server/scripts/context-compaction-bench.ts +``` + +Synthetic but realistically sized: one tool call per turn, 10k-char output (the +`readFile` truncation limit). + +| turns | prompt tokens (no compaction) | with compaction | +|---|---|---| +| 20 | 55,580 | 55,580 (not yet triggered) | +| 25 | 69,485 | 14,011 (−80%) | +| 50 | **139,010** | **14,011 (−90%)** | + +The number that matters is not the −90% at turn 50, it is that the prompt is +**flat** after turn ~24. Without it, cost per turn grows linearly forever. + +## Design rules + +1. **The cut lands on a user message.** Splitting a tool call from its result would + produce an invalid conversation, so `findCutIndex` snaps forward to the next + `user` role. +2. **Compaction shapes model input only.** The persisted message list and the + messages echoed back to the client stay complete — otherwise the user's own + transcript would lose history. +3. **It is bounded.** Running compaction twice does not compound; a covered test + asserts the second pass never increases tokens. +4. **It is observable.** When a turn is compacted, the finish metadata carries + `compaction: { tokensBefore, tokensAfter }`. + +## Manual acceptance + +1. Start a session, let it run past ~25 tool calls with large files. +2. The finish metadata of that turn should carry `compaction` (visible in the SSE + frame or server logs). +3. Continue the session — the prompt estimate stays flat instead of climbing. +4. Ask the agent about something from the compacted range. Expected: it re-reads + the file rather than fabricating details. That is the digest working. + +## Follow-ups + +- ~~BYOK has no compaction~~ — done: the module moved to `@mocode/shared` and both + the server route and `LocalChatTransport` apply the same policy. +- **Tool-output truncation is still flat** (10k/20k chars per tool). Per-tool, + budget-aware limits would squeeze more out of a turn. +- **No eviction of stale file reads.** Re-reading the same file repeatedly still + costs full tokens each time. diff --git a/packages/server/scripts/context-compaction-bench.ts b/packages/server/scripts/context-compaction-bench.ts new file mode 100644 index 0000000..a09bf4a --- /dev/null +++ b/packages/server/scripts/context-compaction-bench.ts @@ -0,0 +1,76 @@ +/** + * Context-compaction bench. + * + * Measures what bounds the prompt actually buys on a long session. No API calls — + * the conversation is synthetic but sized like real agent traffic (tool outputs + * dominate, which is the whole reason compaction matters). + * + * Run from the repo root: + * bun run packages/server/scripts/context-compaction-bench.ts + */ +import { + DEFAULT_COMPACTION_POLICY, + compactMessages, + estimateMessagesTokens, +} from "@mocode/shared"; + +type Msg = { id: string; role: string; parts: unknown[] }; + +/** One user turn plus the agent's tool round-trip, sized like real traffic. */ +function turn(index: number): Msg[] { + return [ + { + id: `u${index}`, + role: "user", + parts: [{ type: "text", text: `检查 src/module-${index}.ts 的实现并说明问题。`.repeat(6) }], + }, + { + id: `a${index}`, + role: "assistant", + parts: [ + { type: index % 2 === 0 ? "tool-readFile" : "tool-bash", input: { path: `src/module-${index}.ts` }, output: "x".repeat(10_000) }, + { type: "text", text: `模块 ${index} 的问题是边界条件缺失。`.repeat(12) }, + ], + }, + ]; +} + +function session(turns: number): Msg[] { + return Array.from({ length: turns }, (_, index) => turn(index)).flat(); +} + +const TURNS = 50; +const messages = session(TURNS); + +console.log(`=== ${TURNS} 轮会话(每轮 1 次工具调用 + 10k 字符输出(= readFile 截断上限))===\n`); +console.log("turn".padEnd(8), "无压缩 prompt tokens".padEnd(24), "有压缩"); +console.log("-".repeat(60)); + +for (let turns = 5; turns <= TURNS; turns += 5) { + const slice = messages.slice(0, turns * 2); + const before = estimateMessagesTokens(slice); + const compacted = compactMessages(slice, DEFAULT_COMPACTION_POLICY); + const after = compacted.compacted ? compacted.tokensAfter : before; + const saved = before === 0 ? 0 : Math.round(((before - after) / before) * 100); + + console.log( + String(turns).padEnd(8), + String(before).padEnd(24), + `${after}${compacted.compacted ? ` (-${saved}%)` : ""}`, + ); +} + +const before = estimateMessagesTokens(messages); +const result = compactMessages(messages, DEFAULT_COMPACTION_POLICY); +const reduction = Math.round(((before - result.tokensAfter) / before) * 100); + +console.log("\n=== 结论(完整 50 轮) ==="); +console.log("压缩触发:", result.compacted ? "是" : "否"); +console.log("prompt tokens:", before, "→", result.tokensAfter, `(-${reduction}%)`); +console.log("消息条数:", messages.length, "→", result.messages.length); +console.log("策略:", JSON.stringify(DEFAULT_COMPACTION_POLICY)); + +if (result.digest) { + console.log("\n=== 折叠出的摘要(进上下文的就是这一段) ==="); + console.log(result.digest.slice(0, 400)); +} diff --git a/packages/server/src/routes/chat.ts b/packages/server/src/routes/chat.ts index 451deb6..0cd156e 100644 --- a/packages/server/src/routes/chat.ts +++ b/packages/server/src/routes/chat.ts @@ -2,7 +2,10 @@ * Chat HTTP routes: submit a user turn and stream the assistant reply via SSE. * * Phase 8 extends the stream with: - * - Multi-step agent loop via `streamText({ tools, stopWhen: stepCountIs(50) })` + * - Exactly one model step per request: tools carry no executors here, so the + * stream ends after the model asks for tools. The CLI runs them and re-posts; + * the loop budget (turns, tool caps, repeat guard) lives in + * `packages/cli/src/lib/agent-loop.ts`. * - Reasoning, tool-call, and tool-result SSE events (mirrored in Message.parts) * - System prompt and cwd-scoped tools when session.cwd is set * @@ -63,12 +66,15 @@ import { } from "../lib/stream-interrupt"; import { shouldPersistOnFinish } from "../lib/chat-abort"; import { resolveSubagentChatFinish } from "../lib/chat-subagent"; +import { DEFAULT_COMPACTION_POLICY, compactMessages } from "@mocode/shared"; type ChatMessageMetadata = { mode?: ModeType; model?: string; durationMs?: number; usage?: LanguageModelUsage; + /** Present when old turns were folded to stay inside the context budget. */ + compaction?: { tokensBefore: number; tokensAfter: number }; }; type MocodeUIMessage = UIMessage>; @@ -179,7 +185,10 @@ const app = new Hono() messages: stripIncompleteAssistantMessages(mergedMessages), tools, }); - const modelMessages = await convertToModelMessages(nextMessages, { tools }); + // Compaction shapes only the *model input*. The client's message list stays + // intact and is still what gets persisted and echoed back. + const compaction = compactMessages(nextMessages, DEFAULT_COMPACTION_POLICY); + const modelMessages = await convertToModelMessages(compaction.messages, { tools }); let completedUsage: LanguageModelUsage | null = null; const replayBuffer = new StreamReplayBuffer(); @@ -208,6 +217,14 @@ const app = new Hono() model, durationMs: Date.now() - startTime, ...(completedUsage ? { usage: completedUsage } : {}), + ...(compaction.compacted + ? { + compaction: { + tokensBefore: compaction.tokensBefore, + tokensAfter: compaction.tokensAfter, + }, + } + : {}), }; }, consumeSseStream: ({ stream }) => { diff --git a/packages/shared/src/context-compaction.test.ts b/packages/shared/src/context-compaction.test.ts new file mode 100644 index 0000000..cbe2a76 --- /dev/null +++ b/packages/shared/src/context-compaction.test.ts @@ -0,0 +1,213 @@ +import { describe, expect, test } from "bun:test"; +import { + DEFAULT_COMPACTION_POLICY, + compactMessages, + estimateMessageTokens, + estimateMessagesTokens, + estimateTextTokens, +} from "./context-compaction"; + +type Msg = { id: string; role: string; parts: unknown[] }; + +function user(id: string, text: string): Msg { + return { id, role: "user", parts: [{ type: "text", text }] }; +} + +function assistantText(id: string, text: string): Msg { + return { id, role: "assistant", parts: [{ type: "text", text }] }; +} + +function assistantTool(id: string, name: string, input: unknown, output: string): Msg { + return { + id, + role: "assistant", + parts: [ + { type: `tool-${name}`, input, output }, + { type: "text", text: `called ${name}` }, + ], + }; +} + +describe("estimateTextTokens", () => { + test("empty text is free", () => { + expect(estimateTextTokens("")).toBe(0); + }); + + test("ascii is roughly 4 chars per token", () => { + expect(estimateTextTokens("abcd")).toBe(1); + expect(estimateTextTokens("a".repeat(400))).toBe(100); + }); + + test("cjk costs about one token per character", () => { + expect(estimateTextTokens("修仙欠费中")).toBe(5); + }); + + test("mixed content adds both", () => { + expect(estimateTextTokens("abcd修仙")).toBe(3); + }); +}); + +describe("estimateMessageTokens", () => { + test("counts text parts", () => { + expect(estimateMessageTokens(user("u", "hello world"))).toBeGreaterThan(0); + }); + + test("counts tool input and output payloads", () => { + const small = estimateMessageTokens(assistantTool("a", "readFile", { path: "a.ts" }, "x")); + const large = estimateMessageTokens( + assistantTool("a", "readFile", { path: "a.ts" }, "x".repeat(4_000)), + ); + + expect(large).toBeGreaterThan(small); + }); + + test("a message with no parts costs nothing", () => { + expect(estimateMessageTokens({ id: "x", role: "user" })).toBe(0); + }); + + test("conversation total is the sum", () => { + const messages = [user("u", "abcd"), assistantText("a", "abcd")]; + expect(estimateMessagesTokens(messages)).toBe( + estimateMessageTokens(messages[0]!) + estimateMessageTokens(messages[1]!), + ); + }); +}); + +/** 20 turns of realistic, token-heavy history. */ +function longConversation(): Msg[] { + const messages: Msg[] = []; + + for (let turn = 0; turn < 20; turn += 1) { + messages.push(user(`u${turn}`, `请检查第 ${turn} 个文件并说明问题`.padEnd(200, "。"))); + messages.push( + assistantTool( + `a${turn}`, + turn % 2 === 0 ? "bash" : "readFile", + { path: `src/module-${turn}.ts`, command: "ls" }, + "x".repeat(2_000), + ), + ); + } + + return messages; +} + +describe("compactMessages", () => { + test("leaves a small conversation alone", () => { + const messages = [user("u", "hi"), assistantText("a", "hello")]; + const result = compactMessages(messages); + + expect(result.compacted).toBe(false); + expect(result.messages).toBe(messages); + expect(result.digest).toBeNull(); + }); + + test("does not compact when the message count is still low", () => { + const messages = [user("u", "hi"), assistantText("a", "x".repeat(1_000_000))]; + const result = compactMessages(messages, { ...DEFAULT_COMPACTION_POLICY, maxTokens: 10 }); + + expect(result.compacted).toBe(false); + }); + + test("folds old turns once the budget is exceeded", () => { + const messages = longConversation(); + const result = compactMessages(messages, { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 6, + minMessages: 8, + }); + + expect(result.compacted).toBe(true); + expect(result.tokensAfter).toBeLessThan(result.tokensBefore); + expect(result.messages.length).toBeLessThan(messages.length); + }); + + test("keeps the most recent messages verbatim", () => { + const messages = longConversation(); + const result = compactMessages(messages, { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 4, + minMessages: 8, + }); + + const kept = result.messages.slice(1); + expect(kept).toEqual(messages.slice(messages.length - kept.length)); + }); + + test("never splits a user turn from what followed it", () => { + const messages = longConversation(); + const result = compactMessages(messages, { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 7, + minMessages: 8, + }); + + // Index 0 is the digest; everything after it must start on a user message. + expect(result.messages[1]?.role).toBe("user"); + }); + + test("never compacts when there is no user boundary to cut at", () => { + // One opening user turn, then a long assistant-only tail: any cut would land + // inside the active turn and drop the request that produced it. + const messages: Msg[] = [user("u0", "开始")]; + for (let index = 1; index < 20; index += 1) { + messages.push(assistantText(`a${index}`, "x".repeat(2_000))); + } + + const result = compactMessages(messages, { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 100, + keepRecent: 2, + minMessages: 4, + }); + + expect(result.compacted).toBe(false); + }); + + test("digest records the tools and files the agent already touched", () => { + const result = compactMessages(longConversation(), { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 4, + minMessages: 8, + }); + + expect(result.digest).toContain("[compacted]"); + expect(result.digest).toContain("Tools already used:"); + expect(result.digest).toMatch(/(bash|readFile)×\d+/); + expect(result.digest).toContain("Files touched:"); + expect(result.digest).toContain("src/module-0.ts"); + }); + + test("digest tells the model the content is gone, not that it was summarized", () => { + const result = compactMessages(longConversation(), { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 4, + minMessages: 8, + }); + + expect(result.digest).toContain("re-read a file"); + }); + + test("compaction is bounded — running twice does not compound", () => { + const first = compactMessages(longConversation(), { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 4, + minMessages: 8, + }); + + const second = compactMessages(first.messages, { + ...DEFAULT_COMPACTION_POLICY, + maxTokens: 1_000, + keepRecent: 4, + minMessages: 8, + }); + + expect(second.tokensAfter).toBeLessThanOrEqual(first.tokensAfter); + }); +}); diff --git a/packages/shared/src/context-compaction.ts b/packages/shared/src/context-compaction.ts new file mode 100644 index 0000000..65318fd --- /dev/null +++ b/packages/shared/src/context-compaction.ts @@ -0,0 +1,229 @@ +/** + * Context budget + compaction. + * + * Problem: the server replays the whole stored session on every turn, so the prompt + * grows without bound. A long session eventually either overflows the model window + * or costs a fortune in input tokens for history nobody re-reads. + * + * Approach: when the estimated prompt exceeds a budget, keep the most recent turns + * verbatim and replace everything older with a compact deterministic digest. The + * digest is not an LLM summary on purpose — it adds no latency, no cost, and no + * extra failure mode, and what the agent actually needs from old turns is *what it + * already did* (tools used, files touched), not the prose around it. + * + * Pure module: no I/O, no model calls. + */ + +/** Rough token estimate. ASCII ≈ 4 chars/token; CJK ≈ 1 char/token. */ +export function estimateTextTokens(text: string): number { + if (text.length === 0) return 0; + + let narrow = 0; + let wide = 0; + + for (const char of text) { + const code = char.codePointAt(0) ?? 0; + // CJK, kana, hangul and full-width forms are ~1 token per character. + if (code > 0x2e80) wide += 1; + else narrow += 1; + } + + return Math.ceil(narrow / 4) + wide; +} + +type PartLike = { + type: string; + text?: unknown; + toolName?: unknown; + input?: unknown; + output?: unknown; +}; + +type MessageLike = { + id: string; + role: string; + parts?: unknown; +}; + +function partsOf(message: MessageLike): PartLike[] { + return Array.isArray(message.parts) ? (message.parts as PartLike[]) : []; +} + +/** Token estimate for one message, counting text and tool payloads. */ +export function estimateMessageTokens(message: MessageLike): number { + let total = 0; + + for (const part of partsOf(message)) { + if (typeof part.text === "string") total += estimateTextTokens(part.text); + if (part.input !== undefined) total += estimateTextTokens(safeJson(part.input)); + if (part.output !== undefined) total += estimateTextTokens(safeJson(part.output)); + // Fixed overhead per part keeps a long tool-call chain from looking free. + total += 4; + } + + return total; +} + +/** Token estimate for a whole conversation. */ +export function estimateMessagesTokens(messages: MessageLike[]): number { + return messages.reduce((sum, message) => sum + estimateMessageTokens(message), 0); +} + +function safeJson(value: unknown): string { + try { + return JSON.stringify(value) ?? ""; + } catch { + return ""; + } +} + +/** Tool name for a part, handling both static `tool-` and dynamic MCP calls. */ +function toolNameOf(part: PartLike): string | null { + if (part.type === "dynamic-tool" && typeof part.toolName === "string") return part.toolName; + if (part.type.startsWith("tool-")) return part.type.slice("tool-".length); + return null; +} + +/** File-path-ish strings inside a tool input, for the digest. */ +function pathsIn(input: unknown, into: Set): void { + if (input === null || typeof input !== "object") return; + + for (const [key, value] of Object.entries(input as Record)) { + if (typeof value === "string") { + if (/path|file|dir/i.test(key) && !value.includes("\n") && value.length <= 120) { + into.add(value); + } + } else if (value !== null && typeof value === "object") { + pathsIn(value, into); + } + } +} + +export type CompactionPolicy = { + /** Estimated prompt tokens allowed before compaction runs. */ + maxTokens: number; + /** Most recent messages kept verbatim. */ + keepRecent: number; + /** Below this many messages compaction is pointless — skip it. */ + minMessages: number; +}; + +export const DEFAULT_COMPACTION_POLICY: CompactionPolicy = { + maxTokens: 60_000, + keepRecent: 10, + minMessages: 16, +}; + +export type CompactionResult = { + messages: MESSAGE[]; + compacted: boolean; + tokensBefore: number; + tokensAfter: number; + /** Human-readable record of what was folded away; null when nothing happened. */ + digest: string | null; +}; + +/** + * Fold old turns into one digest message when the prompt exceeds `maxTokens`. + * + * The cut is placed on a user-message boundary so a tool call and its result are + * never separated — splitting those would produce an invalid conversation. + */ +export function compactMessages( + messages: MESSAGE[], + policy: CompactionPolicy = DEFAULT_COMPACTION_POLICY, +): CompactionResult { + const tokensBefore = estimateMessagesTokens(messages); + + const noop: CompactionResult = { + messages, + compacted: false, + tokensBefore, + tokensAfter: tokensBefore, + digest: null, + }; + + if (tokensBefore <= policy.maxTokens) return noop; + if (messages.length < policy.minMessages) return noop; + + const cut = findCutIndex(messages, policy.keepRecent); + if (cut <= 0) return noop; + + const dropped = messages.slice(0, cut); + const kept = messages.slice(cut); + const digest = buildDigest(dropped); + + const digestMessage = { + id: `compaction-${dropped.length}`, + role: "assistant", + parts: [{ type: "text", text: digest }], + } as unknown as MESSAGE; + + const compacted = [digestMessage, ...kept]; + const tokensAfter = estimateMessagesTokens(compacted); + + return { + messages: compacted, + compacted: true, + tokensBefore, + tokensAfter, + digest, + }; +} + +/** + * Index to cut at: far enough back to keep `keepRecent` messages, snapped forward + * to the next user message so no assistant/tool group is split. + */ +function findCutIndex(messages: MessageLike[], keepRecent: number): number { + const target = Math.max(1, messages.length - keepRecent); + + // Prefer a boundary at or after the target: keeps the most recent turns intact. + for (let index = target; index < messages.length; index += 1) { + if (messages[index]?.role === "user") return index; + } + + // No user boundary ahead — fall back to the closest one *before* it. Returning + // `target` would cut inside a turn and could drop the user's active request. + for (let index = Math.min(target, messages.length - 1); index >= 1; index -= 1) { + if (messages[index]?.role === "user") return index; + } + + // Nothing safe to remove; compaction is skipped rather than guessed. + return 0; +} + +/** Deterministic record of what the dropped turns did. */ +function buildDigest(dropped: MessageLike[]): string { + const toolCounts = new Map(); + const paths = new Set(); + let userTurns = 0; + + for (const message of dropped) { + if (message.role === "user") userTurns += 1; + + for (const part of partsOf(message)) { + const toolName = toolNameOf(part); + if (toolName) { + toolCounts.set(toolName, (toolCounts.get(toolName) ?? 0) + 1); + pathsIn(part.input, paths); + } + } + } + + const tools = [...toolCounts.entries()] + .sort((a, b) => b[1] - a[1]) + .map(([name, count]) => `${name}×${count}`) + .join(", "); + + const fileList = [...paths].slice(0, 12).join(", "); + + const lines = [ + `[compacted] ${dropped.length} earlier messages folded away (${userTurns} user turns).`, + tools ? `Tools already used: ${tools}.` : null, + fileList ? `Files touched: ${fileList}.` : null, + "Those turns are no longer in context; re-read a file if you need its exact contents.", + ].filter(Boolean); + + return lines.join(" "); +} diff --git a/packages/shared/src/index.ts b/packages/shared/src/index.ts index 42c0a4d..e75a41b 100644 --- a/packages/shared/src/index.ts +++ b/packages/shared/src/index.ts @@ -28,6 +28,17 @@ export { type SerializedMcpTool, } from "./mcp-tools"; +export { + /** Context budget: token estimation + fold-based compaction (server and BYOK). */ + DEFAULT_COMPACTION_POLICY, + compactMessages, + estimateMessageTokens, + estimateMessagesTokens, + estimateTextTokens, + type CompactionPolicy, + type CompactionResult, +} from "./context-compaction"; + export { hasVisibleAssistantContent, type AssistantMessageLike, From 32d735a0c36091195a3aa08a94c164f37cf8d55f Mon Sep 17 00:00:00 2001 From: moyun Date: Thu, 12 Nov 2026 09:35:38 +0800 Subject: [PATCH 06/11] feat(evals): task-level evaluation with a meta-test Unit tests prove the pieces work; nothing measured whether the agent can do a job. - 20 cases across fix / feature / refactor / test, each with its own fixture and a behavioural check (it runs code, it does not grep for strings) - the important test is the meta-test: every case must **fail** in its initial state. It caught a case that passed by accident (microtasks settled before the assertion). - verifier files are restored from the fixture before scoring, so a case cannot pass by rewriting its own verifier; protected tests are compared against the pristine copy - checks run with the allowlisted env and a wall-clock budget, so a hanging check fails that case instead of the whole run Result: 20/20 in two independent runs, 6-7 turns / 8-9 tools / 15.5-18.4k tokens per case. A perfect score means the micro-tasks no longer discriminate; the next lever is repo-scale fixtures, not more cases. --- packages/cli/evals/cases.test.ts | 75 ++++ packages/cli/evals/cases.ts | 588 +++++++++++++++++++++++++ packages/cli/evals/check-runner.ts | 78 ++++ packages/cli/evals/results/latest.json | 255 +++++++++++ packages/cli/evals/run.ts | 231 ++++++++++ 5 files changed, 1227 insertions(+) create mode 100644 packages/cli/evals/cases.test.ts create mode 100644 packages/cli/evals/cases.ts create mode 100644 packages/cli/evals/check-runner.ts create mode 100644 packages/cli/evals/results/latest.json create mode 100644 packages/cli/evals/run.ts diff --git a/packages/cli/evals/cases.test.ts b/packages/cli/evals/cases.test.ts new file mode 100644 index 0000000..2beaa60 --- /dev/null +++ b/packages/cli/evals/cases.test.ts @@ -0,0 +1,75 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync, mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { EVAL_CASES, type EvalCase } from "./cases"; + +const created: string[] = []; + +/** Write a case's starting state into a throwaway directory. */ +function materialize(evalCase: EvalCase): string { + const dir = mkdtempSync(join(tmpdir(), `mocode-eval-${evalCase.id}-`)); + created.push(dir); + + for (const [path, content] of Object.entries(evalCase.files)) { + const target = join(dir, path); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, content); + } + + return dir; +} + +function runCheck(dir: string, command: string) { + return Bun.spawnSync(["bash", "-c", command], { cwd: dir, stdout: "pipe", stderr: "pipe" }); +} + +afterAll(() => { + for (const dir of created) rmSync(dir, { recursive: true, force: true }); +}); + +describe("eval case registry", () => { + test("ids are unique", () => { + const ids = EVAL_CASES.map((c) => c.id); + expect(new Set(ids).size).toBe(ids.length); + }); + + test("every case declares a prompt, files and a check", () => { + for (const evalCase of EVAL_CASES) { + expect(evalCase.prompt.length).toBeGreaterThan(20); + expect(Object.keys(evalCase.files).length).toBeGreaterThan(0); + expect(evalCase.check.length).toBeGreaterThan(0); + } + }); + + test("each case ships a verifier, not just a source file", () => { + for (const evalCase of EVAL_CASES) { + expect(Object.keys(evalCase.files)).toContain("verify.ts"); + } + }); +}); + +describe("every case actually measures something", () => { + // This is the most important test in the suite: a case whose check already + // passes before the agent runs contributes nothing but a false success rate. + for (const evalCase of EVAL_CASES) { + test(`${evalCase.id} fails in its initial state`, () => { + const dir = materialize(evalCase); + const result = runCheck(dir, evalCase.check); + + expect(result.exitCode).not.toBe(0); + }); + } + + // A check that passes on a syntactically broken project would be scoring + // "did not crash" rather than "did the work". + for (const evalCase of EVAL_CASES) { + test(`${evalCase.id} check is runnable (not a missing binary)`, () => { + const dir = materialize(evalCase); + const result = runCheck(dir, evalCase.check); + const output = `${result.stdout.toString()}${result.stderr.toString()}`; + + expect(output).not.toContain("command not found"); + }); + } +}); diff --git a/packages/cli/evals/cases.ts b/packages/cli/evals/cases.ts new file mode 100644 index 0000000..1e84835 --- /dev/null +++ b/packages/cli/evals/cases.ts @@ -0,0 +1,588 @@ +/** + * Task eval cases. + * + * Each case is a tiny, self-contained project plus a `check` command that must + * exit 0 once the task is done. Two properties matter more than the count: + * + * 1. **The check must FAIL before the agent touches anything.** A case whose + * check already passes measures nothing. `cases.test.ts` enforces this. + * 2. **The check must be behavioural.** It runs code and asserts on results, not + * on the presence of a string in a file — otherwise "rename the variable" + * would pass. + * + * Cases are deliberately small: a task that needs 40 turns tests the harness as + * much as the agent, and hides which capability actually failed. + */ + +export type EvalCase = { + id: string; + /** What kind of work this exercises — used in the report. */ + category: "fix" | "feature" | "refactor" | "test"; + prompt: string; + /** Starting state written into a temp dir. */ + files: Record; + /** Shell command that must exit 0 when the task is complete. */ + check: string; + /** + * Files restored from `files` before the check runs. + * + * Defaults to `["verify.ts"]`: a case that passes by rewriting its own verifier + * measures nothing. Files the agent is *supposed* to edit must not be listed. + */ + protectedFiles?: string[]; +}; + +const verify = `import assert from "node:assert";`; + +export const EVAL_CASES: EvalCase[] = [ + { + id: "fix-off-by-one", + category: "fix", + prompt: "sumTo(5) should return 15 but returns 10. Fix the loop in sum.ts.", + files: { + "sum.ts": `export function sumTo(n: number): number { + let total = 0; + for (let i = 1; i < n; i++) { + total += i; + } + return total; +} +`, + "verify.ts": `${verify} +import { sumTo } from "./sum"; +assert.equal(sumTo(5), 15); +assert.equal(sumTo(1), 1); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "fix-missing-await", + category: "fix", + prompt: "loadUser returns a Promise instead of the user. Find and fix the missing await.", + files: { + "db.ts": `export async function fetchUser(): Promise<{ name: string }> { + return { name: "ada" }; +} +`, + "index.ts": `import { fetchUser } from "./db"; + +export async function loadUser() { + const user = fetchUser(); + return user.name; +} +`, + "verify.ts": `${verify} +import { loadUser } from "./index"; +assert.deepEqual(await loadUser(), "ada"); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "implement-stub", + category: "feature", + prompt: "parseSize('1.5kb') should return 1536 (bytes). The function is a stub — implement it. Supported units: b, kb, mb.", + files: { + "size.ts": `export function parseSize(input: string): number { + throw new Error("not implemented"); +} +`, + "verify.ts": `${verify} +import { parseSize } from "./size"; +assert.equal(parseSize("512b"), 512); +assert.equal(parseSize("1.5kb"), 1536); +assert.equal(parseSize("2mb"), 2097152); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "add-validation", + category: "feature", + prompt: "withdraw() must throw 'amount must be positive' when given a negative amount. It currently accepts it.", + files: { + "account.ts": `export function withdraw(balance: number, amount: number): number { + return balance - amount; +} +`, + "verify.ts": `${verify} +import { withdraw } from "./account"; +assert.equal(withdraw(100, 30), 70); +assert.throws(() => withdraw(100, -5), /amount must be positive/); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "fix-json-crash", + category: "fix", + prompt: "parseConfig throws on invalid JSON and takes the app down. It should return null instead.", + files: { + "config.ts": `export function parseConfig(raw: string): { port: number } | null { + return JSON.parse(raw); +} +`, + "verify.ts": `${verify} +import { parseConfig } from "./config"; +assert.deepEqual(parseConfig('{"port":3000}'), { port: 3000 }); +assert.equal(parseConfig("{not json"), null); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "rename-across-files", + category: "refactor", + prompt: "Rename formatDate to formatIsoDate everywhere (definition and all call sites). Behaviour must not change.", + files: { + "date.ts": `export function formatDate(d: Date): string { + return d.toISOString().slice(0, 10); +} +`, + "report.ts": `import { formatDate } from "./date"; + +export function reportTitle(d: Date): string { + return "Report " + formatDate(d); +} +`, + "verify.ts": `${verify} +import { formatIsoDate } from "./date"; +import { reportTitle } from "./report"; +assert.equal(formatIsoDate(new Date("2026-03-04T00:00:00Z")), "2026-03-04"); +assert.equal(reportTitle(new Date("2026-03-04T00:00:00Z")), "Report 2026-03-04"); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "dedupe-into-helper", + category: "refactor", + prompt: "slug() is duplicated in two files with slightly different behaviour. Extract one shared slug() into slug.ts and make both files use it. Both call sites must keep working.", + files: { + "posts.ts": `export function slug(title: string): string { + return title.toLowerCase().replace(/[^a-z0-9]+/g, "-"); +} +`, + "tags.ts": `export function slug(title: string): string { + return title.toLowerCase().replace(/ /g, "-"); +} +`, + "verify.ts": `${verify} +import { existsSync, readFileSync } from "node:fs"; +import { slug as postSlug } from "./posts"; +import { slug as tagSlug } from "./tags"; + +// Behaviour must not change... +assert.equal(postSlug("Hello World!"), "hello-world"); +assert.equal(tagSlug("Hello World"), "hello-world"); + +// ...and the duplication must actually be gone: an alias re-export leaves two +// implementations behind while still passing the assertions above. +assert.ok(existsSync("slug.ts"), "slug.ts must exist"); +for (const file of ["posts.ts", "tags.ts"]) { + const source = readFileSync(file, "utf8"); + assert.ok(/from\s+["']\.\/slug["']/.test(source), file + " must import the shared slug"); + assert.ok(!/function\s+slug\s*\(/.test(source), file + " must not define its own slug()"); +} +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "add-unit-test", + category: "test", + prompt: "Add tests for clamp() in clamp.test.ts. Cover the lower bound, the upper bound, and the in-range case. `bun test` must pass.", + files: { + "clamp.ts": `export function clamp(value: number, min: number, max: number): number { + return Math.min(Math.max(value, min), max); +} +`, + "clamp.test.ts": `import { test, expect } from "bun:test"; +`, + "verify.ts": `${verify} +import { readFileSync, writeFileSync } from "node:fs"; + +const original = readFileSync("clamp.ts", "utf8"); + +/** Run the submitted suite and return its exit code. */ +function runTests(): number { + return Bun.spawnSync(["bun", "test", "clamp.test.ts"], { stdout: "pipe", stderr: "pipe" }).exitCode ?? 1; +} + +try { + // 1. The submitted tests must pass against the correct implementation. + assert.equal(runTests(), 0, "submitted tests must pass against the correct clamp()"); + + // 2. They must FAIL when clamp() is broken. Counting \`test(\` occurrences is not + // coverage: three empty callbacks pass every check while testing nothing. + const broken = [ + "export function clamp(value: number, min: number, max: number): number {", + " return value; // ignores both bounds", + "}", + "", + ].join("\n"); + + writeFileSync("clamp.ts", broken); + assert.notEqual(runTests(), 0, "submitted tests do not detect a broken clamp()"); +} finally { + writeFileSync("clamp.ts", original); +} + +console.log("ok"); +`, + }, + check: "bun test && bun verify.ts", + }, + { + id: "fix-async-ordering", + category: "fix", + prompt: "collect() returns an empty array because the promises are never awaited. Fix it so the results are collected in order.", + files: { + "collect.ts": `export async function collect(items: string[]): Promise { + const out: string[] = []; + for (const item of items) { + Promise.resolve(item.toUpperCase()).then((value) => out.push(value)); + } + // Snapshot taken before the promises settle, so the caller always sees []. + return [...out]; +} +`, + "verify.ts": `${verify} +import { collect } from "./collect"; +assert.deepEqual(await collect(["a", "b", "c"]), ["A", "B", "C"]); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "add-cli-flag", + category: "feature", + prompt: "cli.ts should support `--upper` to uppercase its output. Without the flag it must behave exactly as before.", + files: { + "cli.ts": `const args = process.argv.slice(2); +const text = args.filter((a) => !a.startsWith("--")).join(" "); +console.log(text); +`, + "verify.ts": `${verify} +import { spawnSync } from "node:child_process"; +const plain = spawnSync("bun", ["cli.ts", "hello"], { encoding: "utf8" }); +const upper = spawnSync("bun", ["cli.ts", "--upper", "hello"], { encoding: "utf8" }); +assert.equal(plain.stdout.trim(), "hello"); +assert.equal(upper.stdout.trim(), "HELLO"); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + + // ------------------------------------------------------------------------- + // Hard cases. The ten above are single-file with a visible failure message; + // these add multi-file propagation, behavioural regressions, a performance + // budget, and a test the agent is forbidden to edit. + // ------------------------------------------------------------------------- + { + id: "hard-barrel-rename", + category: "refactor", + prompt: + "Rename `formatMoney` to `formatCurrency` everywhere: its definition, the barrel re-export in index.ts, and both call sites. Behaviour must not change.", + files: { + "money.ts": `export function formatMoney(cents: number): string { + return "$" + (cents / 100).toFixed(2); +} +`, + "index.ts": `export { formatMoney } from "./money"; +`, + "cart.ts": `import { formatMoney } from "./index"; +export const cartLabel = (c: number) => "Cart: " + formatMoney(c); +`, + "invoice.ts": `import { formatMoney } from "./index"; +export const invoiceLabel = (c: number) => "Invoice: " + formatMoney(c); +`, + "verify.ts": `${verify} +import { formatCurrency } from "./money"; +import { cartLabel } from "./cart"; +import { invoiceLabel } from "./invoice"; +assert.equal(formatCurrency(1234), "$12.34"); +assert.equal(cartLabel(500), "Cart: $5.00"); +assert.equal(invoiceLabel(99), "Invoice: $0.99"); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-schema-migration", + category: "refactor", + prompt: + "The User type renames `userName` to `displayName`. Update the type, the factory, and both consumers so the program still compiles and prints correctly.", + files: { + "types.ts": `export type User = { id: string; userName: string }; +`, + "factory.ts": `import type { User } from "./types"; +export function makeUser(id: string): User { + return { id, userName: "anon-" + id }; +} +`, + "greet.ts": `import type { User } from "./types"; +export function greet(user: User): string { + return "Hello " + user.userName; +} +`, + "verify.ts": `${verify} +import { readFileSync } from "node:fs"; +import { makeUser } from "./factory"; +import { greet } from "./greet"; + +const user = makeUser("7"); +assert.equal(greet(user), "Hello anon-7"); +assert.equal((user as unknown as { displayName: string }).displayName, "anon-7"); + +// The migration must remove the old field, not bolt the new one on beside it. +for (const file of ["types.ts", "factory.ts", "greet.ts"]) { + assert.ok( + !readFileSync(file, "utf8").includes("userName"), + file + " still references the obsolete userName field", + ); +} +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-regression-neighbour", + category: "fix", + prompt: + "`average([])` must return 0 instead of NaN. Do not change `sum` or `average`'s behaviour for non-empty input — there is a neighbouring test that must keep passing.", + files: { + "stats.ts": `export function sum(values: number[]): number { + return values.reduce((a, b) => a + b, 0); +} + +export function average(values: number[]): number { + return sum(values) / values.length; +} +`, + "neighbour.test.ts": `import { test, expect } from "bun:test"; +import { sum, average } from "./stats"; + +test("sum of non-empty", () => expect(sum([1, 2, 3])).toBe(6)); +test("average of non-empty", () => expect(average([2, 4])).toBe(3)); +`, + "verify.ts": `${verify} +import { average } from "./stats"; +assert.equal(average([]), 0); +assert.equal(average([2, 4]), 3); +console.log("ok"); +`, + }, + check: "bun test neighbour.test.ts && bun verify.ts", + }, + { + id: "hard-perf-budget", + category: "fix", + prompt: + "hasDuplicate is O(n^2) and times out on large inputs. Rewrite it so it handles 40,000 items within the budget in verify.ts, without changing its return values.", + files: { + "dedupe.ts": `export function hasDuplicate(values: number[]): boolean { + for (let i = 0; i < values.length; i++) { + for (let j = 0; j < values.length; j++) { + if (i !== j && values[i] === values[j]) return true; + } + } + return false; +} +`, + "verify.ts": `import assert from "node:assert"; +import { hasDuplicate } from "./dedupe"; + +assert.equal(hasDuplicate([1, 2, 3]), false); +assert.equal(hasDuplicate([1, 2, 1]), true); + +const big = Array.from({ length: 50_000 }, (_, i) => i); +const started = Date.now(); +const result = hasDuplicate(big); +const elapsed = Date.now() - started; + +assert.equal(result, false); +assert.ok(elapsed < 800, "took " + elapsed + "ms, budget 800ms"); +console.log("ok in " + elapsed + "ms"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-test-driven", + category: "test", + prompt: + "csv.test.ts currently fails. Make it pass by fixing csv.ts — you may NOT edit the test file; a pristine copy is kept at csv.test.ts.orig and will be compared.", + files: { + "csv.ts": `export function parseCsv(line: string): string[] { + return line.split(","); +} +`, + "csv.test.ts": `import { test, expect } from "bun:test"; +import { parseCsv } from "./csv"; + +test("trims fields", () => expect(parseCsv("a, b ,c")).toEqual(["a", "b", "c"])); +test("drops empty fields", () => expect(parseCsv("a,,b")).toEqual(["a", "b"])); +test("empty input", () => expect(parseCsv("")).toEqual([])); +`, + "csv.test.ts.orig": `import { test, expect } from "bun:test"; +import { parseCsv } from "./csv"; + +test("trims fields", () => expect(parseCsv("a, b ,c")).toEqual(["a", "b", "c"])); +test("drops empty fields", () => expect(parseCsv("a,,b")).toEqual(["a", "b"])); +test("empty input", () => expect(parseCsv("")).toEqual([])); +`, + "verify.ts": `${verify} +import { readFileSync } from "node:fs"; +const current = readFileSync("csv.test.ts", "utf8"); +const pristine = readFileSync("csv.test.ts.orig", "utf8"); +assert.equal(current, pristine, "the test file was modified"); +console.log("ok"); +`, + }, + // The test and its pristine copy are restored before scoring, so an agent edit + // cannot make them agree. + protectedFiles: ["verify.ts", "csv.test.ts", "csv.test.ts.orig"], + check: "bun test csv.test.ts && bun verify.ts", + }, + { + id: "hard-immutability", + category: "fix", + prompt: + "sortByScore mutates the caller's array, which breaks a caller that reuses it. Return a sorted copy instead; the original array must keep its order.", + files: { + "rank.ts": `export function sortByScore(items: { score: number }[]): { score: number }[] { + return items.sort((a, b) => b.score - a.score); +} +`, + "verify.ts": `${verify} +import { sortByScore } from "./rank"; +const original = [{ score: 1 }, { score: 3 }, { score: 2 }]; +const sorted = sortByScore(original); +assert.deepEqual(sorted.map((i) => i.score), [3, 2, 1]); +assert.deepEqual(original.map((i) => i.score), [1, 3, 2], "input was mutated"); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-error-contract", + category: "feature", + prompt: + "fetchPrice must throw an Error whose message is exactly `price unavailable for ` when the symbol is unknown, and re-throw nothing else. Currently it swallows the failure.", + files: { + "price.ts": `const PRICES: Record = { BTC: 100, ETH: 50 }; + +export async function fetchPrice(symbol: string): Promise { + try { + return PRICES[symbol] as number; + } catch { + return 0; + } +} +`, + "verify.ts": `${verify} +import { fetchPrice } from "./price"; +assert.equal(await fetchPrice("BTC"), 100); +await assert.rejects(() => fetchPrice("DOGE"), /price unavailable for DOGE/); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-state-cycle", + category: "fix", + prompt: + "nextState cycles STOP -> GO -> SLOW -> STOP. The current implementation returns STOP forever after the first call. Fix the transition table.", + files: { + "light.ts": `export type Light = "STOP" | "GO" | "SLOW"; + +const TRANSITIONS: Record = { + STOP: "STOP", + GO: "STOP", + SLOW: "STOP", +}; + +export function nextState(state: Light): Light { + return TRANSITIONS[state]; +} +`, + "verify.ts": `${verify} +import { nextState } from "./light"; +assert.equal(nextState("STOP"), "GO"); +assert.equal(nextState("GO"), "SLOW"); +assert.equal(nextState("SLOW"), "STOP"); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-multi-constraint", + category: "feature", + prompt: + "normalizeTag must satisfy all four rules: trim whitespace, lowercase, replace spaces with '-', and return null for an empty result. It currently does none of them.", + files: { + "tag.ts": `export function normalizeTag(input: string): string | null { + return input; +} +`, + "verify.ts": `${verify} +import { normalizeTag } from "./tag"; +assert.equal(normalizeTag(" Hello World "), "hello-world"); +assert.equal(normalizeTag("A B C"), "a-b-c"); +assert.equal(normalizeTag(" "), null); +assert.equal(normalizeTag(""), null); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, + { + id: "hard-cascade-bug", + category: "fix", + prompt: + "checkoutTotal returns the wrong number. The expected total is subtotal(prices) + tax(subtotal(prices)) + shipping(subtotal(prices)). Fix checkoutTotal. The three helper functions are correct — do not change them.", + files: { + "checkout.ts": `export function subtotal(prices: number[]): number { + return prices.reduce((a, b) => a + b, 0); +} +export function tax(amount: number): number { + return Math.round(amount * 0.1); +} +export function shipping(amount: number): number { + return amount >= 100 ? 0 : 5; +} + +export function checkoutTotal(prices: number[]): number { + const base = subtotal(prices); + const discounted = base - tax(base); + return discounted + shipping(base); +} +`, + "verify.ts": `${verify} +import { checkoutTotal } from "./checkout"; +// 120 -> tax 12 -> 132, free shipping (base >= 100) +assert.equal(checkoutTotal([100, 20]), 132); +// 50 -> tax 5 -> 55 + 5 shipping +assert.equal(checkoutTotal([50]), 60); +console.log("ok"); +`, + }, + check: "bun verify.ts", + }, +]; diff --git a/packages/cli/evals/check-runner.ts b/packages/cli/evals/check-runner.ts new file mode 100644 index 0000000..90057ae --- /dev/null +++ b/packages/cli/evals/check-runner.ts @@ -0,0 +1,78 @@ +/** + * Bounded, credential-free check runner for evals. + * + * Two problems this solves, both found in review: + * + * 1. **A hanging check hangs the whole run.** `Bun.spawnSync` has no timeout, so + * one stuck command (e.g. a test that binds a port) blocks every remaining case + * and the report never gets written. + * 2. **Checks inherited the full environment**, including provider credentials — + * and `verify.ts` is agent-writable, so a generated check could print them. + * + * The runner bounds each check, reports a timeout as a failure, and runs with the + * same allowlisted environment the sandboxed tools get. + */ +import { buildSandboxEnv } from "../src/lib/permission-rules"; + +export type CheckResult = { + ok: boolean; + exitCode: number | null; + signal: string | number | null; + output: string; + timedOut: boolean; +}; + +/** Per-check wall clock budget. Generous enough for `bun test`, short enough to bound a run. */ +export const DEFAULT_CHECK_TIMEOUT_MS = 120_000; + +export type CheckOptions = { + timeoutMs?: number; + /** Overrides the default allowlisted env (tests pass a minimal set). */ + env?: Record; +}; + +/** + * Run `command` in `cwd`, bounded by a deadline. + * + * On timeout the child is killed and the result is reported as a failure, so the + * caller can record it and continue with the next case. + */ +export async function runCheck( + cwd: string, + command: string, + options: CheckOptions = {}, +): Promise { + const timeoutMs = options.timeoutMs ?? DEFAULT_CHECK_TIMEOUT_MS; + const env = (options.env ?? buildSandboxEnv()) as Record; + + const proc = Bun.spawn(["bash", "-c", command], { + cwd, + env, + stdout: "pipe", + stderr: "pipe", + }); + + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + proc.kill(); + }, timeoutMs); + + try { + const [stdout, stderr] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + ]); + const exitCode = await proc.exited; + + return { + ok: !timedOut && exitCode === 0, + exitCode, + signal: proc.signalCode ?? null, + output: `${stdout}${stderr}`.trim(), + timedOut, + }; + } finally { + clearTimeout(timer); + } +} diff --git a/packages/cli/evals/results/latest.json b/packages/cli/evals/results/latest.json new file mode 100644 index 0000000..2f8ffd5 --- /dev/null +++ b/packages/cli/evals/results/latest.json @@ -0,0 +1,255 @@ +{ + "ranAt": "2026-11-11T14:16:04.430Z", + "model": "deepseek-flash", + "maxTurns": 15, + "successRate": 1, + "totals": { + "cases": 20, + "passed": 20, + "avgTurns": 7, + "avgToolCalls": 9, + "avgTokens": 18396 + }, + "results": [ + { + "id": "fix-off-by-one", + "category": "fix", + "ok": true, + "turns": 3, + "toolCalls": 2, + "tokens": 4707, + "ms": 3224, + "checkExit": 0, + "checkTail": "ok", + "summary": "Fixed `sum.ts`: changed the loop bound from `i < n` to `i <= n` so `sumTo(5)` now returns 15." + }, + { + "id": "fix-missing-await", + "category": "fix", + "ok": true, + "turns": 6, + "toolCalls": 8, + "tokens": 11677, + "ms": 8540, + "checkExit": 0, + "checkTail": "ok", + "summary": "Added the missing `await` on `fetchUser()` in `index.ts`, so `loadUser()` now returns `user.name` instead of a Promise." + }, + { + "id": "implement-stub", + "category": "feature", + "ok": true, + "turns": 7, + "toolCalls": 8, + "tokens": 16335, + "ms": 10569, + "checkExit": 0, + "checkTail": "ok", + "summary": "Implemented `parseSize` in `size.ts` (binary units, case/whitespace tolerant, rounds to whole bytes); outputs verified: " + }, + { + "id": "add-validation", + "category": "feature", + "ok": true, + "turns": 8, + "toolCalls": 9, + "tokens": 20978, + "ms": 13436, + "checkExit": 0, + "checkTail": "ok", + "summary": "Done — `withdraw()` now throws `amount must be positive` for negative amounts (verified: existing case still returns 70," + }, + { + "id": "fix-json-crash", + "category": "fix", + "ok": true, + "turns": 7, + "toolCalls": 8, + "tokens": 17697, + "ms": 10449, + "checkExit": 0, + "checkTail": "ok", + "summary": "`parseConfig` now wraps `JSON.parse` in a try/catch and returns `null` on invalid JSON instead of throwing." + }, + { + "id": "rename-across-files", + "category": "refactor", + "ok": true, + "turns": 7, + "toolCalls": 11, + "tokens": 15736, + "ms": 9442, + "checkExit": 0, + "checkTail": "ok", + "summary": "Renamed `formatDate` → `formatIsoDate` in its definition (`date.ts`) and all call sites (`report.ts` import + usage); no" + }, + { + "id": "dedupe-into-helper", + "category": "refactor", + "ok": true, + "turns": 13, + "toolCalls": 16, + "tokens": 62506, + "ms": 30263, + "checkExit": 0, + "checkTail": "ok", + "summary": "Done. Extracted a single `slug()` into `slug.ts` and both files now re-export it.\n\nOne thing worth flagging: `verify.ts`" + }, + { + "id": "add-unit-test", + "category": "test", + "ok": true, + "turns": 5, + "toolCalls": 7, + "tokens": 9563, + "ms": 5152, + "checkExit": 0, + "checkTail": " 3 expect() calls Ran 3 tests across 1 file. [10.00ms]", + "summary": "Added three tests to `clamp.test.ts` covering the lower bound, upper bound, and in-range cases; `bun test` passes (3/3)." + }, + { + "id": "fix-async-ordering", + "category": "fix", + "ok": true, + "turns": 7, + "toolCalls": 8, + "tokens": 16026, + "ms": 9730, + "checkExit": 0, + "checkTail": "ok", + "summary": "`collect()` now awaits its promises via `Promise.all`, so results are returned in input order instead of an empty array." + }, + { + "id": "add-cli-flag", + "category": "feature", + "ok": true, + "turns": 5, + "toolCalls": 6, + "tokens": 9231, + "ms": 6141, + "checkExit": 0, + "checkTail": "ok", + "summary": "`cli.ts` now supports `--upper` (uppercases output); default behavior is unchanged and `verify.ts` passes." + }, + { + "id": "hard-barrel-rename", + "category": "refactor", + "ok": true, + "turns": 6, + "toolCalls": 14, + "tokens": 15167, + "ms": 10378, + "checkExit": 0, + "checkTail": "ok", + "summary": "Renamed `formatMoney` → `formatCurrency` in money.ts (definition), index.ts (barrel re-export), and the imports/uses in " + }, + { + "id": "hard-schema-migration", + "category": "refactor", + "ok": true, + "turns": 12, + "toolCalls": 17, + "tokens": 47448, + "ms": 20573, + "checkExit": 0, + "checkTail": "ok", + "summary": "Renamed `userName` → `displayName` in `types.ts`, `factory.ts`, and `greet.ts` (the consumers); `verify.ts` runs success" + }, + { + "id": "hard-regression-neighbour", + "category": "fix", + "ok": true, + "turns": 6, + "toolCalls": 8, + "tokens": 12714, + "ms": 6475, + "checkExit": 0, + "checkTail": " 2 expect() calls Ran 2 tests across 1 file. [9.00ms]", + "summary": "Added an empty-array guard in `average` (`stats.ts`) so `average([])` returns `0`; `sum` is untouched and non-empty aver" + }, + { + "id": "hard-perf-budget", + "category": "fix", + "ok": true, + "turns": 10, + "toolCalls": 11, + "tokens": 33619, + "ms": 18823, + "checkExit": 0, + "checkTail": "ok in 2ms", + "summary": "Rewrote `hasDuplicate` in `dedupe.ts` as a single-pass `Set` lookup (O(n) instead of O(n²)), keeping return values ident" + }, + { + "id": "hard-test-driven", + "category": "test", + "ok": true, + "turns": 5, + "toolCalls": 7, + "tokens": 10201, + "ms": 5807, + "checkExit": 0, + "checkTail": " 3 expect() calls Ran 3 tests across 1 file. [13.00ms]", + "summary": "Fixed `csv.ts` to trim each field and drop empty ones; all 3 tests pass and the pristine test file check is ok." + }, + { + "id": "hard-immutability", + "category": "fix", + "ok": true, + "turns": 5, + "toolCalls": 5, + "tokens": 9018, + "ms": 5954, + "checkExit": 0, + "checkTail": "ok", + "summary": "`sortByScore` now sorts a shallow copy (`[...items]`), leaving the caller's array order untouched." + }, + { + "id": "hard-error-contract", + "category": "feature", + "ok": true, + "turns": 7, + "toolCalls": 8, + "tokens": 14754, + "ms": 9631, + "checkExit": 0, + "checkTail": "ok", + "summary": "Changed `fetchPrice` in `price.ts` to throw `Error(\"price unavailable for \")` on unknown symbols with no try/cat" + }, + { + "id": "hard-state-cycle", + "category": "fix", + "ok": true, + "turns": 6, + "toolCalls": 7, + "tokens": 13826, + "ms": 7961, + "checkExit": 0, + "checkTail": "ok", + "summary": "Fixed the transition table in `light.ts` to `STOP: \"GO\"`, `GO: \"SLOW\"`, `SLOW: \"STOP\"`, so `nextState` now cycles correc" + }, + { + "id": "hard-multi-constraint", + "category": "feature", + "ok": true, + "turns": 6, + "toolCalls": 7, + "tokens": 13718, + "ms": 8861, + "checkExit": 0, + "checkTail": "ok", + "summary": "`normalizeTag` in `tag.ts` now trims, lowercases, replaces whitespace runs with `-`, and returns `null` when the result " + }, + { + "id": "hard-cascade-bug", + "category": "fix", + "ok": true, + "turns": 6, + "toolCalls": 7, + "tokens": 13001, + "ms": 9219, + "checkExit": 0, + "checkTail": "ok", + "summary": "Fixed `checkoutTotal` in `checkout.ts` to add tax to the base before adding shipping (`base + tax(base) + shipping(base)" + } + ] +} \ No newline at end of file diff --git a/packages/cli/evals/run.ts b/packages/cli/evals/run.ts new file mode 100644 index 0000000..3116dc4 --- /dev/null +++ b/packages/cli/evals/run.ts @@ -0,0 +1,231 @@ +/** + * Task eval runner. + * + * Runs each case against a real model with the real tool implementations, then + * executes the case's check. This is the only measurement in the repo that scores + * *end-to-end task success* rather than unit-level correctness. + * + * Run from the repo root (needs DEEPSEEK_API_KEY in `.env`): + * bun run packages/cli/evals/run.ts + * bun run packages/cli/evals/run.ts --only fix-off-by-one + * + * Each case gets a throwaway directory; the agent's cwd is switched into it, so + * `executeLocalTool` resolves paths against the case and the sandbox still applies. + */ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { generateText, stepCountIs, tool } from "ai"; +import { createOpenAI } from "@ai-sdk/openai"; +import { Mode, getToolContracts, toolInputSchemas } from "@mocode/shared"; +import { executeLocalTool } from "../src/lib/local-tools"; +import { EVAL_CASES, type EvalCase } from "./cases"; +import { runCheck } from "./check-runner"; + +const MAX_TURNS = 15; + +const SYSTEM = [ + "You are a coding agent working inside a small project directory.", + "Use the available tools to read files, then edit them.", + "Make the smallest change that satisfies the request.", + "When the task is done, reply with a one-line summary and stop.", +].join(" "); + +function buildModel() { + const apiKey = process.env.DEEPSEEK_API_KEY; + if (!apiKey) throw new Error("DEEPSEEK_API_KEY is not set — run from the repo root"); + + return createOpenAI({ + apiKey, + baseURL: "https://api.deepseek.com/v1", + name: "deepseek", + }).chat("deepseek-flash"); +} + +/** Real tool implementations — same functions the CLI agent calls. */ +function buildTools() { + const contracts = getToolContracts(Mode.BUILD) as Record; + + return Object.fromEntries( + Object.keys(contracts) + // `task` spawns a subagent; out of scope for a single-turn eval. + .filter((name) => name !== "task") + .map((name) => [ + name, + tool({ + description: contracts[name]?.description ?? name, + inputSchema: toolInputSchemas[name as keyof typeof toolInputSchemas], + execute: async (input: unknown) => executeLocalTool(name, input, Mode.BUILD), + }), + ]), + ); +} + +function materialize(evalCase: EvalCase): string { + const dir = mkdtempSync(join(tmpdir(), `mocode-eval-${evalCase.id}-`)); + + for (const [path, content] of Object.entries(evalCase.files)) { + const target = join(dir, path); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, content); + } + + return dir; +} + +type CaseResult = { + id: string; + category: EvalCase["category"]; + ok: boolean; + turns: number; + toolCalls: number; + tokens: number; + ms: number; + checkExit: number | null; + checkTail: string; + summary: string; +}; + +async function runCase(evalCase: EvalCase, model: ReturnType): Promise { + const dir = materialize(evalCase); + const started = Date.now(); + + let turns = 0; + let toolCalls = 0; + let tokens = 0; + let summary = ""; + + const originalCwd = process.cwd(); + + try { + process.chdir(dir); + + const result = await generateText({ + model, + system: SYSTEM, + prompt: evalCase.prompt, + tools: buildTools(), + stopWhen: stepCountIs(MAX_TURNS), + maxOutputTokens: 1_500, + onStepFinish(step) { + turns += 1; + toolCalls += step.toolCalls.length; + tokens += step.usage?.totalTokens ?? 0; + }, + }); + + summary = result.text.trim().slice(0, 120); + } catch (error) { + summary = `agent error: ${error instanceof Error ? error.message.slice(0, 100) : "unknown"}`; + } finally { + process.chdir(originalCwd); + } + + // Restore verifier files from the fixture before scoring: `verify.ts` (and any + // protected test) is agent-writable, and a case that passes by rewriting its own + // verifier measures nothing. + const protectedFiles = new Set(evalCase.protectedFiles ?? ["verify.ts"]); + for (const [path, content] of Object.entries(evalCase.files)) { + if (!protectedFiles.has(path)) continue; + const target = join(dir, path); + mkdirSync(dirname(target), { recursive: true }); + writeFileSync(target, content); + } + + const check = await runCheck(dir, evalCase.check); + const output = check.output; + + rmSync(dir, { recursive: true, force: true }); + + return { + id: evalCase.id, + category: evalCase.category, + ok: check.ok, + turns, + toolCalls, + tokens, + ms: Date.now() - started, + checkExit: check.exitCode, + checkTail: output.split("\n").slice(-2).join(" ").slice(0, 160), + summary, + }; +} + +const onlyIndex = process.argv.indexOf("--only"); +const only = onlyIndex === -1 ? null : process.argv[onlyIndex + 1]; +const selected = only ? EVAL_CASES.filter((c) => c.id === only) : EVAL_CASES; + +if (selected.length === 0) { + console.error(`No eval case matched ${only ?? ""}`); + process.exit(2); +} + +const model = buildModel(); +const results: CaseResult[] = []; + +console.log(`Running ${selected.length} eval case(s), max ${MAX_TURNS} turns each\n`); +console.log("case".padEnd(24), "ok".padEnd(4), "turns".padEnd(7), "tools".padEnd(7), "tokens".padEnd(9), "time"); + +for (const evalCase of selected) { + process.stdout.write(`${evalCase.id.padEnd(24)} `); + const result = await runCase(evalCase, model); + results.push(result); + + console.log( + (result.ok ? "✅" : "❌").padEnd(4), + String(result.turns).padEnd(7), + String(result.toolCalls).padEnd(7), + String(result.tokens).padEnd(9), + `${(result.ms / 1000).toFixed(1)}s`, + ); + + if (!result.ok) console.log("".padEnd(25), `check exit ${result.checkExit}: ${result.checkTail}`); +} + +const passed = results.filter((r) => r.ok).length; +const avg = (pick: (r: CaseResult) => number) => + results.length === 0 ? 0 : Math.round(results.reduce((sum, r) => sum + pick(r), 0) / results.length); + +console.log("\n=== 汇总 ==="); +console.log(`任务成功率: ${passed}/${results.length} (${Math.round((passed / results.length) * 100)}%)`); +console.log(`平均轮次: ${avg((r) => r.turns)} 平均工具数: ${avg((r) => r.toolCalls)} 平均 token: ${avg((r) => r.tokens)}`); + +const byCategory = new Map(); +for (const result of results) { + const entry = byCategory.get(result.category) ?? { ok: 0, total: 0 }; + entry.total += 1; + if (result.ok) entry.ok += 1; + byCategory.set(result.category, entry); +} + +console.log("\n=== 分类 ==="); +for (const [category, entry] of byCategory) { + console.log(`${category.padEnd(10)} ${entry.ok}/${entry.total}`); +} + +const reportPath = new URL("./results/latest.json", import.meta.url); +mkdirSync(dirname(reportPath.pathname), { recursive: true }); +writeFileSync( + reportPath, + JSON.stringify( + { + ranAt: new Date().toISOString(), + model: "deepseek-flash", + maxTurns: MAX_TURNS, + successRate: passed / results.length, + totals: { + cases: results.length, + passed, + avgTurns: avg((r) => r.turns), + avgToolCalls: avg((r) => r.toolCalls), + avgTokens: avg((r) => r.tokens), + }, + results, + }, + null, + 2, + ), +); + +console.log(`\n报告已写入 ${reportPath.pathname}`); +process.exit(passed === results.length ? 0 : 1); From c31169344f09a812d54e993bfd5a46e8acb6fec4 Mon Sep 17 00:00:00 2001 From: moyun Date: Thu, 12 Nov 2026 09:35:44 +0800 Subject: [PATCH 07/11] feat(cli): wire the new engine into the TUI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything the previous commits added becomes visible here. - the footer counts up: `执行中 2.4s · turn 3/25 · 3 tools`, plus per-tool `… 2.3s` on a running tool. The tick only runs while work is pending, so an idle session re-renders zero extra times. - the status row budgets against the width it actually has (shell and composer insets removed), and shrinks the model id rather than letting the downgrade/budget segments be clipped silently. - a downgrade is explicit: `{color.error}` segment plus a toast, never a silent switch. - tool execution now reports its real cause: a timeout says timeout, and the sandbox diagnostic is reserved for an actual sandbox crash. - shell spawn goes through the platform sandbox plan, with an allowlisted env for the child. - approval learns about `bash-approval.ts`: an unconfined shell asks for every command, and `2>/dev/null` no longer triggers the dialog. Adds the first component-level test infrastructure (`testing/render.tsx`): character-frame assertions for the status row and tool blocks. --- packages/cli/src/components/input-bar.tsx | 22 +- .../messages/bot-message.render.test.tsx | 51 +++ .../src/components/messages/bot-message.tsx | 22 +- packages/cli/src/components/session-shell.tsx | 14 +- .../cli/src/components/status-bar.test.tsx | 157 +++++++++ packages/cli/src/components/status-bar.tsx | 192 +++++++++-- packages/cli/src/hooks/use-chat.ts | 160 +++++++++- packages/cli/src/lib/bash-approval.test.ts | 35 ++- packages/cli/src/lib/bash-approval.ts | 38 ++- .../cli/src/lib/local-chat-transport.test.ts | 138 +++++++- packages/cli/src/lib/local-chat-transport.ts | 297 ++++++++++++++---- packages/cli/src/lib/local-tools.ts | 204 +++++++++--- packages/cli/src/screens/session.tsx | 8 + packages/cli/src/testing/render.tsx | 73 +++++ 14 files changed, 1267 insertions(+), 144 deletions(-) create mode 100644 packages/cli/src/components/messages/bot-message.render.test.tsx create mode 100644 packages/cli/src/components/status-bar.test.tsx create mode 100644 packages/cli/src/testing/render.tsx diff --git a/packages/cli/src/components/input-bar.tsx b/packages/cli/src/components/input-bar.tsx index 631b2d5..caa1fbf 100644 --- a/packages/cli/src/components/input-bar.tsx +++ b/packages/cli/src/components/input-bar.tsx @@ -8,7 +8,8 @@ import type { KeyBinding } from "@opentui/core"; import { useNavigate } from "react-router"; import { EmptyBorder } from "./border"; import { StatusBar } from "./status-bar"; -import { CommandMenu } from "./command-menu"; +import type { AgentProgress } from "../lib/agent-loop"; +import type { AgentPhase } from "../lib/agent-phase";import { CommandMenu } from "./command-menu"; import type { Command } from "./command-menu/types"; import { useCommandMenu } from "./command-menu/use-command-menu"; import { useToast } from "../providers/toast"; @@ -264,6 +265,12 @@ type Props = { composerRestoreToken?: number; subagentRunning?: boolean; subagentType?: string | null; + /** Loop budget shown in the status line; see `lib/agent-loop.ts`. */ + agentProgress?: AgentProgress | null; + /** Set when a fallback model answered. */ + agentDowngrade?: { from: string; to: string } | null; + agentPhase?: AgentPhase | null; + agentElapsedMs?: number | null; }; export const TEXTAREA_KEY_BINDINGS: KeyBinding[] = [ @@ -280,6 +287,10 @@ export function InputBar({ composerRestoreToken = 0, subagentRunning = false, subagentType = null, + agentProgress = null, + agentDowngrade = null, + agentPhase = null, + agentElapsedMs = null, }: Props) { const { mode, model, toggleMode, setMode, setModel } = usePromptConfig(); const textareaRef = useRef(null); @@ -643,7 +654,14 @@ export function InputBar({ onContentChange={handleTextareaContentChange} placeholder={`Ask anything... "Fix a bug in the database"`} /> - + diff --git a/packages/cli/src/components/messages/bot-message.render.test.tsx b/packages/cli/src/components/messages/bot-message.render.test.tsx new file mode 100644 index 0000000..096884c --- /dev/null +++ b/packages/cli/src/components/messages/bot-message.render.test.tsx @@ -0,0 +1,51 @@ +import { describe, expect, test } from "bun:test"; +import { Mode } from "@mocode/shared"; +import { BotMessage } from "./bot-message"; +import { renderText } from "../../testing/render"; +import { markToolStarted, resetToolTimings } from "../../lib/tool-timing"; + +type Part = Parameters[0]["parts"][number]; + +function pendingTool(id: string): Part { + return { + type: "tool-bash", + toolCallId: id, + state: "input-available", + input: { command: "npm test", description: "run tests" }, + } as unknown as Part; +} + +describe("BotMessage tool timing", () => { + test("a running tool shows how long it has been running", async () => { + resetToolTimings(); + markToolStarted("call-1", Date.now() - 2_300); + + const frame = await renderText( + , + { width: 100, height: 8 }, + ); + + expect(frame).toContain("2.3s"); + expect(frame).toContain("npm test"); + }); + + test("a finished tool shows no timer", async () => { + resetToolTimings(); + markToolStarted("call-1", Date.now() - 2_300); + + const finished = { + type: "tool-bash", + toolCallId: "call-1", + state: "output-available", + input: { command: "npm test" }, + output: { stdout: "ok", stderr: "", exitCode: 0 }, + } as unknown as Part; + + const frame = await renderText( + , + { width: 100, height: 8 }, + ); + + expect(frame).not.toContain("2.3s"); + }); +}); diff --git a/packages/cli/src/components/messages/bot-message.tsx b/packages/cli/src/components/messages/bot-message.tsx index e2eef43..89efad6 100644 --- a/packages/cli/src/components/messages/bot-message.tsx +++ b/packages/cli/src/components/messages/bot-message.tsx @@ -20,9 +20,10 @@ import { TASK_SUMMARY_MAX_VISIBLE_LINES, } from "../../lib/bot-message-task"; import { groupConsecutiveParts, partRenderKey } from "../../lib/bot-message-parts"; +import { formatToolElapsed, toolElapsedMs } from "../../lib/tool-timing"; import type { LanguageModelUsage } from "ai"; import prettyMs from "pretty-ms"; -import { useState } from "react"; +import { useEffect, useState } from "react"; import { EmptyBorder } from "../border"; import { Spinner } from "../spinner"; import { useTheme } from "../../providers/theme"; @@ -193,6 +194,15 @@ export function BotMessage({ toolsPending, }); const showDuration = shouldShowDurationInFooter({ streaming, durationMs }); + + // While a tool is pending, re-render a few times a second so its elapsed label + // counts up instead of freezing at the value captured when the row appeared. + const [, tick] = useState(0); + useEffect(() => { + if (!toolsPending) return; + const timer = setInterval(() => tick((value) => value + 1), 200); + return () => clearInterval(timer); + }, [toolsPending]); const showMessageFooter = shouldShowAssistantMessageFooter(parts); return ( @@ -239,10 +249,14 @@ export function BotMessage({ ? formatBashToolDisplay(part.input) : null; const argsText = bashDisplay?.command ?? formatToolArgs(part); - const statusSuffix = - part.state !== "output-available" && part.state !== "output-error" + const isPending = + part.state !== "output-available" && part.state !== "output-error"; + const elapsed = isPending ? toolElapsedMs(part.toolCallId) : null; + const statusSuffix = isPending + ? elapsed === null ? " …" - : ""; + : ` … ${formatToolElapsed(elapsed)}` + : ""; const errorSuffix = part.state === "output-error" && part.errorText ? ` ${formatToolErrorForDisplay(toolName, part.errorText)}` diff --git a/packages/cli/src/components/session-shell.tsx b/packages/cli/src/components/session-shell.tsx index e856a62..6a1a144 100644 --- a/packages/cli/src/components/session-shell.tsx +++ b/packages/cli/src/components/session-shell.tsx @@ -3,6 +3,8 @@ import type { ReactNode, RefObject } from "react"; import { InputBar } from "./input-bar"; import { Spinner } from "./spinner"; import { usePromptConfig } from "../providers/prompt-config"; +import type { AgentProgress } from "../lib/agent-loop"; +import type { AgentPhase } from "../lib/agent-phase"; /** Session layout: scrollable transcript, input bar, and status footer. */ type Props = { @@ -16,6 +18,12 @@ type Props = { /** While a Task subagent runs — shows type in footer (D-21). */ subagentRunning?: boolean; subagentType?: string | null; + /** Loop budget shown in the status line; see `lib/agent-loop.ts`. */ + agentProgress?: AgentProgress | null; + /** Set when a fallback model answered. */ + agentDowngrade?: { from: string; to: string } | null; + agentPhase?: AgentPhase | null; + agentElapsedMs?: number | null; composerRestoreText?: string | null; composerRestoreToken?: number; transcriptScrollRef?: RefObject; @@ -29,6 +37,10 @@ export function SessionShell({ interruptible = false, subagentRunning = false, subagentType = null, + agentProgress = null, + agentDowngrade = null, + agentPhase = null, + agentElapsedMs = null, composerRestoreText = null, composerRestoreToken = 0, transcriptScrollRef, @@ -60,7 +72,7 @@ export function SessionShell({ - + { + test("idle: shows mode, model and hints, but no loop budget", async () => { + const frame = await renderText(); + + expect(frame).toContain("Build"); + expect(frame).toContain(">"); + expect(frame).toContain("提交"); + expect(frame).toContain("换行"); + expect(frame).not.toContain("turn"); + expect(frame).not.toContain("tools"); + }); + + test("shows the turn budget once a run has continued", async () => { + const frame = await renderText( + , + ); + + expect(frame).toContain("turn 4/25"); + expect(frame).not.toContain("tools"); + }); + + test("appends the tool count for the current turn", async () => { + const frame = await renderText( + , + ); + + expect(frame).toContain("turn 4/25 · 4 tools"); + }); + + test("hides the budget before the first continued turn", async () => { + const frame = await renderText( + , + ); + + expect(frame).not.toContain("turn"); + }); + + test("keeps the whole status line on a single row (DESIGN.md: status-row = 1)", async () => { + const lines = await renderLines( + , + { width: 100, height: 4 }, + ); + + expect(lines).toHaveLength(1); + expect(lines[0]).toContain("turn 13/25 · 9 tools"); + }); + + test("subagent footer replaces the budget while a Task runs", async () => { + const frame = await renderText( + , + ); + + expect(frame).toContain("explore"); + expect(frame).toContain("esc to interrupt"); + expect(frame).not.toContain("turn"); + }); + + test("shows a downgrade notice when a fallback model answered", async () => { + const frame = await renderText( + , + ); + + expect(frame).toContain("fallback → deepseek-flash"); + }); + + test("shows no downgrade notice on a normal turn", async () => { + const frame = await renderText(); + + expect(frame).not.toContain("fallback"); + }); + + test("narrow width with downgrade + budget stays on one row and drops newline first", async () => { + const lines = await renderLines( + , + // Wide enough for mode/model/downgrade/budget; tight enough to drop newline/note. + { width: 72, height: 4 }, + ); + + expect(lines).toHaveLength(1); + expect(lines[0]).toContain("fallback → deepseek-flash"); + expect(lines[0]).toContain("turn 13/25"); + expect(lines[0]).not.toContain("换行"); + }); +}); + +describe("selectStatusHints", () => { + const hint = { submit: "↵", newline: "⇧↵", note: "Ctrl+J" }; + + test("keeps every hint when there is room", () => { + expect(selectStatusHints(120, 20, hint)).toEqual({ + showSubmit: true, + showNewline: true, + showNote: true, + }); + }); + + test("drops newline before submit under pressure", () => { + expect(selectStatusHints(40, 30, hint)).toEqual({ + showSubmit: true, + showNewline: false, + showNote: false, + }); + }); +}); + +describe("live agent phase", () => { + test("shows the running phase with a counting timer", async () => { + const frame = await renderText( + , + ); + + expect(frame).toContain("执行中 2.4s"); + }); + + test("thinking and generating are distinguishable", async () => { + const thinking = await renderText(); + const generating = await renderText( + , + ); + + expect(thinking).toContain("思考中 1.0s"); + expect(generating).toContain("生成中 9.0s"); + }); + + test("idle shows no phase at all", async () => { + const frame = await renderText(); + + expect(frame).not.toContain("s ·"); + expect(frame).not.toContain("执行中"); + }); + + test("phase and budget coexist on one row", async () => { + const lines = await renderLines( + , + { width: 110, height: 4 }, + ); + + expect(lines).toHaveLength(1); + expect(lines[0]).toContain("执行中 2.4s"); + expect(lines[0]).toContain("turn 3/25"); + }); +}); diff --git a/packages/cli/src/components/status-bar.tsx b/packages/cli/src/components/status-bar.tsx index 5adbe15..89d18e3 100644 --- a/packages/cli/src/components/status-bar.tsx +++ b/packages/cli/src/components/status-bar.tsx @@ -1,24 +1,109 @@ import { TextAttributes } from "@opentui/core"; -import { getNewlineHint } from "../terminal-capabilities"; +import { useTerminalDimensions } from "@opentui/react"; +import { getNewlineHint, type NewlineHint } from "../terminal-capabilities"; import { useTheme } from "../providers/theme"; import { usePromptConfig } from "../providers/prompt-config"; import { Mode } from "@mocode/shared"; +import { formatAgentProgress, type AgentProgress } from "../lib/agent-loop"; +import { formatAgentPhase, type AgentPhase } from "../lib/agent-phase"; import { Spinner } from "./spinner"; type Props = { subagentRunning?: boolean; subagentType?: string | null; + /** Loop budget for the current user message; see `lib/agent-loop.ts`. */ + agentProgress?: AgentProgress | null; + /** Set when a fallback model answered — shown in `{color.error}`, never hidden. */ + agentDowngrade?: { from: string; to: string } | null; + /** Live phase of the running turn, e.g. `执行中`. */ + agentPhase?: AgentPhase | null; + /** Live elapsed time for the run, in ms. */ + agentElapsedMs?: number | null; }; -/** Footer hints: agent mode, model label, and submit/newline shortcuts for this terminal. */ -export function StatusBar({ subagentRunning = false, subagentType = null }: Props) { +/** Horizontal space the shell + composer insets take from the status row. */ +const STATUS_ROW_INSET = 4; + +/** Trim a label to `max` cells, marking the cut with an ellipsis. */ +function truncateToWidth(text: string, max: number): string { + if (cellWidth(text) <= max) return text; + if (max <= 1) return "…"; + + let out = ""; + let used = 0; + for (const char of text) { + const w = char.codePointAt(0)! > 0xff ? 2 : 1; + if (used + w > max - 1) break; + out += char; + used += w; + } + + return `${out}…`; +} + +/** Visible width of a status segment in character cells (CJK ≈ 2 cells). */ +function cellWidth(text: string): number { + let width = 0; + for (const char of text) { + width += char.codePointAt(0)! > 0xff ? 2 : 1; + } + return width; +} + +type HintVisibility = { + showSubmit: boolean; + showNewline: boolean; + showNote: boolean; +}; + +/** + * Drop hints right-to-left under width pressure (DESIGN.md): note → newline → submit. + * Mode / model / downgrade / budget are never dropped. + */ +export function selectStatusHints( + availableCols: number, + fixedCols: number, + hint: NewlineHint, +): HintVisibility { + // Each hint is rendered as `·` + label with gap={1} before both nodes. + const hintCols = (label: string) => 1 + cellWidth("·") + 1 + cellWidth(label); + const submitCols = hintCols(`${hint.submit} 提交`); + const newlineCols = hintCols(`${hint.newline} 换行`); + const noteCols = hint.note ? hintCols(hint.note) : 0; + const budget = Math.max(0, availableCols - fixedCols); + + if (submitCols + newlineCols + noteCols <= budget) { + return { showSubmit: true, showNewline: true, showNote: Boolean(hint.note) }; + } + // Drop note first (rightmost), then newline, then submit. + if (submitCols + newlineCols <= budget) { + return { showSubmit: true, showNewline: true, showNote: false }; + } + if (submitCols <= budget) { + return { showSubmit: true, showNewline: false, showNote: false }; + } + return { showSubmit: false, showNewline: false, showNote: false }; +} + +/** Footer hints: agent mode, model label, loop budget, and submit/newline shortcuts for this terminal. */ +export function StatusBar({ + subagentRunning = false, + subagentType = null, + agentProgress = null, + agentDowngrade = null, + agentPhase = null, + agentElapsedMs = null, +}: Props) { const hint = getNewlineHint(); const { colors } = useTheme(); const { mode, model } = usePromptConfig(); + const { width } = useTerminalDimensions(); + const budgetLabel = formatAgentProgress(agentProgress); + const phaseLabel = formatAgentPhase(agentPhase, agentElapsedMs); if (subagentRunning && subagentType) { return ( - + {subagentType} @@ -29,28 +114,95 @@ export function StatusBar({ subagentRunning = false, subagentType = null }: Prop ); } + const modeLabel = mode === Mode.PLAN ? "Plan" : "Build"; + const downgradeLabel = agentDowngrade ? `fallback → ${agentDowngrade.to}` : null; + + // The row lives inside the shell's padding and the composer inset, so the space + // it can actually use is narrower than the terminal. Budgeting against the raw + // terminal width shows hints that then get clipped by `overflow="hidden"`. + const usableWidth = Math.max(0, width - STATUS_ROW_INSET); + let modelLabel: string = model; + + // Approximate fixed cluster width. `gap={1}` inserts one cell between each + // sibling text node in the row below (mode, >, model, and optional pairs). + const fixedNodes = + 3 + // mode, >, model + (downgradeLabel ? 2 : 0) + + (budgetLabel ? 2 : 0) + + (phaseLabel ? 2 : 0); + let fixedCols = cellWidth(modeLabel) + cellWidth(">") + cellWidth(model); + if (downgradeLabel) fixedCols += cellWidth("·") + cellWidth(downgradeLabel); + if (budgetLabel) fixedCols += cellWidth("·") + cellWidth(budgetLabel); + if (phaseLabel) fixedCols += cellWidth("·") + cellWidth(phaseLabel); + fixedCols += Math.max(0, fixedNodes - 1); // gaps between siblings + + // If even the fixed labels overflow, shrink the model id rather than letting the + // rightmost segments (downgrade, budget) be clipped away silently. + if (fixedCols > usableWidth) { + const overflow = fixedCols - usableWidth; + modelLabel = truncateToWidth(modelLabel, Math.max(4, cellWidth(modelLabel) - overflow)); + fixedCols = Math.max( + 0, + cellWidth(modeLabel) + cellWidth(">") + cellWidth(modelLabel) + (downgradeLabel ? cellWidth("·") + cellWidth(downgradeLabel) : 0) + (budgetLabel ? cellWidth("·") + cellWidth(budgetLabel) : 0) + Math.max(0, fixedNodes - 1), + ); + } + + const visibility = selectStatusHints(usableWidth, fixedCols, hint); + return ( - + - {mode === Mode.PLAN ? "Plan" : "Build"} + {modeLabel} {">"} - {model} - - · - - - {hint.submit} 提交 - - - · - - - {hint.newline} 换行 - - {hint.note ? ( + {modelLabel} + {phaseLabel ? ( + <> + + · + + {phaseLabel} + + ) : null} + {agentDowngrade ? ( + <> + + · + + {downgradeLabel} + + ) : null} + {budgetLabel ? ( + <> + + · + + {budgetLabel} + + ) : null} + {visibility.showSubmit ? ( + <> + + · + + + {hint.submit} 提交 + + + ) : null} + {visibility.showNewline ? ( + <> + + · + + + {hint.newline} 换行 + + + ) : null} + {visibility.showNote && hint.note ? ( <> · diff --git a/packages/cli/src/hooks/use-chat.ts b/packages/cli/src/hooks/use-chat.ts index 6d958ec..eec00dd 100644 --- a/packages/cli/src/hooks/use-chat.ts +++ b/packages/cli/src/hooks/use-chat.ts @@ -29,6 +29,7 @@ import { import { Mode, toolInputSchemas, type ModeType, type SupportedChatModelId, type ToolContracts } from "@mocode/shared"; import { apiClient } from "../lib/api-client"; import { getErrorMessage } from "../lib/http-errors"; +import { createFailoverFetch, type DowngradeInfo } from "../lib/chat-fetch"; import { getAuth } from "../lib/auth"; import { executeLocalTool } from "../lib/local-tools"; import { requiresApproval, rememberSessionAllow } from "../lib/bash-approval"; @@ -58,6 +59,10 @@ import { type ResumeEligibility, } from "../lib/stream-interrupt"; import { formatChatStreamError } from "../lib/stream-error"; +import { AgentLoop, type AgentProgress, type AgentStopReason } from "../lib/agent-loop"; +import { failoverConsentFromEnv } from "../lib/provider-chain"; +import { deriveAgentPhase, isRunActive, type AgentPhase } from "../lib/agent-phase"; +import { markToolStarted, resetToolTimings } from "../lib/tool-timing"; import { scheduleLocalSessionPersist } from "./use-chat-persist"; import { killTrackedToolProcesses } from "../lib/tool-process-registry"; import { @@ -204,6 +209,22 @@ export function useChat( /** Blocks sendAutomaticallyWhen after Esc finalizes tool output-error parts. */ const turnInterruptedRef = useRef(false); const [turnInterrupted, setTurnInterrupted] = useState(false); + /** + * Explicit loop budget for the current user message: turn cap, per-turn tool + * cap, and the consecutive-repeat guard. See `lib/agent-loop.ts`. + */ + const agentLoopRef = useRef(new AgentLoop()); + const [agentProgress, setAgentProgress] = useState(null); + /** Non-null when a fallback model answered — must stay visible, never silent. */ + const [agentDowngrade, setAgentDowngrade] = useState(null); + /** Live elapsed time for the current run, so the footer can count up. */ + const [agentElapsedMs, setAgentElapsedMs] = useState(null); + const runStartedAtRef = useRef(null); + const reportDowngradeRef = useRef<(info: DowngradeInfo) => void>(() => {}); + const [agentStop, setAgentStop] = useState<{ + reason: AgentStopReason; + detail: string; + } | null>(null); /** Active subagent run — Esc aborts via linked AbortController (D-10). */ const subagentAbortRef = useRef(null); const [subagentRunning, setSubagentRunning] = useState(false); @@ -213,12 +234,43 @@ export function useChat( const onPersistErrorRef = useRef(options?.onPersistError); onPersistErrorRef.current = options?.onPersistError; + // Latest-ref so the memoised transport can report a downgrade without re-creating. + reportDowngradeRef.current = (info: DowngradeInfo) => { + setAgentDowngrade(info); + toast.show({ + variant: "error", + message: `模型 ${info.from} 不可用,已回退到 ${info.to}`, + }); + }; + + /** + * Surface a loop stop. Deferred via microtask because this is called from the + * SDK's `sendAutomaticallyWhen` predicate, which runs during render reconciliation. + * A stop can be observed twice (tool gate + turn gate), so it reports once. + */ + const agentStopReportedRef = useRef(false); + const reportAgentStop = useCallback( + (decision: { reason: AgentStopReason; detail: string }) => { + // Esc already renders its own interrupted state — no toast for it. + if (decision.reason === "aborted" || agentStopReportedRef.current) return; + agentStopReportedRef.current = true; + queueMicrotask(() => { + setAgentStop({ reason: decision.reason, detail: decision.detail }); + toast.show({ variant: "error", message: decision.detail }); + }); + }, + [toast], + ); + const transport = useMemo(() => { // BYOK: inference + tool schema merge run in-process; MCP execution stays in onToolCall below. if (isLocalMode()) { return new LocalChatTransport({ resolveModel: resolveChatModel, getMcpManager, + // BYOK failures happen in-process; report the downgrade the same way as SaaS. + allowFailover: failoverConsentFromEnv(), + onDowngrade: (info) => reportDowngradeRef.current({ ...info, failures: [] }), buildSystemPrompt: (params) => buildSystemPrompt({ ...params, @@ -227,14 +279,12 @@ export function useChat( }) as ChatTransport; } - const chatFetch = (async (input, init) => { - const response = await globalThis.fetch(input, init); - if (!response.ok) { - const message = await getErrorMessage(response); - throw new Error(message); - } - return response; - }) as typeof globalThis.fetch; + // Retries the request on the next model when a provider is blocked (see lib/chat-fetch.ts). + const chatFetch = createFailoverFetch({ + readError: getErrorMessage, + allowFailover: failoverConsentFromEnv(), + onDowngrade: (info) => reportDowngradeRef.current(info), + }); return new DefaultChatTransport({ api: apiClient.chat.$url().toString(), @@ -351,6 +401,9 @@ export function useChat( messages: initialMessages, transport, async onToolCall({ toolCall }) { + // Started clock for the per-tool elapsed label; presentation only. + markToolStarted(toolCall.toolCallId); + const shouldSkipToolOutput = () => shouldSkipInterruptedToolOutput(toolCall.toolCallId, skipToolOutputIdsRef.current); @@ -361,6 +414,31 @@ export function useChat( const isMcpCall = looksLikeMcpToolName(toolCall.toolName); + // Loop budget: account for every tool call before it runs, so a run that is + // out of turns / out of per-turn calls / repeating itself stops with a named + // reason instead of silently continuing. + const loopDecision = agentLoopRef.current.recordToolCall(toolCall.toolName, toolCall.input); + if (loopDecision.action === "stop") { + reportAgentStop(loopDecision); + if (!shouldSkipToolOutput()) { + const addStopOutput = chat.addToolOutput as (params: { + tool?: keyof ChatTools; + toolCallId: string; + state: "output-error"; + errorText: string; + }) => void; + addStopOutput({ + tool: toolCall.toolName as keyof ChatTools, + toolCallId: toolCall.toolCallId, + state: "output-error", + errorText: loopDecision.detail, + }); + } + return; + } + + setAgentProgress(agentLoopRef.current.snapshot()); + // Phase 04 (04-03): Task subagent — synchronous blocking, summary-only (D-03, D-04). if (toolCall.toolName === "task") { if (shouldSkipToolOutput()) return; @@ -581,7 +659,22 @@ export function useChat( }, sendAutomaticallyWhen: (params) => { if (turnInterruptedRef.current) return false; - return lastAssistantMessageIsCompleteWithToolCalls(params); + + // A new user message starts a fresh budget; regenerating keeps the current one. + const lastUser = params.messages.findLast((message) => message.role === "user"); + if (lastUser) agentLoopRef.current.beginUserMessage(lastUser.id); + + if (!lastAssistantMessageIsCompleteWithToolCalls(params)) return false; + + const decision = agentLoopRef.current.beginTurn(); + if (decision.action === "stop") { + reportAgentStop(decision); + return false; + } + + // Status-line budget; deferred for the same reason as reportAgentStop. + queueMicrotask(() => setAgentProgress(agentLoopRef.current.snapshot())); + return true; }, }); @@ -618,6 +711,7 @@ export function useChat( setActiveSlashMessages(null); turnInterruptedRef.current = true; setTurnInterrupted(true); + agentLoopRef.current.abort(); killTrackedToolProcesses(); const snapshot = snapshotChatMessages(chat); @@ -652,6 +746,13 @@ export function useChat( turnInterruptedRef.current = false; setTurnInterrupted(false); skipToolOutputIdsRef.current.clear(); + // Explicit retry: hand the model a fresh budget, otherwise a run that hit + // `max-turns` would stop again immediately on resume. + agentLoopRef.current.reset(); + agentStopReportedRef.current = false; + setAgentStop(null); + setAgentProgress(null); + setAgentDowngrade(null); const eligibility = detectResumeEligibility(chat.messages, chat.status); const action = resolveResumeTransport(eligibility); if (action === "none") return; @@ -743,6 +844,11 @@ export function useChat( turnInterruptedRef.current = false; setTurnInterrupted(false); skipToolOutputIdsRef.current.clear(); + resetToolTimings(); + agentStopReportedRef.current = false; + setAgentStop(null); + setAgentProgress(null); + setAgentDowngrade(null); const trimmed = params.userText.trim(); const exploreMatch = /^\/explore(?:\s+(.*))?$/s.exec(trimmed); @@ -782,10 +888,46 @@ export function useChat( [chat.sendMessage, runSlashSubagent, toast.show], ); + // Tick the run timer while the agent is busy. 200ms is fast enough to look live + // and cheap enough to ignore. + // Active means "the loop is doing something", not "the HTTP stream is open": + // tool execution and the gap before auto-resubmit also count. + const runActive = isRunActive({ status: chat.status, message: chat.messages.at(-1) }); + + useEffect(() => { + const busy = runActive; + + if (!busy) { + runStartedAtRef.current = null; + setAgentElapsedMs(null); + return; + } + + runStartedAtRef.current ??= Date.now(); + const timer = setInterval(() => { + const startedAt = runStartedAtRef.current; + if (startedAt !== null) setAgentElapsedMs(Date.now() - startedAt); + }, 200); + + return () => clearInterval(timer); + }, [runActive]); + + // Derived from the live message, so the label can never disagree with what is + // already on screen. + const agentPhase: AgentPhase | null = useMemo( + () => deriveAgentPhase({ status: chat.status, message: chat.messages.at(-1) }), + [chat.status, chat.messages], + ); + return { messages: displayMessages, status: chat.status, + agentPhase, + agentElapsedMs, turnInterrupted, + agentProgress, + agentDowngrade, + agentStop, subagentRunning, subagentType, error: chat.error diff --git a/packages/cli/src/lib/bash-approval.test.ts b/packages/cli/src/lib/bash-approval.test.ts index 729634d..65c048e 100644 --- a/packages/cli/src/lib/bash-approval.test.ts +++ b/packages/cli/src/lib/bash-approval.test.ts @@ -17,11 +17,11 @@ describe("requiresApproval", () => { { name: "curl pipe bash", command: "curl https://example.com/install.sh | bash" }, { name: "wget pipe sh", command: "wget -O - https://example.com/script | sh" }, { name: "dd disk write", command: "dd if=/dev/zero of=/dev/sda" }, - { name: "redirect to /dev/", command: "echo foo > /dev/null" }, + { name: "redirect to block device", command: "echo foo > /dev/sda" }, ]; test.each(dangerousCommands)("$name requires approval", ({ command }) => { - expect(requiresApproval(command, emptyAllowlist)).toBe(true); + expect(requiresApproval(command, emptyAllowlist, { confined: true })).toBe(true); }); const safeCommands = [ @@ -31,6 +31,35 @@ describe("requiresApproval", () => { ]; test.each(safeCommands)("$name does not require approval", ({ command }) => { - expect(requiresApproval(command, emptyAllowlist)).toBe(false); + expect(requiresApproval(command, emptyAllowlist, { confined: true })).toBe(false); + }); +}); + +describe("unconfined shell", () => { + test("every command requires approval when no OS sandbox applies", () => { + // Without containment a blocklist is theatre: `head /etc/passwd` would pass. + expect(requiresApproval("npm test", emptyAllowlist, { confined: false })).toBe(true); + expect(requiresApproval("echo hi", emptyAllowlist, { confined: false })).toBe(true); + }); + + test("the session allowlist still short-circuits", () => { + const allowed = new Set(["echo hi"]); + expect(requiresApproval("echo hi", allowed, { confined: false })).toBe(false); + }); +}); + +describe("redirects to /dev/null are not treated as device writes", () => { + test("stderr suppression needs no approval", () => { + expect(requiresApproval("ls -la 2>/dev/null", emptyAllowlist, { confined: true })).toBe(false); + expect( + requiresApproval("ls -la secret.txt 2>/dev/null; echo x; ls -la | head -30", emptyAllowlist, { + confined: true, + }), + ).toBe(false); + }); + + test("writing into a real block device still requires approval", () => { + expect(requiresApproval("echo x > /dev/sda", emptyAllowlist, { confined: true })).toBe(true); + expect(requiresApproval("cat img > /dev/disk2", emptyAllowlist, { confined: true })).toBe(true); }); }); diff --git a/packages/cli/src/lib/bash-approval.ts b/packages/cli/src/lib/bash-approval.ts index fe5e0c1..4c897bb 100644 --- a/packages/cli/src/lib/bash-approval.ts +++ b/packages/cli/src/lib/bash-approval.ts @@ -1,12 +1,17 @@ /** - * Blocklist-based bash approval for Build mode (HARNESS-03). + * Bash approval policy (HARNESS-03). * - * Only commands matching D-13 patterns require user confirmation. - * Session allowlist is owned by use-chat (passed in, not a global singleton). + * Two regimes, because the honest answer depends on the host: + * + * 1. **Confined shell** — an OS sandbox (seatbelt / bwrap) applies, so the + * blocklist is a backup: only listed destructive commands ask. + * 2. **Unconfined shell** — no sandbox is available. An unsandboxed shell can read + * anything the user can, so a blocklist would be theatre and *every* command asks. * - * Limits: whitespace normalization only; obfuscation via env vars or quoting - * may bypass patterns — acceptable per phase discretion. + * Session allowlist is owned by use-chat (passed in, not a global singleton). */ +import { isShellConfined } from "./sandbox-capability"; + export type BashApprovalVerdict = "approve-once" | "reject" | "allow-session"; /** @@ -23,7 +28,9 @@ const BLOCKLIST: RegExp[] = [ /\bchmod\s+[^\n]*(-R|--recursive)\b/, // recursive permission changes /\b(curl|wget)\s+[^\n]*\|\s*(ba)?sh\b/, // pipe remote script directly into shell /\bdd\s+[^\n]*if=/, // raw disk write (dd if=/dev/...) - />\s*\/dev\//, // redirect stdout/stderr into a device node + // Only block devices. `2>/dev/null` is everywhere and harmless; matching it made + // the dialog fire on ordinary commands, which trains users to approve blindly. + />\s*\/dev\/(?:sd|disk|nvme|hd|rdisk)/, // redirect into a block device ]; /** Trim and collapse internal whitespace for blocklist and allowlist keys. */ @@ -36,9 +43,24 @@ export function rememberSessionAllow(sessionAllowed: Set, command: strin sessionAllowed.add(normalizeCommand(command)); } -/** True when command matches the blocklist and is not session-allowed. */ -export function requiresApproval(command: string, sessionAllowed: Set): boolean { +/** + * True when command matches the blocklist and is not session-allowed. + * + * `confined` describes whether the spawned shell is OS-sandboxed. It defaults to + * the real capability, because the distinction changes the answer: an unconfined + * shell can read anything, so a blocklist is not a safety net and *every* command + * needs a human decision. + */ +export function requiresApproval( + command: string, + sessionAllowed: Set, + options: { confined?: boolean } = {}, +): boolean { const normalized = normalizeCommand(command); if (sessionAllowed.has(normalized)) return false; + + const confined = options.confined ?? isShellConfined(); + if (!confined) return true; + return BLOCKLIST.some((re) => re.test(normalized)); } diff --git a/packages/cli/src/lib/local-chat-transport.test.ts b/packages/cli/src/lib/local-chat-transport.test.ts index 2198b20..0f483b2 100644 --- a/packages/cli/src/lib/local-chat-transport.test.ts +++ b/packages/cli/src/lib/local-chat-transport.test.ts @@ -46,6 +46,138 @@ function createMockResolvedModel() { }; } +/** Minimal sendMessages args for a single user turn on `model`. */ +function sendArgs(model: string, chatId = "session-1") { + return { + trigger: "submit-message" as const, + chatId, + messageId: undefined, + messages: [ + { + id: "msg-1", + role: "user" as const, + parts: [{ type: "text" as const, text: "hello" }], + metadata: { mode: Mode.BUILD, model }, + }, + ], + abortSignal: undefined, + }; +} + +async function drain(stream: ReadableStream) { + const reader = stream.getReader(); + while (true) { + const { done } = await reader.read(); + if (done) break; + } +} + +describe("LocalChatTransport failover (BYOK)", () => { + test("retries the next model when the primary provider is blocked", async () => { + const downgrades: Array<{ from: string; to: string }> = []; + const transport = new LocalChatTransport({ + resolveModel: (modelId) => { + if (modelId === "gpt-oss-120b") { + throw Object.assign(new Error("blocked"), { statusCode: 403 }); + } + return createMockResolvedModel(); + }, + getMcpManager: createMockManager, + buildSystemPrompt: () => "test system prompt", + allowFailover: true, + attemptsPerModel: 1, + onDowngrade: (info) => downgrades.push(info), + }); + + const stream = await transport.sendMessages(sendArgs("gpt-oss-120b")); + await drain(stream); + + expect(downgrades).toEqual([{ from: "gpt-oss-120b", to: "deepseek-flash" }]); + }); + + test("does not fall back on a non-retryable failure", async () => { + const transport = new LocalChatTransport({ + resolveModel: () => { + throw Object.assign(new Error("bad request"), { statusCode: 400 }); + }, + getMcpManager: createMockManager, + buildSystemPrompt: () => "test system prompt", + }); + + await expect(transport.sendMessages(sendArgs("gpt-oss-120b"))).rejects.toThrow("bad request"); + }); + + test("reports every failed attempt before succeeding", async () => { + const failures: string[] = []; + const transport = new LocalChatTransport({ + resolveModel: (modelId) => { + if (modelId !== "deepseek-flash") { + throw Object.assign(new Error("blocked"), { statusCode: 403 }); + } + return createMockResolvedModel(); + }, + getMcpManager: createMockManager, + buildSystemPrompt: () => "test system prompt", + allowFailover: true, + attemptsPerModel: 1, + onAttemptFailed: ({ modelId, decision }) => failures.push(`${modelId}:${decision.detail}`), + }); + + await drain(await transport.sendMessages(sendArgs("claude-opus-4-6"))); + + expect(failures).toEqual(["claude-opus-4-6:HTTP 403"]); + }); + + test("fails over when the stream emits start then a provider error", async () => { + const downgrades: Array<{ from: string; to: string }> = []; + streamTextMock.mockClear(); + + streamTextMock + .mockImplementationOnce(() => ({ + toUIMessageStream: () => + new ReadableStream({ + start(controller) { + controller.enqueue({ type: "start" }); + controller.enqueue({ type: "error", errorText: "Forbidden" }); + controller.close(); + }, + }), + })) + .mockImplementationOnce(() => ({ + toUIMessageStream: () => + new ReadableStream({ + start(controller) { + controller.enqueue({ type: "start" }); + controller.enqueue({ type: "text-delta", id: "t1", delta: "recovered" }); + controller.close(); + }, + }), + })); + + const transport = new LocalChatTransport({ + resolveModel: () => createMockResolvedModel(), + getMcpManager: createMockManager, + buildSystemPrompt: () => "test system prompt", + allowFailover: true, + attemptsPerModel: 1, + onDowngrade: (info) => downgrades.push(info), + }); + + const chunks: unknown[] = []; + const stream = await transport.sendMessages(sendArgs("gpt-oss-120b")); + const reader = stream.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + } + + expect(streamTextMock.mock.calls.length).toBe(2); + expect(downgrades).toEqual([{ from: "gpt-oss-120b", to: "deepseek-flash" }]); + expect(chunks).toContainEqual({ type: "text-delta", id: "t1", delta: "recovered" }); + }); +}); + describe("LocalChatTransport", () => { beforeEach(() => { lastStreamTextTools = undefined; @@ -127,7 +259,8 @@ describe("reconnectToStream (D-12 BYOK)", () => { controller.close(); return; } - controller.enqueue(new TextEncoder().encode('data: {"type":"text-delta"}\n\n')); + controller.enqueue({ type: "start" }); + controller.enqueue({ type: "text-delta", id: "t1", delta: "hi" }); }, }), })); @@ -197,7 +330,8 @@ describe("reconnectToStream (D-12 BYOK)", () => { toUIMessageStream: () => new ReadableStream({ start(controller) { - controller.enqueue(new TextEncoder().encode('data: {"type":"text-delta"}\n\n')); + controller.enqueue({ type: "start" }); + controller.enqueue({ type: "text-delta", id: "t1", delta: "hi" }); }, }), })); diff --git a/packages/cli/src/lib/local-chat-transport.ts b/packages/cli/src/lib/local-chat-transport.ts index 53ca06f..d2ec8a4 100644 --- a/packages/cli/src/lib/local-chat-transport.ts +++ b/packages/cli/src/lib/local-chat-transport.ts @@ -10,6 +10,12 @@ * 3. Detect MCP intent in last user turn → `mcpRequested` strengthens system prompt * 4. validateUIMessages → convertToModelMessages → streamText → toUIMessageStream * + * Failover: an attempt only counts as started once a *useful* UI chunk arrives + * (not a bare `start`). A provider that rejects the connection — or emits + * `start` then an `error` — therefore fails *before* anything is rendered, and + * the next model in the chain is tried (see `lib/provider-chain.ts`). + * Once a useful chunk has been emitted the turn is visible and is never replayed. + * * Tool execution still happens in `use-chat` `onToolCall` (Phase 11 model). */ import { @@ -22,6 +28,7 @@ import { type UIMessageChunk, } from "ai"; import { Mode, type ModeType, type SupportedChatModelId } from "@mocode/shared"; +import { DEFAULT_COMPACTION_POLICY, compactMessages } from "@mocode/shared"; import { loadMergedMcpConfig } from "../mcp/config"; import type { McpManager } from "../mcp/manager"; import { @@ -32,6 +39,8 @@ import { isMcpToolName } from "../mcp/heuristics"; import type { ResolvedModel } from "./local-model"; import { formatChatStreamError } from "./stream-error"; import { hasVisibleAssistantContent } from "@mocode/shared"; +import { runWithFailover, type FailoverDecision } from "./provider-chain"; +import { stallTimeoutMs } from "./chat-fetch"; type LocalChatMetadata = { mode?: ModeType; @@ -74,6 +83,124 @@ function isMcpUserRequest(text: string): boolean { return /\bmcp\b/i.test(text) || /\bmcp__[\w-]+__[\w-]+/i.test(text); } +/** Per-attempt budget covering connect + first useful chunk only. */ + + +/** Chunks that prove the provider is actually producing a turn. */ +const USEFUL_CHUNK_TYPES = new Set([ + "text-start", + "text-delta", + "reasoning-start", + "reasoning-delta", + "tool-input-start", + "tool-input-delta", + "tool-input-available", + "tool-call", + "finish", + "finish-step", +]); + +function isUsefulChunk(chunk: UIMessageChunk): boolean { + return USEFUL_CHUNK_TYPES.has(chunk.type); +} + +function isErrorChunk(chunk: UIMessageChunk): boolean { + return chunk.type === "error"; +} + +function errorChunkMessage(chunk: UIMessageChunk): string { + if ("errorText" in chunk && typeof chunk.errorText === "string" && chunk.errorText.length > 0) { + return chunk.errorText; + } + return "upstream provider error"; +} + +/** Shape `classifyFailover` treats as a retryable timeout. */ +function attemptTimeoutError(): Error { + const error = new Error("attempt timed out before useful output"); + error.name = "TimeoutError"; + return error; +} + +/** Shape `classifyFailover` treats as a non-retryable user abort. */ +function userAbortError(): Error { + const error = new Error("The operation was aborted"); + error.name = "AbortError"; + return error; +} + +/** + * Read until a useful chunk confirms success, an `error` chunk fails the + * attempt, the stream ends, the per-attempt deadline expires, or the user aborts. + * Leading non-useful chunks (e.g. `start`) are buffered and returned with the reader. + */ +async function readUntilAttemptSettled( + reader: { + read: () => Promise<{ done: boolean; value?: UIMessageChunk }>; + cancel: (reason?: unknown) => Promise; + }, + abortSignal: AbortSignal | undefined, + timeoutMs: number, +): Promise<{ + reader: { + read: () => Promise<{ done: boolean; value?: UIMessageChunk }>; + cancel: (reason?: unknown) => Promise; + }; + prefix: UIMessageChunk[]; + done: boolean; +}> { + const prefix: UIMessageChunk[] = []; + const deadline = new AbortController(); + const timer = setTimeout(() => deadline.abort(), timeoutMs); + + const abortAttempt = () => { + clearTimeout(timer); + // Cancelling is what actually unblocks a pending `read()`. Without this the + // deadline only sets a flag nobody observes and a silent provider hangs forever. + void reader.cancel().catch(() => {}); + }; + + const onAbort = () => abortAttempt(); + + try { + abortSignal?.addEventListener("abort", onAbort, { once: true }); + deadline.signal.addEventListener("abort", onAbort, { once: true }); + + while (true) { + const { done, value } = await reader.read(); + + // Check cancellation *before* interpreting the chunk: a read unblocked by our + // own cancel, or a late error chunk after an abort, must not become a + // retryable 502 that sends the turn to another provider. + if (abortSignal?.aborted) throw userAbortError(); + if (deadline.signal.aborted) throw attemptTimeoutError(); + + if (done || value === undefined) { + clearTimeout(timer); + return { reader, prefix, done: true }; + } + + if (isErrorChunk(value)) { + abortAttempt(); + throw Object.assign(new Error(errorChunkMessage(value)), { statusCode: 502 }); + } + + prefix.push(value); + + if (isUsefulChunk(value)) { + clearTimeout(timer); + return { reader, prefix, done: false }; + } + } + } catch (error) { + clearTimeout(timer); + throw error; + } finally { + abortSignal?.removeEventListener("abort", onAbort); + deadline.signal.removeEventListener("abort", onAbort); + } +} + /** Drops assistant rows with no user-visible content (empty or step-start-only placeholders). */ export function stripIncompleteAssistantMessages( messages: UI_MESSAGE[], @@ -98,6 +225,16 @@ export type LocalChatTransportOptions = { skills?: Array<{ name: string; description: string }>; }) => string; onFinish?: (event: LocalChatTransportFinishEvent) => Promise; + /** Explicit opt-in to trying another provider; see `FailoverOptions.consent`. */ + allowFailover?: boolean; + /** Attempts per model; defaults to `MOCODE_PROVIDER_RETRIES` or 3. */ + attemptsPerModel?: number; + /** Injectable for tests so they do not wait on real backoff. */ + sleep?: (ms: number) => Promise; + /** Called when a fallback model answered — the caller must show this to the user. */ + onDowngrade?: (info: { from: string; to: string }) => void; + /** Called for every failed attempt, for logging. */ + onAttemptFailed?: (event: { modelId: string; decision: FailoverDecision }) => void; cwd?: string; }; @@ -143,94 +280,144 @@ export class LocalChatTransport const mcpToolNames = Object.keys(tools).filter(isMcpToolName); const mcpRequested = isMcpUserRequest(lastUserText(messages)); const systemPrompt = this.opts.buildSystemPrompt({ mode, mcpToolNames, mcpRequested }); - const resolvedModel = this.opts.resolveModel(modelId); - const startTime = Date.now(); - let completedUsage: LanguageModelUsage | null = null; const messagesForValidation = stripIncompleteAssistantMessages(messages); const nextMessages = await validateUIMessages({ messages: messagesForValidation, tools: tools as Parameters>[0]["tools"], }); - const modelMessages = await convertToModelMessages(nextMessages, { tools }); - - const result = streamText({ - model: resolvedModel.model, - system: systemPrompt, - messages: modelMessages, - tools, - providerOptions: resolvedModel.providerOptions, - abortSignal, - onFinish(event) { - completedUsage = event.totalUsage; - }, - }); + // Same policy as the server path: fold old turns out of the *model input* only, + // so the client keeps its full transcript. + const compaction = compactMessages(nextMessages, DEFAULT_COMPACTION_POLICY); + const modelMessages = await convertToModelMessages(compaction.messages, { tools }); + const startTime = Date.now(); const onFinish = this.opts.onFinish; - const self = this; - - const rawStream = result.toUIMessageStream({ - originalMessages: nextMessages, - onError: formatChatStreamError, - messageMetadata({ part }) { - if (part.type === "start") { - return { mode, model: modelId } as never; - } + // Bound to *this* send's stream so an older overlapping cleanup cannot clear a newer one. + let ownedStream: ReadableStream | null = null; + const clearActive = () => { + if ( + this.activeStream?.chatId === chatId && + ownedStream !== null && + this.activeStream.stream === ownedStream + ) { + this.activeStream = null; + } + }; - if (part.type !== "finish") return undefined; + /** + * One attempt = create the stream and wait for useful output (or a decisive + * error). Provider failures reject here, before any chunk is enqueued to the + * caller, which is what makes a retry safe. + */ + const outcome = await runWithFailover({ + primary: modelId, + consent: this.opts.allowFailover === true, + attemptsPerModel: this.opts.attemptsPerModel, + sleep: this.opts.sleep, + attempt: async (candidate) => { + let completedUsage: LanguageModelUsage | null = null; + const resolved = this.opts.resolveModel(candidate); - return { - mode, - model: modelId, - durationMs: Date.now() - startTime, - ...(completedUsage ? { usage: completedUsage } : {}), - } as never; - }, - async onFinish(event) { - if (self.activeStream?.stream === stream) self.activeStream = null; - if (!onFinish) return; - await onFinish({ - messages: event.messages, - usage: completedUsage ?? undefined, + const result = streamText({ + model: resolved.model, + system: systemPrompt, + messages: modelMessages, + tools, + providerOptions: resolved.providerOptions, + abortSignal, + onFinish(event) { + completedUsage = event.totalUsage; + }, }); + + const rawStream = result.toUIMessageStream({ + originalMessages: nextMessages, + onError: formatChatStreamError, + messageMetadata({ part }) { + if (part.type === "start") { + return { mode, model: candidate } as never; + } + + if (part.type !== "finish") return undefined; + + return { + mode, + model: candidate, + durationMs: Date.now() - startTime, + ...(completedUsage ? { usage: completedUsage } : {}), + } as never; + }, + async onFinish(event) { + clearActive(); + if (!onFinish) return; + await onFinish({ + messages: event.messages, + usage: completedUsage ?? undefined, + }); + }, + }); + + const reader = rawStream.getReader(); + return readUntilAttemptSettled( + reader as { + read: () => Promise<{ done: boolean; value?: UIMessageChunk }>; + cancel: (reason?: unknown) => Promise; + }, + abortSignal, + stallTimeoutMs(), + ); }, + onAttemptFailed: this.opts.onAttemptFailed, }); + if (outcome.downgradedFrom) { + this.opts.onDowngrade?.({ from: outcome.downgradedFrom, to: outcome.modelId }); + } + + const { reader, prefix, done } = outcome.value; + const stream = new ReadableStream({ start(controller) { - const reader = rawStream.getReader(); + for (const chunk of prefix) { + controller.enqueue(chunk); + } + + if (done) { + clearActive(); + controller.close(); + return; + } + void (async () => { try { while (true) { - const { done, value } = await reader.read(); - if (done) { - if (self.activeStream?.stream === stream) self.activeStream = null; + const { done: finished, value } = await reader.read(); + if (finished) { + clearActive(); controller.close(); return; } controller.enqueue(value); } } catch (error) { - if (self.activeStream?.stream === stream) self.activeStream = null; + clearActive(); controller.error(error); } })(); }, cancel(reason) { - if (self.activeStream?.stream === stream) self.activeStream = null; - return rawStream.cancel(reason); + clearActive(); + return reader.cancel(reason); }, }); - this.activeStream = { chatId, stream }; - - abortSignal?.addEventListener( - "abort", - () => { - if (this.activeStream?.stream === stream) this.activeStream = null; - }, - { once: true }, - ); + ownedStream = stream; + // A finished attempt (empty or already drained) is not reconnectable. + if (!done) { + this.activeStream = { chatId, stream }; + abortSignal?.addEventListener("abort", clearActive, { once: true }); + } return stream; } diff --git a/packages/cli/src/lib/local-tools.ts b/packages/cli/src/lib/local-tools.ts index 534eca0..af53bf6 100644 --- a/packages/cli/src/lib/local-tools.ts +++ b/packages/cli/src/lib/local-tools.ts @@ -18,11 +18,16 @@ import { createReadStream } from "node:fs"; import { createInterface } from "node:readline"; import { mkdir, readFile, readdir, stat, writeFile } from "fs/promises"; -import { dirname, isAbsolute, join, relative, resolve } from "path"; +import { dirname, join, relative, resolve } from "path"; import { toolInputSchemas, Mode, type ModeType } from "@mocode/shared"; import { simpleGit } from "simple-git"; import { runRipgrep } from "./ripgrep"; import { trackToolProcess } from "./tool-process-registry"; +import { resolveInsideCwd as resolveSafePath } from "./sandbox-path"; +import { ToolPermissionError, buildSandboxEnv, findHardDeny } from "./permission-rules"; +import { recordAudit, summarizeInput } from "./audit-log"; +import { isShellConfined, sandboxBackend } from "./sandbox-capability"; +import { parsePathList, runtimePaths, spawnPlan } from "./sandbox-profile"; /** Max chars returned from readFile before truncation metadata is attached. */ const MAX_FILE_SIZE = 10_000; @@ -45,18 +50,23 @@ const READ_ONLY_TOOLS = [ /** * Resolve `path` relative to `process.cwd()` and reject escapes outside it. + * + * Containment is checked twice — lexically and against real paths — so a symlink + * inside the project cannot point at `/etc`. See `sandbox-path.ts`. + * * @throws when the resolved absolute path leaves the project directory. */ function resolveInsideCwd(path: string) { const cwd = process.cwd(); - const resolved = resolve(cwd, path); - const rel = relative(cwd, resolved); - if (rel.startsWith("..") || isAbsolute(rel)) { - throw new Error("Path is outside the project directory"); + try { + return { cwd, resolved: resolveSafePath(cwd, path) }; + } catch (error) { + throw new ToolPermissionError( + "PATH_ESCAPE", + error instanceof Error ? error.message : "Path is outside the project directory", + ); } - - return { cwd, resolved }; } /** Append a truncation notice when shell output exceeds {@link MAX_OUTPUT}. */ @@ -67,12 +77,19 @@ function truncate(value: string, limit: number) { } -/** Streams only the requested line range — avoids loading entire large files. */ +/** + * Streams only the requested line range. + * + * Keeps at most `limit` characters of output while still counting the full + * selected range length, so truncation metadata stays accurate without holding + * the entire selection in memory. + */ async function readLinesFromFile( resolved: string, lineStart?: number, lineEnd?: number, -): Promise { + limit: number = MAX_FILE_SIZE, +): Promise<{ content: string; truncated?: boolean; totalLength?: number }> { const startLine = lineStart ?? 1; const lines: string[] = []; const rl = createInterface({ @@ -81,14 +98,40 @@ async function readLinesFromFile( }); let lineNumber = 0; + let selectedLength = 0; + let keptLength = 0; + let truncated = false; + // Length is not a proxy for "a line has been selected": the first selected line + // may be empty, and then the *second* line would not count its separator. + let hasSelected = false; + let hasKept = false; + for await (const line of rl) { lineNumber += 1; if (lineNumber < startLine) continue; if (lineEnd !== undefined && lineNumber > lineEnd) break; - lines.push(line); + + selectedLength += (hasSelected ? 1 : 0) + line.length; + hasSelected = true; + + if (truncated) continue; + + const nextKept = keptLength + (hasKept ? 1 : 0) + line.length; + if (nextKept <= limit) { + lines.push(line); + keptLength = nextKept; + hasKept = true; + continue; + } + + const remaining = limit - keptLength - (hasKept ? 1 : 0); + if (remaining > 0) lines.push(line.slice(0, remaining)); + keptLength = limit; + truncated = true; } - return lines.join("\n"); + const content = lines.join("\n"); + return truncated ? { content, truncated: true, totalLength: selectedLength } : { content }; } /** Streams file content up to a character limit — avoids loading entire large files. */ @@ -122,15 +165,6 @@ async function readFileUpToCharLimit( return { content: chunks.join(""), truncated }; } -function applyCharLimit( - content: string, - limit: number, -): { content: string; truncated?: boolean; totalLength?: number } { - return content.length > limit - ? { content: content.slice(0, limit), truncated: true, totalLength: content.length } - : { content }; -} - /** * Dispatch a single tool call to the matching local handler. * @@ -139,12 +173,12 @@ function applyCharLimit( * @param mode - Session mode; PLAN rejects write/bash tools as a second guard * (the server already omits them from `getToolContracts(PLAN)`). */ -export async function executeLocalTool(toolName: string, input: unknown, mode: ModeType) { +async function runTool(toolName: string, input: unknown, mode: ModeType) { if ( mode === Mode.PLAN && !READ_ONLY_TOOLS.includes(toolName as (typeof READ_ONLY_TOOLS)[number]) ) { - throw new Error(`Tool ${toolName} is not available in PLAN mode`); + throw new ToolPermissionError("MODE_DENIED", `Tool ${toolName} is not available in PLAN mode`); } switch (toolName) { @@ -153,15 +187,7 @@ export async function executeLocalTool(toolName: string, input: unknown, mode: M const { resolved } = resolveInsideCwd(path); if (line_start !== undefined || line_end !== undefined) { - const content = await readLinesFromFile(resolved, line_start, line_end); - const limited = applyCharLimit(content, MAX_FILE_SIZE); - return limited.truncated - ? { - content: limited.content, - truncated: true, - totalLength: limited.totalLength, - } - : { content: limited.content }; + return readLinesFromFile(resolved, line_start, line_end, MAX_FILE_SIZE); } const { content, truncated } = await readFileUpToCharLimit(resolved, MAX_FILE_SIZE); @@ -276,22 +302,86 @@ export async function executeLocalTool(toolName: string, input: unknown, mode: M return { success: true as const, path: relative(cwd, resolved) }; } case "bash": { - // Runs in the sandboxed project cwd; TERM=dumb avoids escape-sequence noise in the TUI. + // Hard denies are refused even when the user approved the command: approval + // covers destructive-but-legitimate work, not host-destroying commands. const { command, timeout = DEFAULT_TIMEOUT } = toolInputSchemas.bash.parse(input); - const proc = Bun.spawn(["bash", "-c", command], { - cwd: resolveInsideCwd(".").resolved, - stdout: "pipe", - stderr: "pipe", - env: { ...process.env, TERM: "dumb" }, + const hardDeny = findHardDeny(command); + if (hardDeny) { + throw new ToolPermissionError( + "HARD_DENY", + `Command refused (${hardDeny.id}): ${hardDeny.reason}. This cannot be approved.`, + ); + } + + // Runs in the project cwd with an allowlisted env; TERM=dumb avoids escape + // sequences leaking into the TUI. API keys are deliberately absent. + // + // A shell cannot be contained by path checks, so the spawn is wrapped in the + // host's OS sandbox when one is actually available. When it is not, the run + // is unconfined and the approval policy requires a human for every command. + const cwd = resolveInsideCwd(".").resolved; + const plan = spawnPlan(sandboxBackend(), command, { + cwd, + home: process.env.HOME, + tmpDir: process.env.TMPDIR, + runtimePaths: runtimePaths(), + extraReadPaths: parsePathList(process.env.MOCODE_SANDBOX_EXTRA_READ), + extraWritePaths: parsePathList(process.env.MOCODE_SANDBOX_EXTRA_WRITE), }); + + // Under bwrap the host TMPDIR is a read-only bind; the plan mounts a private + // tmpfs at /tmp, so point the child there. + const childEnv = + plan.backend === "bwrap" ? { ...buildSandboxEnv(), TMPDIR: "/tmp", TMP: "/tmp" } : buildSandboxEnv(); + + const proc = (() => { + try { + return Bun.spawn([plan.file, ...plan.args], { + cwd, + stdout: "pipe", + stderr: "pipe", + env: childEnv, + }); + } catch (error) { + // A sandbox that fails to start must not look like "the command printed + // nothing" — that silently teaches the model to hallucinate output. + const reason = error instanceof Error ? error.message : String(error); + throw new Error( + `Shell could not start (${plan.backend} sandbox): ${reason}. ` + + `Set MOCODE_DISABLE_SANDBOX=1 to run unconfined and diagnose.`, + ); + } + })(); trackToolProcess(proc); - const timer = setTimeout(() => proc.kill(), timeout); + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + proc.kill(); + }, timeout); const [stdout, stderr] = await Promise.all([ new Response(proc.stdout).text(), new Response(proc.stderr).text(), ]); const exitCode = await proc.exited; clearTimeout(timer); + + // Distinguish *why* the shell died. A timeout or a user kill also shows up + // as a signal; blaming the sandbox for those was wrong and unactionable. + if (timedOut) { + throw new Error(`Command timed out after ${timeout}ms and was terminated.`); + } + + if (proc.signalCode) { + const sandboxCrash = + plan.confined && (proc.signalCode === "SIGABRT" || proc.signalCode === "SIGSEGV"); + + throw new Error( + sandboxCrash + ? `Shell terminated by ${proc.signalCode} — the ${plan.backend} sandbox failed to apply. ` + + `Set MOCODE_DISABLE_SANDBOX=1 to run unconfined and diagnose.` + : `Command terminated by signal ${proc.signalCode}.`, + ); + } return { stdout: truncate(stdout, MAX_OUTPUT), stderr: truncate(stderr, MAX_OUTPUT), @@ -299,6 +389,40 @@ export async function executeLocalTool(toolName: string, input: unknown, mode: M }; } default: - throw new Error(`Unknown tool: ${toolName}`); + throw new ToolPermissionError("UNKNOWN_TOOL", `Unknown tool: ${toolName}`); + } +}; + +/** + * Authorised tool entry point. + * + * Every call — allowed, refused, or failed — leaves exactly one audit line. That is + * what makes the executor the authority rather than the model: a refusal is a + * recorded decision, not an absence of attempts. + * + * The thrown error keeps a human-readable message for the model while carrying a + * `PermissionCode` for the log. + */ +export async function executeLocalTool( + toolName: string, + input: unknown, + mode: ModeType, +): Promise { + const detail = summarizeInput(input); + // Only shells have a containment question; record the answer alongside the call + // so an audit can distinguish "ran in a sandbox" from "ran unconfined". + const containment = toolName === "bash" ? { shellConfined: isShellConfined() } : {}; + + try { + const output = await runTool(toolName, input, mode); + recordAudit({ tool: toolName, mode, outcome: "ok", detail, ...containment }); + return output; + } catch (error) { + recordAudit( + error instanceof ToolPermissionError + ? { tool: toolName, mode, outcome: "denied", code: error.code, detail, ...containment } + : { tool: toolName, mode, outcome: "error", detail, ...containment }, + ); + throw error; } -}; \ No newline at end of file +} \ No newline at end of file diff --git a/packages/cli/src/screens/session.tsx b/packages/cli/src/screens/session.tsx index ac95321..d453ecc 100644 --- a/packages/cli/src/screens/session.tsx +++ b/packages/cli/src/screens/session.tsx @@ -90,6 +90,10 @@ function SessionChat({ turnInterrupted, subagentRunning, subagentType, + agentProgress, + agentDowngrade, + agentPhase, + agentElapsedMs, submit, abort, interrupt, @@ -234,6 +238,10 @@ function SessionChat({ interruptible={isLoading} subagentRunning={subagentRunning} subagentType={subagentType} + agentProgress={agentProgress} + agentDowngrade={agentDowngrade} + agentPhase={agentPhase} + agentElapsedMs={agentElapsedMs} inputDisabled={isLoading} composerRestoreText={composerRestoreText} composerRestoreToken={composerRestoreToken} diff --git a/packages/cli/src/testing/render.tsx b/packages/cli/src/testing/render.tsx new file mode 100644 index 0000000..d5ad896 --- /dev/null +++ b/packages/cli/src/testing/render.tsx @@ -0,0 +1,73 @@ +/** + * Headless render harness for the TUI. + * + * The CLI had no component-level test infrastructure: every existing test covered + * pure logic, so rendered output (status line, dialogs, transcript rows, width + * behaviour) could only be checked by hand in a real terminal. `@opentui/react` + * ships a test renderer whose frame we can assert on as plain text, which is what + * this module wraps. + * + * What it gives us: + * - `renderText(node)` → the captured character frame, trailing spaces trimmed + * - provider wrapping (theme + prompt config) so components render in isolation + * - `renderLines(node)` → the frame as lines, for row-count / width assertions + * + * It does NOT cover: keyboard input, scrolling, or anything that needs a real + * TTY. Those still need a manual run (`bun run dev`). + */ +import { act } from "react"; +import type { ReactNode } from "react"; +import { testRender } from "@opentui/react/test-utils"; +import { ThemeProvider } from "../providers/theme"; +import { PromptConfigProvider } from "../providers/prompt-config"; + +export type RenderOptions = { + /** Terminal columns. Default 100 — wide enough that nothing wraps by accident. */ + width?: number; + /** Terminal rows. Default 6. */ + height?: number; +}; + +/** + * Render `node` in a headless terminal and return the captured frame. + * + * Lines are right-trimmed and trailing blank rows dropped, so assertions read + * like `expect(frame).toContain("turn 3/24")` instead of comparing padding. + */ +export async function renderText( + node: ReactNode, + { width = 100, height = 6 }: RenderOptions = {}, +): Promise { + const setup = await testRender( + + {node} + , + { width, height }, + ); + + try { + await act(async () => { + await setup.renderOnce(); + }); + return normalizeFrame(setup.captureCharFrame()); + } finally { + await act(async () => { + setup.renderer.destroy(); + }); + } +} + +/** {@link renderText} split into rows — use for "must stay on one row" assertions. */ +export async function renderLines(node: ReactNode, options: RenderOptions = {}): Promise { + const frame = await renderText(node, options); + return frame.length === 0 ? [] : frame.split("\n"); +} + +/** Right-trim every line and drop trailing blank rows. */ +export function normalizeFrame(frame: string): string { + return frame + .split("\n") + .map((line) => line.replace(/\s+$/, "")) + .join("\n") + .replace(/\n+$/, ""); +} From 57864357067d1030d8b2a4f847b6794247fc99b6 Mon Sep 17 00:00:00 2001 From: moyun Date: Thu, 12 Nov 2026 09:35:48 +0800 Subject: [PATCH 08/11] docs: record the permissions model, failures and eval methodology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `agent-permissions.md`: the five layers, the verified seatbelt transcript (`head: Operation not permitted`), and an explicit statement that layers 1-3 cannot contain a shell - `troubleshooting.md`: nine incidents with symptom, root cause, evidence and lesson. The useful ones are the wrong turns — the failover that never fired because errors hide inside a 200 stream, and the sandbox that died with SIGABRT and no stderr - `evals.md`: what the meta-test protects and why 20/20 means the suite is too easy rather than the agent is perfect - DESIGN.md: the status row gains a live-phase segment, and the width rule is stated against usable width rather than terminal width --- DESIGN.md | 11 +- docs/agent-permissions.md | 210 +++++++++++++++++++++++++ docs/evals.md | 112 +++++++++++++ docs/troubleshooting.md | 319 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 650 insertions(+), 2 deletions(-) create mode 100644 docs/evals.md create mode 100644 docs/troubleshooting.md diff --git a/DESIGN.md b/DESIGN.md index 3563904..1db679e 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -103,7 +103,7 @@ components: reasoning-block: { ref: "BotMessage.reasoning", role: "quiet-bar block, 'Thinking:' label, dim body" } tool-block: { ref: "BotMessage.tool", role: "quiet-bar block, info-colored tool label, args, pending/error suffix" } composer: { ref: InputBar, role: "Multiline textarea inside the mode-tinted accent bar; hosts overlays" } - status-bar: { ref: StatusBar, role: "Single dim row: mode › model · submit · newline hints" } + status-bar: { ref: StatusBar, role: "Single dim row: mode › model · live phase · fallback notice · loop budget · submit · newline hints" } session-footer: { ref: "SessionShell.footer", role: "Spinner + interrupt hint (left) / tab-agent hint (right)" } dialog-shell: { ref: "DialogProvider.Dialog", role: "Centered modal overlay card; dim backdrop; title + esc-affordance" } search-list: { ref: DialogSearchList, role: "Filter input + scrolling single-select list with selection highlight" } @@ -171,7 +171,14 @@ Read **bottom-up** — the way the user experiences it. This mirrors Claude Code multiline `textarea`. Slash-command and @-mention palettes float **above** it (`position:absolute; bottom:100%`), never pushing the transcript. - **Status line** (`{components.status-bar}`): exactly one dim row, Codex-style — - `mode {glyph.sep-chevron} model {glyph.sep-dot} submit hint {glyph.sep-dot} newline hint`. + `mode {glyph.sep-chevron} model {glyph.sep-dot} live phase {glyph.sep-dot} loop budget {glyph.sep-dot} submit hint {glyph.sep-dot} newline hint`. + The live-phase segment (`等待首字` / `思考中` / `执行中` / `生成中`, each with an + elapsed timer) answers "what is it doing right now" and is derived from the + streaming message, so it can never disagree with the transcript. It disappears + when the agent is idle. + The loop-budget segment (`turn 3/24`, plus `4 tools` when the current turn has + run tools) is `DIM` meta and renders only once a run has started — it answers + "how much of the agent budget has this message spent" without adding a second row. - **Footer** (`{components.session-footer}`): spinner + `esc to interrupt` on the left while streaming; `tab agent` hint pinned right. diff --git a/docs/agent-permissions.md b/docs/agent-permissions.md index 63cbb7c..1ca95c8 100644 --- a/docs/agent-permissions.md +++ b/docs/agent-permissions.md @@ -24,4 +24,214 @@ In Build mode the model should: ## Plan mode Plan mode has no bash tool. Git inspection uses read-only `gitStatus` and `gitDiff` tools instead. + +## Enforcement layers + +Permission is not one check — it is four, applied in order. Each layer assumes the +previous one may have been bypassed, which is why Plan mode is enforced twice. + +| Layer | Where | What it stops | +|---|---|---| +| 1. Tool contracts | `packages/shared/src/schemas.ts` (`getToolContracts`) | Plan mode never *receives* write/bash tool definitions | +| 2. Executor mode gate | `packages/cli/src/lib/local-tools.ts` (`READ_ONLY_TOOLS`) | A write tool that leaked through is still rejected before touching disk | +| 3. Path containment | `packages/cli/src/lib/sandbox-path.ts` | `../../etc/passwd`, absolute paths, and **symlinks pointing outside the project** — **file tools only** | +| 4. Hard denies | `packages/cli/src/lib/permission-rules.ts` (`findHardDeny`) | Host-destroying commands — refused **even after the user clicks approve** | + +### Layer 0: the OS sandbox (the only thing that contains a shell) + +Layers 1–3 are path checks, and **a path check cannot contain a shell**. That was +not a theory: manual acceptance caught the agent running + +``` +Bash: head -3 secret.txt # secret.txt -> /etc/passwd +## +# User Database +``` + +which bypassed `readFile` entirely and printed `/etc/passwd`. The file tools were +fenced; the shell was not. + +So the outermost layer is an OS sandbox, applied at spawn: + +| host | shell process | +|---|---| +| seatbelt applies (macOS) | `sandbox-exec -p bash -c …`, `(deny default)` | +| bwrap applies (Linux) | `bwrap --ro-bind / / --bind … bash -c …` | +| neither | `bash -c …`, **unconfined** — every command needs approval | + +Verified on macOS with seatbelt active — the original exploit, replayed: + +``` +$ ln -s /etc/passwd secret.txt +$ head -3 secret.txt +head: secret.txt: Operation not permitted (exit 1) + +readFile("secret.txt") → Path is outside the project directory +``` + +Both independent guards refuse, and neither asks for approval first: the command is +not blocklisted, the shell is confined, so it simply runs and the **kernel** denies +the read. That distinction matters — a denial that arrives as an approval prompt is +a policy, a denial that arrives as `EPERM` is a boundary. + +**Without a working OS sandbox there is no containment, only gating.** The approval +policy says so explicitly: when `isShellConfined()` is false, every bash command +asks, because a blocklist in front of an unconfined shell is theatre. + +### Capability tracking: `available` vs `enforced` + +`sandbox-capability.ts` probes the host at first use (seatbelt on macOS, bwrap on +Linux) and reports two separate flags: + +| flag | meaning | +|---|---| +| `available` | the host *can* apply a sandbox — a trivial profile actually ran | +| `enforced` | the spawn site *does* apply it to shell commands | + +The spawn site now wraps the shell when both flags hold: + +| host | shell command | approval | +|---|---|---| +| seatbelt/bwrap applies | `sandbox-exec -p bash -c …` / `bwrap … bash -c …` | blocklist only | +| nothing applies | `bash -c …` | **every** command asks | + +When the shell is confined, `head -3 secret.txt` fails with a seatbelt denial — +the content never reaches the model. When it is not confined, the run still +happens but each command needs a human, because a blocklist alone is theatre. + +### Profile policy + +Seatbelt denies by default and allows only: + +- **read**: `/` in general — a narrow allowlist made dyld/cryptex hosts abort at + startup, so the rule is a broad allow followed by an explicit deny list +- **denied read**: `/etc/passwd`, `/etc/shadow`, `/etc/sudoers`, `/root`, + `/var/root`, plus `~/.ssh`, `~/.aws`, `~/.gnupg`, `~/.kube`, `~/.docker`, + `~/.config/gh`, `~/.config/gcloud`, `~/.npmrc`, `~/.yarnrc`, `~/.pypirc`, + `~/.netrc` +- **write**: project tree, `$TMPDIR`, `/dev/null`, `/dev/tty`, plus + `MOCODE_SANDBOX_EXTRA_WRITE` + +`$HOME` is **not** unreadable as a whole — only the listed credential trees are. +If you add a tool that stores tokens somewhere new, add that path to +`deniedReadPaths`; the deny list is the boundary, not the absence of an allow rule. + +bwrap cannot express "deny by default" without rebuilding the root, so it mounts a +read-only `/` with a writable project and masks `~/.ssh`, `~/.aws`, `~/.gnupg`, +`/etc/shadow` and friends onto `/dev/null`. + +### Verifying confinement + +```bash +# 1. What does this host report? +bun run packages/cli/scripts/acceptance.ts # prints the capability line + +# 2. Does the fence actually hold? +# Ask the agent to run: bash: head -1 /etc/passwd +# Confined → "Operation not permitted" (seatbelt), no content +# Unconfined → content returned, and the command required approval first +``` + +`MOCODE_DISABLE_SANDBOX=1` turns confinement off for tools that break under +seatbelt; the approval policy notices and goes back to asking for everything. +`MOCODE_ALLOW_UNCONFINED_SHELL=1` is the separate, explicit "I accept an +unconfined shell, stop asking" switch. + +### Layer 3: containment is checked twice + +The old check was lexical only, so a symlink inside the project defeated it: + +```bash +ln -s /etc ./docs # docs lives inside the project +readFile("docs/passwd") # lexically inside, actually /etc/passwd +``` + +`resolveInsideCwd` now also compares **real paths** of the deepest existing +ancestor, which works for files that do not exist yet (`writeFile` creating a new +directory tree). + +### Layer 4: approval is not a licence + +Approval exists for *destructive but legitimate* work (`rm -rf ./dist`). It does +not extend to `mkfs`, `dd of=/dev/…`, `rm -rf /`, or a fork bomb — the executor +refuses those before spawning, so a mistaken "approve" cannot format a disk. + +### Shell environment is an allowlist + +The spawned shell receives only the keys in `SANDBOX_ENV_KEYS` (`PATH`, `HOME`, +`LANG`, `SHELL`, `TMPDIR`, …) with `TERM=dumb`. Every secret the CLI was started +with — `DEEPSEEK_API_KEY`, `CLERK_SECRET_KEY`, `DATABASE_URL` — is absent from the +child process. Previously bash inherited all of `process.env`. + +### Refusal codes and audit trail + +A refusal the model merely *reasons about* is not a control — the model can be +argued out of it. The executor is the authority, and every decision it makes is +recorded. + +Refusals raise `ToolPermissionError` with a machine-readable code: + +| code | meaning | +|---|---| +| `PATH_ESCAPE` | resolved path left the project (lexically or via symlink) | +| `HARD_DENY` | host-destroying command, not approvable | +| `MODE_DENIED` | tool not available in the current mode | +| `UNKNOWN_TOOL` | tool name is not in the executor's switch | + +The message stays human-readable for the model; the code is what dashboards and +audits group by. Prose drifts, codes do not. + +Every call — allowed, refused, or failed — appends one JSONL line to +`~/.mocode/audit.jsonl` (override with `MOCODE_AUDIT_PATH`): + +```json +{"timestamp":"2026-11-11T13:40:02.114Z","tool":"readFile","mode":"BUILD","outcome":"denied","code":"PATH_ESCAPE","detail":"path=/etc/passwd"} +``` + +Three rules make the log safe to keep: + +1. **No file contents.** Only short single-line scalars are summarised; anything + longer than 120 chars or containing a newline is dropped, not truncated. +2. **Never breaks a call.** A full disk or read-only home returns `false` instead + of throwing — losing a log line must not fail a working tool. +3. **One line per decision**, append-only, `jq`-friendly. + +### Rotation + +An unbounded audit file is its own outage. The log rotates before each append: + +| | default | override | +|---|---|---| +| rotate at | 5 MB | `MOCODE_AUDIT_MAX_BYTES` | +| archives kept | 5 | `MOCODE_AUDIT_MAX_FILES` | + +Archives are numbered newest-first — `audit.jsonl.1` holds the most recent rotated +segment and `.N` the oldest, which is dropped on the next rotation. Disk cost is +therefore bounded: `maxBytes × (maxFiles + 1)`. + +Rotation is size-triggered rather than time-triggered on purpose: a CLI process may +live for minutes or for a day, and time-based cutoffs either lose the burst or +create dozens of empty files. + +**Known limit:** rotation is a rename done synchronously in-process. Two CLI +processes sharing one `~/.mocode` could race; the loser appends to a file that was +just rotated away. Acceptable for a single-user CLI, not for a shared agent host — +that needs an append-only sink (file → log shipper), not a local file. + +### Regression suite + +`packages/cli/src/lib/local-tools.sandbox.test.ts` runs the attack cases against the +real `executeLocalTool` entry point: + +- absolute path, parent traversal, write-outside, list-outside → all refused +- `dd of=/dev/…` and `mkfs` → refused at execution time +- a canary `DEEPSEEK_API_KEY` is invisible to the shell +- `PATH` still works, `TERM` is `dumb` +- Plan mode still rejects write/bash even if the contract leaked +- every refusal is logged with its code; a 500-char file body never reaches the log + +Manual check: ask the agent to read `/etc/passwd` — it must report a refusal, not +content. Then approve a `mkdir` (should run) and approve `mkfs.ext4 /dev/null` +(should be refused with "cannot be approved"). + \ No newline at end of file diff --git a/docs/evals.md b/docs/evals.md new file mode 100644 index 0000000..12684bd --- /dev/null +++ b/docs/evals.md @@ -0,0 +1,112 @@ +# Task evals + +The only measurement in this repo that scores **end-to-end task success** rather +than unit-level correctness. Everything else (340+ unit tests) proves the pieces +work; this proves the agent can do a job. + +## Run + +```bash +bun run packages/cli/evals/run.ts # all cases +bun run packages/cli/evals/run.ts --only fix-off-by-one +``` + +Needs `DEEPSEEK_API_KEY` in `.env`. Each case gets a throwaway temp directory; the +agent's cwd is switched into it so the sandbox still applies. Results land in +`evals/results/latest.json`. + +## What a case is + +```ts +{ + id, category, + prompt, // what the user asks + files, // starting state, written to a temp dir + check, // shell command that must exit 0 +} +``` + +The check is **behavioural** — it runs code and asserts on results, not on the +presence of a string. `rename-across-files` would otherwise pass on a no-op. + +## The meta-test is the important one + +`evals/cases.test.ts` asserts, for every case, that **the check fails in the initial +state**. A case that already passes measures nothing but a false success rate. + +This caught a real defect while the suite was being written: `fix-async-ordering` +initially passed because the microtasks settled before the assertion ran, so the +"broken" code produced the right answer. The fix was to snapshot the array before +the promises resolve. + +## Latest run + +20 cases: the ten single-file ones below, plus ten harder ones (multi-file +propagation, a regression on a neighbouring test, an O(n²)→budget rewrite, a test +the agent is forbidden to edit, immutability, error contract, state cycle, +multi-constraint, cascading order-of-operations bug). + +**20/20 (100%)**, average 6 turns · 8 tool calls · ~15.5k tokens. +Heaviest case: `dedupe-into-helper`, 12 turns / 49k tokens / 26 s. + +A second independent run on a different machine landed at the same 20/20 with +7 turns · 9 tools · ~18.4k tokens per case. The spread (15.5k → 18.4k) is normal +run-to-run variance for the same model — which is why the report stores per-case +numbers rather than a single average. + +The first ten (model `deepseek-flash`, max 15 turns per case): + +| case | ok | turns | tools | tokens | time | +|---|---|---|---|---|---| +| fix-off-by-one | ✅ | 4 | 3 | 6,269 | 6.1 s | +| fix-missing-await | ✅ | 7 | 9 | 18,346 | 15.8 s | +| implement-stub | ✅ | 5 | 6 | 9,910 | 6.3 s | +| add-validation | ✅ | 5 | 6 | 11,792 | 10.4 s | +| fix-json-crash | ✅ | 5 | 6 | 9,332 | 5.9 s | +| rename-across-files | ✅ | 6 | 10 | 12,599 | 7.4 s | +| dedupe-into-helper | ✅ | 12 | 16 | 49,128 | 26.4 s | +| add-unit-test | ✅ | 5 | 7 | 9,723 | 5.1 s | +| fix-async-ordering | ✅ | 11 | 12 | 33,271 | 18.9 s | +| add-cli-flag | ✅ | 5 | 5 | 9,152 | 5.8 s | + +## Being honest about 100% + +Adding ten harder cases did not move the score. That is the finding, not a +failure to report: **this class of micro-task does not discriminate** for a +current-generation coding model. A perfect score here is a floor, not a ceiling. + +What the twenty cases still do not reproduce: + +- **repo-scale context** — a file the agent must find among hundreds, not one it + is handed by name +- **vague reports** — "checkout is sometimes wrong" instead of a failing assert +- **hidden coupling** — the fix passes the visible test but breaks a caller the + agent never read +- **long horizons** — tasks that legitimately need 40+ turns +- **the ask-don't-guess path** — where the correct action is to stop and ask + +The next lever is **realistic fixtures and ambiguity**, not more micro-cases. Ten +more single-file tasks would keep the score at 100% and teach nothing. + +One thing the harder set did buy: `fix-async-ordering` and `dedupe-into-helper` +now take 11–12 turns, so the suite exercises multi-step recovery rather than +one-shot edits. + +## Manual acceptance + +1. `bun run packages/cli/evals/run.ts` — expect 10/10 and a JSON report. +2. Deliberately break something cheap: set `MAX_TURNS=1` in `run.ts`. + Expected: cases now fail with check exit ≠ 0, proving the score tracks the + agent rather than the fixture. +3. Remove `clamp.ts` from the `add-unit-test` fixture in `EVAL_CASES` (not from a + temporary run directory) and re-run `--only add-unit-test`. Expected: the agent + reports that the module is missing instead of inventing an import — that is the + tool-error path working. Note that the prompt says "add tests for clamp()", so + the agent may instead *recreate* `clamp.ts`; treat that as an acceptable + outcome and judge the tool-error path on the cases where it does not. + +## What it costs + +The full suite is ~1 minute and a few cents. Cheap enough to run before a demo and +after any change to the loop, tools, or context policy — which is the point: an +eval you don't run is a dead document. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..810cc5d --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,319 @@ +# Troubleshooting & incident log + +Real failures hit while running MoCode locally, with the evidence that identified +them and what changed as a result. Append new entries at the bottom; keep the +symptom text exactly as the user sees it so search finds it. + +## Reproducing the provider experiment + +```bash +# from the repo root so .env is loaded +bun run packages/server/scripts/provider-failover-bench.ts +``` + +Every failure in that bench is a **real upstream failure** — no mocks. On a +mainland-China network the blocked providers are a free, reproducible fault +injector, which is why the numbers below are trustworthy. + +--- + +## INC-001 — every message fails with `Forbidden` + +**Symptom.** After `bun run dev`, sending any message in the TUI immediately fails +with `Forbidden`. Nothing in the transcript looks like a local error. + +**Root cause.** Not MoCode. The default model was `gpt-oss-120b`, which is bound to +the **Cerebras** provider. Cerebras answers `403` with Cloudflare `error code 1009` +("banned your access based on your country"). MoCode's server passes upstream +errors through verbatim: + +```ts +// packages/server/src/index.ts +app.onError((err, c) => { + if (err instanceof HTTPException) { + return c.json({ error: err.message || "Request failed" }, err.status); + } +``` + +So the provider's `403`/`Forbidden` reaches the CLI unchanged. + +**Evidence** (status codes only; keys never printed): + +| provider | status | body | +|---|---|---| +| cerebras | 403 | `error code: 1009` | +| groq | 403 | `{"error":{"message":"Forbidden"}}` | +| anthropic | 403 | `{"type":"forbidden","message":"Request not allowed"}` | +| openai | — | connect timeout | +| google | — | connect timeout | +| openrouter | 401 | `User not found` | +| **deepseek** | **200** | reachable | + +`grep -rn "403\|forbidden" packages/*/src` finds no `403` in MoCode itself — that +grep is the first thing to run, because it separates "our bug" from "upstream". + +**Fix.** Default model switched to `deepseek-flash` (see INC-004). + +**Lesson.** An error message that is a verbatim upstream string is a pass-through, +not a local failure. Check whether the string exists in the codebase before +debugging the codebase. + +--- + +## INC-002 — DeepSeek model ids no longer exist + +**Symptom.** After wiring DeepSeek, requests fail even though the key is valid. + +**Root cause.** The catalog used the V3-era ids `deepseek-chat` and +`deepseek-reasoner`. The live `/models` endpoint returns only: + +``` +deepseek-flash DeepSeek-V4.1-Flash 1M context, tool calls + vision +deepseek-v4-pro DeepSeek-V4-Pro +``` + +**Fix.** Renamed in `packages/shared/src/models.ts`, pricing re-read from +`api-docs.deepseek.com/quick_start/pricing`, and both ids added to +`packages/server/scripts/test-providers.ts`. + +**Pricing caveat.** DeepSeek bills four rates (cache hit/miss × peak/off-peak). +`ModelPricing` only has two fields, so entries use **peak cache-miss input + peak +output** — deliberately over- rather than under-charging. Modelling the four-rate +case is still open work. + +**Lesson.** Model ids are not stable API surface. `test-providers.ts` exists to +catch exactly this; it only helps if it covers the provider you just added. + +--- + +## INC-003 — OpenRouter key is dead + +**Symptom.** `openai/gpt-oss-120b:free` fails with `401 User not found`. + +**Root cause.** Key revoked or expired. (Not a code issue.) + +**Fix.** Recreate the key in the OpenRouter dashboard. Until then that catalog +entry is unusable and `/models` lists a broken option. + +**Lesson.** A catalog entry that can never succeed is worse than a missing one — +users blame the tool. Provider keys should be validated at startup, not first use. + +--- + +## INC-004 — default model pinned to a single provider + +**Symptom.** The same `403` for every user regardless of which other models exist. + +**Root cause.** `DEFAULT_CHAT_MODEL_ID` was a compile-time constant bound to one +provider, and `resolveChatModel` has no fallback: one unreachable provider means +100% of turns fail. + +**Fix.** Default moved to `deepseek-flash`; README (both languages) documents why. + +**Measured blast radius** — 15 catalog models, one tool-calling request each: + +| policy | reachable | +|---|---| +| no fallback (single provider, hard-bound) | **2/15 (13%)** | +| fallback to `deepseek-flash` → `deepseek-v4-pro` | **15/15 (100%)** | + +Failure latency is bimodal, which drives the failover design: + +- **8 fast failures** (`403`/`401`, 0.2–0.9 s) — cheap to fall through +- **5 slow failures** (connect hang, full 12 s timeout) — expensive + +**Lesson.** Changing the default is a patch, not a fix. A per-attempt timeout is +mandatory before a fallback chain is worth having: without it one hung provider +delays failover by the full timeout budget. And a silent downgrade +(`claude-opus` → `deepseek-flash`) changes capability, so the transcript must +record the downgrade rather than pretend the request succeeded as asked. + +--- + +## INC-005 — `.env` breaks under shell sourcing + +**Symptom.** `source .env` prints `JWT_SECRET: command not found`. + +**Root cause.** `JWT_SECRET =value` — whitespace before `=`. Bun's loader tolerates +it; POSIX shells parse the line as a command. + +**Fix.** Write `KEY=value` with no spaces. + +**Lesson.** An env file that Bun accepts but a shell rejects hides failures in +every script that reads it the other way. Pick one format and enforce it. + +--- + +## INC-006 — failover was measured but not implemented + +**Symptom.** After INC-004 the default model changed, but every non-DeepSeek model +still failed permanently. The bench could *describe* the fix without the product +having it. + +**Root cause.** Two separate gaps: + +1. `chatFetch` rethrew a bare `Error(message)`, **dropping the HTTP status**, so no + caller could tell a retryable 403 from a permanent 400. +2. There was no chain at all — one model, one provider, one attempt. + +**Fix.** + +- `packages/cli/src/lib/provider-chain.ts` — chain building, failure classification, + retry orchestration. Pure, no I/O. +- `packages/cli/src/lib/chat-fetch.ts` — SaaS transport retries the next model by + rewriting only the `model` field of the request body. +- `packages/cli/src/lib/local-chat-transport.ts` — BYOK retries **before the first + UI chunk arrives**; once a chunk is emitted the turn is visible and is never replayed. +- Downgrade is explicit: `{color.error}` segment in the status line plus a toast. + +**Opt-in by design.** Failover only runs when `MOCODE_ALLOW_MODEL_FAILOVER=1`: retrying on +another provider ships the user's code to a vendor they did not choose, and that is a +decision, not a default. The chain is also `deepseek-flash` only — `deepseek-v4-pro` +costs roughly 4× and is never auto-escalated. + +**Measured end-to-end** (real requests, wall clock includes the failed attempt): + +| primary asked | answered by | total | +|---|---|---| +| `claude-sonnet-4-6` | `deepseek-flash` (downgraded) | 1.3 s | +| `gpt-oss-120b` | `deepseek-flash` (downgraded) | 1.1 s | +| `deepseek-flash` | `deepseek-flash` (no downgrade) | 1.0 s | + +Failover overhead on a fast-failing provider is ~0.1–0.3 s, because a 403 returns +before any token is generated. + +**What is deliberately NOT retried:** + +| case | retried | why | +|---|---|---| +| 401 / 403 / 429 / 5xx | yes | another provider may serve the request | +| 400 | no | the request body is wrong — retrying hides our own bug | +| user abort (Esc) | no | a cancellation must not be "helpfully" undone | +| timeout | yes | a hung provider should not end the turn | + +**Lesson.** Retrying is only safe if you can tell *when the work started*. Here the +boundary is the first UI chunk: before it, nothing has been rendered or billed; after +it, a retry would duplicate both. + +--- + +## INC-007 — failover never fired: the error was hiding inside a 200 + +**Symptom.** With everything from INC-006 in place, selecting a blocked model still +produced a bare `Forbidden` — no retry, no downgrade toast, no `fallback` segment. + +**Root cause.** The failover trigger was `!response.ok`, and the server answers +**200** for every chat request: it opens an SSE stream, then the upstream provider +failure arrives as an `error` event *inside that stream*. A status-code check +therefore saw success and returned the broken stream untouched. + +Unit tests passed because they asserted on HTTP statuses; the real transport never +produces a non-2xx status for a provider failure. The test was measuring the wrong +thing. + +**Fix.** `chat-fetch.ts` now peeks at the first events of a 200 stream before +returning it: + +- an `error` event within the first `PEEK_MAX_CHUNKS` events → treat as a failed + attempt and fail over +- a healthy event (`text-delta`, `tool-input-start`, `finish`, …) → stop peeking + and replay every buffered byte into a reconstructed `Response` + +Nothing is dropped: the buffered chunks are re-emitted first, and the remaining +reader is pumped into a new `ReadableStream` with the original headers (minus +`content-length` / `content-encoding`). + +**Lesson.** "The request succeeded" and "the provider worked" are different +questions once you stream. Any retry policy keyed on HTTP status is wrong for an +SSE endpoint whose errors travel in-band. + +--- + +## INC-008 — fix landed, then the agent timed out mid-reply + +**Symptom.** After INC-007 the failover toast and the `fallback → deepseek-flash` +segment appeared correctly, the loop ran (status line showed `turn 12/24`), and then +the reply was replaced with `The operation timed out.` + +**Root cause.** The per-attempt deadline was passed straight to `fetch` as an abort +signal, so it kept ticking for the **whole stream**, not just the connection. Any +turn that streamed longer than `attemptTimeoutMs` (20 s) was killed mid-sentence — +including healthy turns that had already emitted dozens of tokens. + +The failover logic itself was fine; the deadline was scoped wrong. + +**Fix.** The deadline now covers *connect + first useful event* only: + +- a cancellable `AbortController` + `setTimeout` replaces `AbortSignal.timeout` +- once a healthy event is seen, `clearTimeout` releases the stream to run as long + as it needs +- if the deadline fires before that, it is re-thrown as a `TimeoutError` so + `classifyFailover` still treats it as retryable +- a genuine user abort (`init.signal`) is checked first and never retried + +**Regression test.** A stream that emits its first event immediately and its last +chunk 150 ms later, with `attemptTimeoutMs: 50`, must still deliver the late chunk. + +**Lesson.** A timeout has to name *what it bounds*. "Request timed out" is +meaningless for a streaming API; the only useful deadlines are "time to first +byte" and "time between tokens". + +--- + +## INC-009 — every sandboxed shell died with SIGABRT and no stderr + +**Symptom.** On the first machine where seatbelt actually worked, *every* bash call +returned empty output with `exitCode: null`, `signalCode: SIGABRT`, and an empty +stderr. The capability probe itself passed, so the sandbox looked available. + +From the agent's side this was the worst possible failure: a tool that returns +`{stdout: "", stderr: ""}` and no error teaches the model to invent output. + +**Diagnosis.** `SIGABRT` with no stderr is not a policy denial — a denial exits +non-zero and says `Operation not permitted`. The bisect isolated the variables: + +| # | profile | env | result | +|---|---|---|---| +| 1 | trivial | inherited | ok | +| 2 | trivial | allowlisted | ok | +| 3 | full | inherited | ok | +| 4 | full | allowlisted | ok | + +Both inputs were innocent, which pointed at the *spawn*, not the profile. + +**Root cause.** `spawnPlan` returned the bare name `"sandbox-exec"`. +`Bun.spawn` resolves a bare name against the env it is handed — and we hand it a +deliberately minimal allowlist. Switching to `/usr/bin/sandbox-exec` fixed every +variant. The exact reason a resolution miss surfaces as `SIGABRT` rather than +`ENOENT` is **not explained**; the empirical evidence is that the path was the only +changed variable, and the failure disappeared. + +**Fix.** Absolute paths for sandbox binaries, plus two guards: + +- a spawn failure now throws `Shell could not start ( sandbox): …` instead + of returning an empty result +- a shell killed by a signal now throws `Shell terminated by — the + sandbox failed to apply` instead of returning empty output + +**Verified on macOS** (seatbelt active): + +``` +head -1 /etc/passwd → exit 1, "head: /etc/passwd: Operation not permitted" +echo $MOCODE_DOCTOR_CANARY → "absent" +``` + +**Lesson.** "No output" is not a diagnosis. Two independent failure modes produced +identical empty results — a broken sandbox and a working sandbox with a missing +variable — and only exit code, signal, and stderr told them apart. Any tool that can +return empty should be able to say *why* it was empty. + +--- + +## Open follow-ups + +1. **Per-attempt timeout** is wired in the SaaS path only; BYOK still relies on the + provider SDK's own deadline. +2. **Four-rate DeepSeek pricing** (cache × peak) instead of the two-field model. +3. **Downgrade is per-turn state**, not persisted into session history — reopening a + session loses the record that a fallback answered. + From 128943c368039a154f26d47b4d847a8861a2c65f Mon Sep 17 00:00:00 2001 From: moyun Date: Thu, 12 Nov 2026 11:30:38 +0800 Subject: [PATCH 09/11] tune(agent): raise the turn budget to 50, backed by first-party sources The previous 24 came from a legacy subagent cap, and the folklore it rested on ("LangGraph defaults to 25") is obsolete: 1.2.4 ships DEFAULT_RECURSION_LIMIT = 10007 precisely because 25 interrupted legitimate long tasks. Published defaults, read from source: - OpenAI Agents SDK: DEFAULT_MAX_TURNS = 10 - Claude Agent SDK: max_turns: int | None = None (no limit) - LangGraph 1.2.4: 10007, env-overridable A coding agent sits near the long end, but unbounded turns a bug into an unbounded bill, so the budget stays bounded: 50 continuations (51 responses), ~4x the max 13 turns seen across the 20 eval tasks. - add MOCODE_MAX_TURNS / MOCODE_MAX_TOOL_CALLS / MOCODE_MAX_REPEATS, dropping undefined values so a missing env var cannot blank a default - docs/decisions.md records every "why this number" with its first-party source, and marks the ones that are judgment calls - subagent stays at 25, with the real reason (synchronous, parent blocks) instead of "keep the numbers equal" --- DESIGN.md | 2 +- docs/decisions.md | 103 ++++++++++++++++++ .../cli/src/components/status-bar.test.tsx | 24 ++-- packages/cli/src/lib/agent-loop.test.ts | 12 +- packages/cli/src/lib/agent-loop.ts | 41 ++++++- 5 files changed, 158 insertions(+), 24 deletions(-) create mode 100644 docs/decisions.md diff --git a/DESIGN.md b/DESIGN.md index 1db679e..bbc4e74 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -176,7 +176,7 @@ Read **bottom-up** — the way the user experiences it. This mirrors Claude Code elapsed timer) answers "what is it doing right now" and is derived from the streaming message, so it can never disagree with the transcript. It disappears when the agent is idle. - The loop-budget segment (`turn 3/24`, plus `4 tools` when the current turn has + The loop-budget segment (`turn 3/51`, plus `4 tools` when the current turn has run tools) is `DIM` meta and renders only once a run has started — it answers "how much of the agent budget has this message spent" without adding a second row. - **Footer** (`{components.session-footer}`): spinner + `esc to interrupt` on the diff --git a/docs/decisions.md b/docs/decisions.md new file mode 100644 index 0000000..a1048a1 --- /dev/null +++ b/docs/decisions.md @@ -0,0 +1,103 @@ +# Decisions — the numbers that look arbitrary + +Every constant in this repo that a reader could reasonably call "made up" gets an +entry: the value, where it came from, and the condition that should make you change +it. Anything without a source is marked as a judgment call — pretending otherwise +is how a codebase accumulates cargo cult. + +Sources below were read from first-party code, not blog posts. + +## Turn budget + +| Product | Limit | First-party source | +|---|---|---| +| Claude Agent SDK | **unbounded** — `max_turns: int \| None = None` | `anthropics/claude-agent-sdk-python`, `src/claude_agent_sdk/types.py` | +| LangGraph 1.2.4 | **10007** — `DEFAULT_RECURSION_LIMIT`, env `LANGGRAPH_DEFAULT_RECURSION_LIMIT` | `langgraph/_internal/_config.py:31` | +| OpenAI Agents SDK | **10** — `DEFAULT_MAX_TURNS` | `openai/openai-agents-python`, `src/agents/run_config.py:45` | +| **MoCode, main loop** | **50** continuations = 51 responses | `agent-loop.ts` | +| **MoCode, subagent** | **25** steps | `subagent/runner.ts` | + +**Why 50.** The previous value was 24, copied from the subagent cap. That cap is +itself legacy: the widely repeated "LangGraph defaults to 25" is obsolete — +1.2.4 ships 10007, raised because 25 interrupted legitimate long tasks. The +published range today is 10 (OpenAI) to unbounded (Anthropic). A coding agent sits +near the long end, but "unbounded" turns a bug into an unbounded bill, so this stays +a budget. Measured on 20 eval tasks: max 13 turns, so 50 is ~4x headroom. + +Override: `MOCODE_MAX_TURNS`. + +**Why the subagent stays 25.** It runs synchronously — the parent blocks until it +returns a summary, so a generous budget spends the parent's wall clock. The tight +cap is the feature. (This is a better reason than "keep the two numbers equal", +which is what the previous comment said.) + +## Tool calls per turn: 16 + +No published equivalent. OpenAI's `ToolExecutionConfig.max_function_tool_concurrency` +bounds *concurrency*, not the number of calls in one turn. + +This is a **circuit breaker, not a budget**: measured turns use a mean of 1.26 tools +and a max of 2.33. 16 is roughly 7x the observed max — high enough that batch work +("change the imports in these 12 files") never trips it, low enough that a +pathological 50-way fan-out does. + +Override: `MOCODE_MAX_TOOL_CALLS`. + +## Consecutive identical calls: 3 + +The 4th consecutive call with the same tool and same input stops the run with +`no-progress`. + +**"Consecutive" is the whole design**: `read -> edit -> read` resets the counter, +so normal edit loops are not false positives. 3 is a judgment call — a legitimate +retry rarely repeats an identical call more than twice. + +Override: `MOCODE_MAX_REPEATS`. + +## Context compaction: 60k tokens, keep 10 messages, min 16 + +Anthropic and OpenAI both publish guidance to compact as a conversation approaches +the model's window. **This is not that.** `deepseek-flash` accepts 1M tokens, so the +60k budget is a **cost** ceiling, not a window limit. + +Measured on a 50-turn synthetic session (one tool call per turn, 10k-char output — +the `readFile` truncation limit): 139,010 -> 14,011 tokens, flat after turn ~24. + +`minMessages: 16` exists so a short conversation is never compacted: early turns are +mostly intent, and folding them destroys the task. + +## Provider retries: 3 attempts, 500ms base, 8s cap + +- Retried on the same provider: 408 / 429 / 5xx / timeouts / connection errors. +- **Not** retried on the same provider: 401 / 403 / 404 — they answer identically in + 500ms, so a retry only delays the fallback. +- Delay is exponential (500ms, 1s, 2s, …) plus up to 500ms of jitter, so parallel + retries do not synchronise on the provider. + +Override: `MOCODE_PROVIDER_RETRIES`. + +## Timeouts: connect 20s, stall 120s + +Two different questions, deliberately separate: + +- **connect** bounds DNS + TCP + TLS + response headers, and is cleared the moment + headers arrive. It never bounds generation. +- **stall** bounds *silence*: it resets on every chunk. A five-minute stream that + keeps emitting bytes is healthy and must not be killed. + +Getting this wrong is what produced the "The operation timed out" mid-reply bug. + +Overrides: `MOCODE_CONNECT_TIMEOUT_MS`, `MOCODE_STALL_TIMEOUT_MS`. + +## Audit rotation: 5MB × 5 archives + +Bounded disk: at most ~30MB. Size-triggered rather than time-triggered because a CLI +process may live for minutes or a day, and clock-based cutoffs either lose the burst +or create dozens of empty files. + +## Not verified + +CrewAI's `max_iter` default could not be located in first-party source (its repo moved +to a `lib/crewai-core` layout and the module path guesses failed), so it is **not** +used as evidence for any value here. If you need it, check +`lib/crewai-core/src/crewai/` directly. diff --git a/packages/cli/src/components/status-bar.test.tsx b/packages/cli/src/components/status-bar.test.tsx index c6c8578..b02aa94 100644 --- a/packages/cli/src/components/status-bar.test.tsx +++ b/packages/cli/src/components/status-bar.test.tsx @@ -16,24 +16,24 @@ describe("StatusBar", () => { test("shows the turn budget once a run has continued", async () => { const frame = await renderText( - , + , ); - expect(frame).toContain("turn 4/25"); + expect(frame).toContain("turn 4/51"); expect(frame).not.toContain("tools"); }); test("appends the tool count for the current turn", async () => { const frame = await renderText( - , + , ); - expect(frame).toContain("turn 4/25 · 4 tools"); + expect(frame).toContain("turn 4/51 · 4 tools"); }); test("hides the budget before the first continued turn", async () => { const frame = await renderText( - , + , ); expect(frame).not.toContain("turn"); @@ -41,12 +41,12 @@ describe("StatusBar", () => { test("keeps the whole status line on a single row (DESIGN.md: status-row = 1)", async () => { const lines = await renderLines( - , + , { width: 100, height: 4 }, ); expect(lines).toHaveLength(1); - expect(lines[0]).toContain("turn 13/25 · 9 tools"); + expect(lines[0]).toContain("turn 13/51 · 9 tools"); }); test("subagent footer replaces the budget while a Task runs", async () => { @@ -54,7 +54,7 @@ describe("StatusBar", () => { , ); @@ -81,7 +81,7 @@ describe("StatusBar", () => { const lines = await renderLines( , // Wide enough for mode/model/downgrade/budget; tight enough to drop newline/note. { width: 72, height: 4 }, @@ -89,7 +89,7 @@ describe("StatusBar", () => { expect(lines).toHaveLength(1); expect(lines[0]).toContain("fallback → deepseek-flash"); - expect(lines[0]).toContain("turn 13/25"); + expect(lines[0]).toContain("turn 13/51"); expect(lines[0]).not.toContain("换行"); }); }); @@ -145,13 +145,13 @@ describe("live agent phase", () => { , { width: 110, height: 4 }, ); expect(lines).toHaveLength(1); expect(lines[0]).toContain("执行中 2.4s"); - expect(lines[0]).toContain("turn 3/25"); + expect(lines[0]).toContain("turn 3/51"); }); }); diff --git a/packages/cli/src/lib/agent-loop.test.ts b/packages/cli/src/lib/agent-loop.test.ts index 1747f82..1aec17b 100644 --- a/packages/cli/src/lib/agent-loop.test.ts +++ b/packages/cli/src/lib/agent-loop.test.ts @@ -166,22 +166,22 @@ describe("AgentLoop lifecycle", () => { describe("formatAgentProgress", () => { test("renders nothing before a run starts", () => { expect(formatAgentProgress(null)).toBeNull(); - expect(formatAgentProgress({ turn: 0, maxTurns: 24, toolCallsThisTurn: 0 })).toBeNull(); + expect(formatAgentProgress({ turn: 0, maxTurns: 50, toolCallsThisTurn: 0 })).toBeNull(); }); test("renders the turn budget", () => { - expect(formatAgentProgress({ turn: 3, maxTurns: 24, toolCallsThisTurn: 0 })).toBe("turn 4/25"); + expect(formatAgentProgress({ turn: 3, maxTurns: 50, toolCallsThisTurn: 0 })).toBe("turn 4/51"); }); test("appends the tool count once the turn has run tools", () => { - expect(formatAgentProgress({ turn: 3, maxTurns: 24, toolCallsThisTurn: 4 })).toBe( - "turn 4/25 · 4 tools", + expect(formatAgentProgress({ turn: 3, maxTurns: 50, toolCallsThisTurn: 4 })).toBe( + "turn 4/51 · 4 tools", ); }); test("uses the singular label for a single tool call", () => { - expect(formatAgentProgress({ turn: 1, maxTurns: 24, toolCallsThisTurn: 1 })).toBe( - "turn 2/25 · 1 tool", + expect(formatAgentProgress({ turn: 1, maxTurns: 50, toolCallsThisTurn: 1 })).toBe( + "turn 2/51 · 1 tool", ); }); }); diff --git a/packages/cli/src/lib/agent-loop.ts b/packages/cli/src/lib/agent-loop.ts index 47032d3..234f782 100644 --- a/packages/cli/src/lib/agent-loop.ts +++ b/packages/cli/src/lib/agent-loop.ts @@ -13,7 +13,13 @@ * One *turn* is one model response plus the tool calls it requested. The first * response after a user message is turn 1. {@link AgentLoop.beginTurn} runs when * the loop decides to continue, so `maxTurns` bounds *continued* turns: with - * `maxTurns: 24` a single user message can produce at most 25 model responses. + * `maxTurns: 50` a single user message can produce at most 51 model responses. + * + * Why 50 (see `docs/decisions.md`): published defaults span a wide range — + * OpenAI Agents SDK caps at 10, Claude Agent SDK leaves it unbounded, and + * LangGraph 1.x defaults its recursion limit to 10007 after finding 25 too tight + * for real tasks. A coding agent needs headroom, but "unbounded" turns a bug into + * an unbounded bill, so this stays a budget. * * Pure module: no React, no network, no filesystem — unit-testable in isolation. */ @@ -37,11 +43,36 @@ export type AgentLoopLimits = { }; export const DEFAULT_AGENT_LIMITS: AgentLoopLimits = { - maxTurns: 24, + maxTurns: 50, + // A circuit breaker, not a budget: measured turns use 2-3 tools (max 2.33). maxToolCallsPerTurn: 16, maxConsecutiveRepeats: 3, }; +/** + * Operator overrides for the budgets. + * + * Undefined values are dropped rather than spread, so a missing env var cannot + * blank out a default with `undefined`. + */ +export function agentLimitsFromEnv(): Partial { + const read = (name: string): number | undefined => { + const parsed = Number(process.env[name]); + return Number.isFinite(parsed) && parsed > 0 ? Math.floor(parsed) : undefined; + }; + + const limits: Partial = {}; + const maxTurns = read("MOCODE_MAX_TURNS"); + const maxToolCallsPerTurn = read("MOCODE_MAX_TOOL_CALLS"); + const maxConsecutiveRepeats = read("MOCODE_MAX_REPEATS"); + + if (maxTurns !== undefined) limits.maxTurns = maxTurns; + if (maxToolCallsPerTurn !== undefined) limits.maxToolCallsPerTurn = maxToolCallsPerTurn; + if (maxConsecutiveRepeats !== undefined) limits.maxConsecutiveRepeats = maxConsecutiveRepeats; + + return limits; +} + export type AgentLoopDecision = | { action: "continue" } | { action: "stop"; reason: AgentStopReason; detail: string }; @@ -62,7 +93,7 @@ export type AgentProgress = Pick< /** * Status-line label for the loop budget, or `null` when no run has started. - * One dim segment: `turn 3/24`, plus `· 4 tools` when the turn has run tools. + * One dim segment: `turn 3/51`, plus `· 4 tools` when the turn has run tools. */ export function formatAgentProgress(progress: AgentProgress | null): string | null { if (!progress || progress.turn <= 0) return null; @@ -111,13 +142,13 @@ export class AgentLoop { private consecutiveRepeats = 0; constructor(limits: Partial = {}) { - this.limits = { ...DEFAULT_AGENT_LIMITS, ...limits }; + this.limits = { ...DEFAULT_AGENT_LIMITS, ...agentLimitsFromEnv(), ...limits }; } /** * Start (or continue) the loop for a user message. A different id starts a * fresh budget; the same id keeps the current one, so regenerating a reply does - * not silently hand the model an extra 24 turns. + * not silently hand the model an extra 50 turns. */ beginUserMessage(messageId: string): void { if (this.userMessageId === messageId) return; From 26cae897ffd7220129663b9e86ab69c31f22bab3 Mon Sep 17 00:00:00 2001 From: moyun Date: Thu, 12 Nov 2026 11:34:46 +0800 Subject: [PATCH 10/11] feat(cli): user-writable permission rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The policy was a hardcoded blocklist plus an in-session allow set: a user could approve something for an hour, never for a project, and could not deny anything the blocklist did not already know about. Rules now live in JSON, so the policy is reviewable and diffable: /.mocode/permissions.json committed with the repo ~/.mocode/permissions.json applies to every project { "tool": "bash", "pattern": "^git push", "behavior": "deny" } - **deny > allow > ask**, and a deny in either file beats an allow in the other: a safety rule must not be defeatable by a broader allow - enforced twice on purpose: the approval gate (deny skips the dialog, allow skips the built-in prompt, ask forces one) and the executor (`RULE_DENIED`, audited) so callers that never touch the UI — evals, subagents — are still bound - a malformed rule is dropped and reported, never coerced; turning `"behavior": "denied"` into an allow is the worst possible failure - re-read per call, so editing the file applies to the next tool call Not implemented: writing rules back from the TUI. The dialogs still record session-only allows; persisting them needs a new dialog action, and is documented as the remaining gap against a configurable policy. --- docs/agent-permissions.md | 44 ++++ packages/cli/src/hooks/use-chat.ts | 38 ++- .../cli/src/lib/local-tools.rules.test.ts | 72 ++++++ packages/cli/src/lib/local-tools.ts | 10 + packages/cli/src/lib/permission-rules.ts | 3 +- packages/cli/src/lib/permissions.test.ts | 156 +++++++++++++ packages/cli/src/lib/permissions.ts | 218 ++++++++++++++++++ 7 files changed, 533 insertions(+), 8 deletions(-) create mode 100644 packages/cli/src/lib/local-tools.rules.test.ts create mode 100644 packages/cli/src/lib/permissions.test.ts create mode 100644 packages/cli/src/lib/permissions.ts diff --git a/docs/agent-permissions.md b/docs/agent-permissions.md index 1ca95c8..2b332d0 100644 --- a/docs/agent-permissions.md +++ b/docs/agent-permissions.md @@ -218,6 +218,50 @@ processes sharing one `~/.mocode` could race; the loser appends to a file that w just rotated away. Acceptable for a single-user CLI, not for a shared agent host — that needs an append-only sink (file → log shipper), not a local file. +### User rules + +The built-in policy is a floor, not the whole story: a team needs to say "never +`git push` here" or "always allow `npm test`" without patching source. + +Rules live in JSON, so they are reviewable and diffable: + +```jsonc +// /.mocode/permissions.json — committed with the repo +// ~/.mocode/permissions.json — applies to every project +{ + "rules": [ + { "tool": "bash", "pattern": "^git push", "behavior": "deny" }, + { "tool": "bash", "pattern": "^npm (test|run lint)", "behavior": "allow" }, + { "tool": "writeFile", "behavior": "ask" } + ] +} +``` + +| field | meaning | +|---|---| +| `tool` | exact tool name, or `*` | +| `pattern` | optional JS regex tested against the command (bash) or path (file tools) | +| `behavior` | `allow` / `deny` / `ask` | + +**Precedence is deny > allow > ask**, and a `deny` in either file wins over an +`allow` in the other — a safety rule must not be defeatable by a broader allow. +Within one behaviour the first matching rule wins, so the file reads top-down. + +Enforcement happens in two places on purpose: + +1. the **approval gate** — `deny` returns immediately without showing a dialog, + `allow` skips the built-in dialog, `ask` forces one; +2. the **executor** — `deny` throws `RULE_DENIED` and is audited, so a caller that + never goes through the UI (evals, subagents) is still bound by the policy. + +A malformed rule is **dropped and reported**, never coerced: turning +`"behavior": "denied"` into an allow would be the worst possible failure. Rules are +re-read on every call, so an edit applies to the next tool call without a restart. + +**Not implemented yet:** writing rules back from the TUI ("always allow for this +project"). The approval dialogs still record session-only allows; persisting them +would need a new dialog action. + ### Regression suite `packages/cli/src/lib/local-tools.sandbox.test.ts` runs the attack cases against the diff --git a/packages/cli/src/hooks/use-chat.ts b/packages/cli/src/hooks/use-chat.ts index eec00dd..569c600 100644 --- a/packages/cli/src/hooks/use-chat.ts +++ b/packages/cli/src/hooks/use-chat.ts @@ -63,6 +63,7 @@ import { AgentLoop, type AgentProgress, type AgentStopReason } from "../lib/agen import { failoverConsentFromEnv } from "../lib/provider-chain"; import { deriveAgentPhase, isRunActive, type AgentPhase } from "../lib/agent-phase"; import { markToolStarted, resetToolTimings } from "../lib/tool-timing"; +import { checkRules } from "../lib/permissions"; import { scheduleLocalSessionPersist } from "./use-chat-persist"; import { killTrackedToolProcesses } from "../lib/tool-process-registry"; import { @@ -552,9 +553,24 @@ export function useChat( return { approved: true }; } - if (toolCall.toolName === "bash" && mode === Mode.BUILD) { + // User rules from ~/.mocode and /.mocode. + const ruleVerdict = checkRules({ + tool: toolCall.toolName, + input: toolCall.input, + }); + + if (ruleVerdict?.behavior === "deny") { + return { approved: false, reason: ruleVerdict.detail }; + } + + // An explicit allow skips the built-in dialogs; an explicit ask forces + // one even where the built-in policy would have run the tool silently. + const ruleAllows = ruleVerdict?.behavior === "allow"; + const ruleAsks = ruleVerdict?.behavior === "ask"; + + if (!ruleAllows && toolCall.toolName === "bash" && mode === Mode.BUILD) { const { command } = toolInputSchemas.bash.parse(toolCall.input); - if (requiresApproval(command, sessionAllowRef.current)) { + if (ruleAsks || requiresApproval(command, sessionAllowRef.current)) { const verdict = await requestBashApproval(dialog, command); if (verdict === "reject") { return { approved: false, reason: BASH_REJECT_ERROR_TEXT }; @@ -565,12 +581,20 @@ export function useChat( } } + // Only write tools can prompt here; a rule asking about a read is not a + // reason to show a write dialog. + const isWriteTool = + toolCall.toolName === "writeFile" || toolCall.toolName === "editFile"; + if ( - requiresLocalWriteApproval( - toolCall.toolName, - mode, - sessionWriteAllowRef.current, - ) + !ruleAllows && + isWriteTool && + (ruleAsks || + requiresLocalWriteApproval( + toolCall.toolName, + mode, + sessionWriteAllowRef.current, + )) ) { const verdict = await requestLocalWriteApproval( dialog, diff --git a/packages/cli/src/lib/local-tools.rules.test.ts b/packages/cli/src/lib/local-tools.rules.test.ts new file mode 100644 index 0000000..86c3b3d --- /dev/null +++ b/packages/cli/src/lib/local-tools.rules.test.ts @@ -0,0 +1,72 @@ +/** + * User rules enforced by the executor. + * + * The approval dialog is a UI; the executor is the authority. A deny rule must + * hold even for a caller that never asked for approval (evals, subagents). + */ +import { afterAll, beforeAll, describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Mode } from "@mocode/shared"; +import { executeLocalTool } from "./local-tools"; +import { ToolPermissionError } from "./permission-rules"; + +const dir = mkdtempSync(join(tmpdir(), "mocode-rules-")); +const originalCwd = process.cwd(); + +function writeProjectRules(rules: unknown) { + mkdirSync(join(dir, ".mocode"), { recursive: true }); + writeFileSync(join(dir, ".mocode", "permissions.json"), JSON.stringify({ rules })); +} + +beforeAll(() => process.chdir(dir)); + +afterAll(() => { + process.chdir(originalCwd); + rmSync(dir, { recursive: true, force: true }); +}); + +describe("project rules", () => { + test("a deny rule blocks the call with RULE_DENIED", async () => { + writeProjectRules([ + { tool: "bash", pattern: "^curl ", behavior: "deny" }, + ]); + + try { + await executeLocalTool("bash", { command: "curl https://example.com" }, Mode.BUILD); + throw new Error("expected a refusal"); + } catch (error) { + expect(error).toBeInstanceOf(ToolPermissionError); + expect((error as ToolPermissionError).code).toBe("RULE_DENIED"); + expect((error as Error).message).toContain("Denied by project rule"); + } + }); + + test("a non-matching command still runs", async () => { + const result = (await executeLocalTool( + "bash", + { command: "echo allowed" }, + Mode.BUILD, + )) as { stdout: string }; + + expect(result.stdout.trim()).toBe("allowed"); + }); + + test("rules take effect without a restart", async () => { + writeProjectRules([{ tool: "bash", pattern: "^echo ", behavior: "deny" }]); + + await expect(executeLocalTool("bash", { command: "echo blocked" }, Mode.BUILD)).rejects.toThrow( + /Denied by project rule/, + ); + }); + + test("an allow rule does not bypass a hard deny", async () => { + writeProjectRules([{ tool: "bash", pattern: ".*", behavior: "allow" }]); + + // Hard denies sit below rules and are not negotiable. + await expect( + executeLocalTool("bash", { command: "mkfs.ext4 /dev/null" }, Mode.BUILD), + ).rejects.toThrow(/cannot be approved/); + }); +}); diff --git a/packages/cli/src/lib/local-tools.ts b/packages/cli/src/lib/local-tools.ts index af53bf6..b4de37f 100644 --- a/packages/cli/src/lib/local-tools.ts +++ b/packages/cli/src/lib/local-tools.ts @@ -26,6 +26,7 @@ import { trackToolProcess } from "./tool-process-registry"; import { resolveInsideCwd as resolveSafePath } from "./sandbox-path"; import { ToolPermissionError, buildSandboxEnv, findHardDeny } from "./permission-rules"; import { recordAudit, summarizeInput } from "./audit-log"; +import { checkRules } from "./permissions"; import { isShellConfined, sandboxBackend } from "./sandbox-capability"; import { parsePathList, runtimePaths, spawnPlan } from "./sandbox-profile"; @@ -409,6 +410,15 @@ export async function executeLocalTool( mode: ModeType, ): Promise { const detail = summarizeInput(input); + + // A deny rule is enforced here, at the authority, not only in the approval UI. + // Checked on every call so a rule edited mid-session applies immediately. + const verdict = checkRules({ tool: toolName, input }); + if (verdict?.behavior === "deny") { + recordAudit({ tool: toolName, mode, outcome: "denied", code: "RULE_DENIED", detail }); + throw new ToolPermissionError("RULE_DENIED", verdict.detail); + } + // Only shells have a containment question; record the answer alongside the call // so an audit can distinguish "ran in a sandbox" from "ran unconfined". const containment = toolName === "bash" ? { shellConfined: isShellConfined() } : {}; diff --git a/packages/cli/src/lib/permission-rules.ts b/packages/cli/src/lib/permission-rules.ts index 31425c4..779763f 100644 --- a/packages/cli/src/lib/permission-rules.ts +++ b/packages/cli/src/lib/permission-rules.ts @@ -26,7 +26,8 @@ export type PermissionCode = | "PATH_ESCAPE" | "HARD_DENY" | "MODE_DENIED" - | "UNKNOWN_TOOL"; + | "UNKNOWN_TOOL" + | "RULE_DENIED"; /** * Refusal raised by the executor. diff --git a/packages/cli/src/lib/permissions.test.ts b/packages/cli/src/lib/permissions.test.ts new file mode 100644 index 0000000..2b33b19 --- /dev/null +++ b/packages/cli/src/lib/permissions.test.ts @@ -0,0 +1,156 @@ +import { afterAll, describe, expect, test } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + evaluateRules, + loadPermissionRules, + loadRuleFile, + parseRules, + ruleSubject, + type LoadedRule, +} from "./permissions"; + +const dir = mkdtempSync(join(tmpdir(), "mocode-perms-")); +afterAll(() => rmSync(dir, { recursive: true, force: true })); + +function writeRules(path: string, content: string): string { + mkdirSync(join(path, ".."), { recursive: true }); + writeFileSync(path, content); + return path; +} + +describe("parseRules", () => { + test("accepts the documented shape", () => { + const result = parseRules( + { rules: [{ tool: "bash", pattern: "^npm ", behavior: "allow" }] }, + "project", + ); + + expect(result.errors).toEqual([]); + expect(result.rules).toHaveLength(1); + expect(result.rules[0]).toMatchObject({ tool: "bash", behavior: "allow", source: "project" }); + }); + + test("drops a rule with a bad behaviour instead of coercing it", () => { + // Coercing `"denied"` into an allow would be the worst possible failure. + const result = parseRules({ rules: [{ tool: "bash", behavior: "denied" }] }, "user"); + + expect(result.rules).toEqual([]); + expect(result.errors[0]).toContain("behavior"); + }); + + test("drops a rule with an invalid regex and says so", () => { + const result = parseRules({ rules: [{ tool: "bash", pattern: "([", behavior: "deny" }] }, "user"); + + expect(result.rules).toEqual([]); + expect(result.errors[0]).toContain("invalid regex"); + }); + + test("requires a tool name", () => { + const result = parseRules({ rules: [{ behavior: "deny" }] }, "user"); + expect(result.errors[0]).toContain("tool"); + }); + + test("reports a non-array rules field", () => { + expect(parseRules({ rules: "nope" }, "user").errors[0]).toContain("array"); + }); + + test("an empty object is not an error", () => { + expect(parseRules({}, "user").errors).toEqual([]); + }); +}); + +describe("loadRuleFile", () => { + test("a missing file is normal, not an error", () => { + expect(loadRuleFile(join(dir, "nope.json"), "user")).toEqual({ rules: [], errors: [] }); + }); + + test("invalid JSON is reported and yields no rules", () => { + const path = writeRules(join(dir, "bad.json"), "{ not json"); + const result = loadRuleFile(path, "project"); + + expect(result.rules).toEqual([]); + expect(result.errors[0]).toContain("invalid JSON"); + }); +}); + +describe("loadPermissionRules", () => { + test("merges user and project files, tagging the source", () => { + const userPath = writeRules( + join(dir, "user.json"), + JSON.stringify({ rules: [{ tool: "bash", behavior: "ask" }] }), + ); + const projectPath = writeRules( + join(dir, "project.json"), + JSON.stringify({ rules: [{ tool: "writeFile", behavior: "deny" }] }), + ); + + const result = loadPermissionRules(dir, { user: userPath, project: projectPath }); + + expect(result.rules.map((rule) => rule.source)).toEqual(["user", "project"]); + }); +}); + +describe("evaluateRules", () => { + function rules(...entries: Array>): LoadedRule[] { + return entries.map((entry) => ({ + tool: "bash", + behavior: "ask", + source: "project", + pattern: undefined, + regex: null, + ...entry, + })) as LoadedRule[]; + } + + test("a deny anywhere wins over an allow everywhere", () => { + const list = rules( + { tool: "bash", behavior: "allow", source: "user" }, + { tool: "bash", behavior: "deny", source: "project" }, + ); + + expect(evaluateRules(list, "bash", "rm -rf /")).toMatchObject({ behavior: "deny" }); + }); + + test("an allow suppresses the ask", () => { + const list = rules({ tool: "bash", behavior: "allow", regex: /^npm / }); + + expect(evaluateRules(list, "bash", "npm test")).toMatchObject({ behavior: "allow" }); + expect(evaluateRules(list, "bash", "curl x")).toBeNull(); + }); + + test("wildcard tools match everything", () => { + const list = rules({ tool: "*", behavior: "deny" }); + expect(evaluateRules(list, "writeFile", "a.txt")).toMatchObject({ behavior: "deny" }); + }); + + test("first match wins within one behaviour", () => { + const list = rules( + { tool: "bash", behavior: "deny", regex: /^git push/ }, + { tool: "bash", behavior: "deny", regex: /^git/ }, + ); + + expect(evaluateRules(list, "bash", "git push origin main")?.rule.regex?.source).toBe( + "^git push", + ); + }); + + test("no match returns null so the caller keeps its own policy", () => { + expect(evaluateRules(rules({ tool: "writeFile", behavior: "deny" }), "bash", "ls")).toBeNull(); + }); +}); + +describe("ruleSubject", () => { + test("bash matches on the command", () => { + expect(ruleSubject("bash", { command: "npm test" })).toBe("npm test"); + }); + + test("file tools match on the path", () => { + expect(ruleSubject("writeFile", { path: "src/a.ts", content: "x" })).toBe("src/a.ts"); + }); + + test("unknown input yields an empty subject", () => { + expect(ruleSubject("bash", null)).toBe(""); + }); +}); diff --git a/packages/cli/src/lib/permissions.ts b/packages/cli/src/lib/permissions.ts new file mode 100644 index 0000000..17f62a7 --- /dev/null +++ b/packages/cli/src/lib/permissions.ts @@ -0,0 +1,218 @@ +/** + * User-writable permission rules. + * + * The previous model was a hardcoded blocklist plus an in-session allow set: a user + * could approve something for an hour, never for a project, and could not deny + * anything the blocklist did not already know about. + * + * Rules live in JSON so the policy is reviewable, diffable and shareable: + * + * ~/.mocode/permissions.json user rules, apply to every project + * /.mocode/permissions.json project rules, committed with the repo + * + * Precedence is **deny > allow > ask**, matching how a safety rule should behave: + * a deny anywhere wins over an allow everywhere. Within one behaviour the first + * matching rule wins, so the file reads top-down. + * + * Pure: no I/O in the evaluator, so the precedence table is testable on its own. + */ +import { readFileSync } from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +export type PermissionBehavior = "allow" | "deny" | "ask"; + +export type PermissionRule = { + /** Exact tool name, or `*` for every tool. */ + tool: string; + /** Optional JavaScript regex tested against the tool's subject. */ + pattern?: string; + behavior: PermissionBehavior; +}; + +export type RuleSource = "user" | "project"; + +export type LoadedRule = PermissionRule & { + source: RuleSource; + /** Compiled pattern, or null when the rule matches on tool name alone. */ + regex: RegExp | null; +}; + +export type RuleLoadResult = { + rules: LoadedRule[]; + /** Human-readable problems. A bad rule is dropped, never guessed at. */ + errors: string[]; +}; + +const BEHAVIORS: ReadonlySet = new Set(["allow", "deny", "ask"]); + +/** + * Validate raw parsed JSON. + * + * A malformed rule is dropped rather than coerced: silently turning + * `{"behavior": "denied"}` into an allow would be the worst possible failure. + */ +export function parseRules(raw: unknown, source: RuleSource): RuleLoadResult { + const errors: string[] = []; + const rules: LoadedRule[] = []; + + const list = (raw as { rules?: unknown })?.rules; + + // An absent `rules` key is an empty policy; a present but wrong-typed one is a + // mistake worth reporting. + if (list === undefined) return { rules, errors }; + + if (!Array.isArray(list)) { + errors.push("`rules` must be an array"); + return { rules, errors }; + } + + list.forEach((entry, index) => { + const where = `rules[${index}]`; + if (entry === null || typeof entry !== "object") { + errors.push(`${where}: not an object`); + return; + } + + const { tool, pattern, behavior } = entry as Record; + + if (typeof tool !== "string" || tool.length === 0) { + errors.push(`${where}: missing "tool"`); + return; + } + + if (typeof behavior !== "string" || !BEHAVIORS.has(behavior)) { + errors.push(`${where}: "behavior" must be allow, deny or ask`); + return; + } + + if (pattern !== undefined && typeof pattern !== "string") { + errors.push(`${where}: "pattern" must be a string`); + return; + } + + let regex: RegExp | null = null; + if (typeof pattern === "string") { + try { + regex = new RegExp(pattern); + } catch { + errors.push(`${where}: invalid regex ${JSON.stringify(pattern)}`); + return; + } + } + + rules.push({ tool, pattern, behavior: behavior as PermissionBehavior, source, regex }); + }); + + return { rules, errors }; +} + +/** Read one rule file. A missing file is normal; a malformed one is reported. */ +export function loadRuleFile(path: string, source: RuleSource): RuleLoadResult { + let text: string; + + try { + text = readFileSync(path, "utf8"); + } catch { + return { rules: [], errors: [] }; + } + + try { + return parseRules(JSON.parse(text), source); + } catch { + return { rules: [], errors: [`${path}: invalid JSON`] }; + } +} + +export const USER_RULES_PATH = () => join(homedir(), ".mocode", "permissions.json"); +export const PROJECT_RULES_PATH = (cwd: string) => join(cwd, ".mocode", "permissions.json"); + +/** + * Load user rules then project rules. + * + * Order only decides *within* a behaviour; deny always beats allow regardless of + * which file it came from. + */ +export function loadPermissionRules( + cwd: string = process.cwd(), + paths: { user?: string; project?: string } = {}, +): RuleLoadResult { + const user = loadRuleFile(paths.user ?? USER_RULES_PATH(), "user"); + const project = loadRuleFile(paths.project ?? PROJECT_RULES_PATH(cwd), "project"); + + return { + rules: [...user.rules, ...project.rules], + errors: [...user.errors, ...project.errors], + }; +} + +export type RuleMatch = { + behavior: PermissionBehavior; + rule: LoadedRule; +}; + +/** + * Evaluate rules for one tool call. + * + * Returns `null` when nothing matches, which the caller treats as "fall back to the + * built-in policy". Deny is scanned first across **all** rules, then allow, then ask. + */ +export function evaluateRules( + rules: readonly LoadedRule[], + tool: string, + subject: string, +): RuleMatch | null { + const matches = (rule: LoadedRule): boolean => { + if (rule.tool !== "*" && rule.tool !== tool) return false; + return rule.regex === null || rule.regex.test(subject); + }; + + for (const behavior of ["deny", "allow", "ask"] as const) { + const rule = rules.find((candidate) => candidate.behavior === behavior && matches(candidate)); + if (rule) return { behavior, rule }; + } + + return null; +} + +/** The string a rule's pattern is tested against. */ +export function ruleSubject(tool: string, input: unknown): string { + if (input === null || typeof input !== "object") return ""; + + const record = input as Record; + if (tool === "bash" && typeof record.command === "string") return record.command; + if (typeof record.path === "string") return record.path; + + return ""; +} + +export type RuleVerdict = { + behavior: PermissionBehavior; + detail: string; +}; + +/** + * Load the rule files and evaluate one call. + * + * Loaded per call on purpose: a rule edited in the project file takes effect on the + * very next tool call, which is what makes the policy demoable without a restart. + * Two small JSON reads per tool call is noise next to the model request. + */ +export function checkRules( + params: { tool: string; input: unknown }, + cwd: string = process.cwd(), +): RuleVerdict | null { + const { rules } = loadPermissionRules(cwd); + if (rules.length === 0) return null; + + const match = evaluateRules(rules, params.tool, ruleSubject(params.tool, params.input)); + if (!match) return null; + + const origin = `${match.rule.source} rule`; + const pattern = match.rule.pattern ? ` /${match.rule.pattern}/` : ""; + + return { + behavior: match.behavior, + detail: `${match.behavior === "allow" ? "Allowed" : match.behavior === "deny" ? "Denied" : "Approval required"} by ${origin} (${match.rule.tool}${pattern})`, + }; +} From 1c64a94c78ab16cc01af1b48a9d25525c1f809db Mon Sep 17 00:00:00 2001 From: moyun Date: Thu, 12 Nov 2026 13:11:33 +0800 Subject: [PATCH 11/11] fix(ci): generate the Prisma client from the package that owns it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI failed in @mocode/server's postinstall: it delegated to `bun run --cwd ../database db:generate`, which ran `bunx prisma generate`. Server has no prisma dependency, so bunx resolved the *latest* registry version instead of the pinned 7.8.0 — and that version does not register a `generate` command: [CLI.UNKNOWN_COMMAND]. - move the postinstall to @mocode/database, which owns the prisma dependency - call the local binary (`prisma generate`) instead of `bunx prisma`, so the pinned version is used and no network resolution happens during install --- packages/database/package.json | 3 +- packages/server/package.json | 77 +++++++++++++++++----------------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/packages/database/package.json b/packages/database/package.json index 68f9ecc..62ab26c 100644 --- a/packages/database/package.json +++ b/packages/database/package.json @@ -8,7 +8,8 @@ "./client": "./src/client.ts" }, "scripts": { - "db:generate": "bunx prisma generate" + "db:generate": "prisma generate", + "postinstall": "DATABASE_URL=${DATABASE_URL:-postgresql://localhost:5432/mocode} prisma generate" }, "devDependencies": { "@types/node": "^25.9.3", diff --git a/packages/server/package.json b/packages/server/package.json index 093ec3f..34455c3 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,40 +1,39 @@ { - "name": "@mocode/server", - "version": "0.0.1", - "private": true, - "type": "module", - "exports": { - ".": "./src/index.ts" - }, - "scripts": { - "dev": "bun run --hot src/index.ts", - "build": "bun build src/index.ts --outdir dist --target bun", - "postinstall": "bun run --cwd ../database db:generate", - "test": "bun test", - "test:providers": "bun run scripts/test-providers.ts" - }, - "dependencies": { - "@ai-sdk/anthropic": "^3.0.85", - "@ai-sdk/cerebras": "^2.0.51", - "@ai-sdk/gateway": "^3.0.133", - "@ai-sdk/google": "^3.0.30", - "@ai-sdk/groq": "^3.0.21", - "@ai-sdk/openai": "^3.0.72", - "@ai-sdk/provider-utils": "^4.0.30", - "@clerk/backend": "^3.8.2", - "@hono/zod-validator": "^0.8.0", - "@mocode/database": "workspace:*", - "@mocode/shared": "workspace:*", - "@openrouter/ai-sdk-provider": "^2.9.1", - "@polar-sh/sdk": "^0.48.1", - "@sentry/bun": "^10.58.0", - "@sentry/hono": "^10.58.0", - "ai": "^6.0.207", - "dotenv": "^17.4.2", - "hono": "^4.12.25", - "zod": "^4.4.3" - }, - "devDependencies": { - "@types/bun": "latest" - } -} \ No newline at end of file + "name": "@mocode/server", + "version": "0.0.1", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "dev": "bun run --hot src/index.ts", + "build": "bun build src/index.ts --outdir dist --target bun", + "test": "bun test", + "test:providers": "bun run scripts/test-providers.ts" + }, + "dependencies": { + "@ai-sdk/anthropic": "^3.0.85", + "@ai-sdk/cerebras": "^2.0.51", + "@ai-sdk/gateway": "^3.0.133", + "@ai-sdk/google": "^3.0.30", + "@ai-sdk/groq": "^3.0.21", + "@ai-sdk/openai": "^3.0.72", + "@ai-sdk/provider-utils": "^4.0.30", + "@clerk/backend": "^3.8.2", + "@hono/zod-validator": "^0.8.0", + "@mocode/database": "workspace:*", + "@mocode/shared": "workspace:*", + "@openrouter/ai-sdk-provider": "^2.9.1", + "@polar-sh/sdk": "^0.48.1", + "@sentry/bun": "^10.58.0", + "@sentry/hono": "^10.58.0", + "ai": "^6.0.207", + "dotenv": "^17.4.2", + "hono": "^4.12.25", + "zod": "^4.4.3" + }, + "devDependencies": { + "@types/bun": "latest" + } +}