From 51e9c982f6bbfb2161ae7ff806db3b130a258000 Mon Sep 17 00:00:00 2001 From: zhangxinhao Date: Fri, 13 Nov 2026 11:17:13 +0800 Subject: [PATCH 1/5] feat(v2-server): report why a model response was cut and refuse a call before upstream The route's response now records the cut reason (idle or total limit, cancelled, interrupted) for CutReason, and RefuseCall answers a call with a protocol-shaped 422 whose text avoids the words Agents re-send on. --- v2-server/internal/nodeservices/README.md | 2 +- v2-server/internal/nodeservices/provider.go | 68 ++++++++++++++++++--- 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/v2-server/internal/nodeservices/README.md b/v2-server/internal/nodeservices/README.md index 59113c71..1a75fd67 100644 --- a/v2-server/internal/nodeservices/README.md +++ b/v2-server/internal/nodeservices/README.md @@ -38,7 +38,7 @@ renew 必须匹配原 execution,过期后不能续期。binding 和 Model 快 `NewRemoteProvider` 转发 OpenAI chat completions,`NewRemoteMessagesProvider` 转发 Anthropic Messages(后端 API key 放在 `x-api-key`,或将配置的 auth token 放在 `Authorization: Bearer …`,只透传 `anthropic-version` / `anthropic-beta`);两者都支持 JSON/SSE,强制使用已选模型,保留成功响应的 usage 等字段。真实 API key 留在后端内存,不转发 Agent token、调用者自定义 header 或上游错误正文; -禁止 HTTP 和重定向。Agent 断开、grant 撤销(`End`、删除授权、`Close`)会取消进行中的上游请求;grant 到期只拒绝新请求和续期,不取消已经开始的请求。单次远端请求有空闲超时(默认 2 分钟,每收到一段上游数据重新计时,也覆盖等待响应头)和总上限(默认 30 分钟),由 `SetLimits` 设置。响应头发出后因上述任一原因中途停止时,provider 只标记这个响应(经 `Unwrap` 穿过包装它的 writer),由 Agent 模型路由在 `Complete` 返回后以 `http.ErrAbortHandler` 中止,Agent 读到的是失败而不是正常结束的流;不经这条路由的进程内调用方(纯模型轮次)只看到被截断的正文。`CompleteObserved` 另返回 `cut` 供调用方记账。 +禁止 HTTP 和重定向。Agent 断开、grant 撤销(`End`、删除授权、`Close`)会取消进行中的上游请求;grant 到期只拒绝新请求和续期,不取消已经开始的请求。单次远端请求有空闲超时(默认 2 分钟,每收到一段上游数据重新计时,也覆盖等待响应头)和总上限(默认 30 分钟),由 `SetLimits` 设置。响应头发出后因上述任一原因中途停止时,provider 只标记这个响应(经 `Unwrap` 穿过包装它的 writer),由 Agent 模型路由在 `Complete` 返回后以 `http.ErrAbortHandler` 中止,Agent 读到的是失败而不是正常结束的流;不经这条路由的进程内调用方(纯模型轮次)只看到被截断的正文。`CompleteObserved` 另返回 `cut` 供调用方记账。标记里带着原因(`model_idle_timeout`、`model_max_duration`、`cancelled`、`interrupted`),包装 provider 的调用方在 `Complete` 返回后用 `CutReason(w)` 读取;`RefuseCall` 在响应头前按协议回一个 422 错误,文字不含 Agent 会据以重发的字样(网络失败、超时、5xx 状态码),Agent 收到后不再重发。 不支持运行时模型切换或任何隐式环境变量凭据读取。 ## 验证 diff --git a/v2-server/internal/nodeservices/provider.go b/v2-server/internal/nodeservices/provider.go index 767e0bcd..a4cc6330 100644 --- a/v2-server/internal/nodeservices/provider.go +++ b/v2-server/internal/nodeservices/provider.go @@ -32,8 +32,8 @@ const ( ) var ( - errIdleTimeout = errors.New("model_idle_timeout") - errMaxDuration = errors.New("model_max_duration") + errIdleTimeout = errors.New(CutIdle) + errMaxDuration = errors.New(CutMaxDuration) ) // Limits returns the idle and total limits of one call. @@ -167,7 +167,7 @@ func (p *RemoteProvider) CompleteObserved(w http.ResponseWriter, r *http.Request reason = "cancelled" } slog.WarnContext(r.Context(), "model response cut off", "reason", reason) - markCut(w) + markCut(w, reason) return resp.StatusCode, true } return resp.StatusCode, false @@ -268,6 +268,9 @@ func (FixtureProvider) Complete(w http.ResponseWriter, r *http.Request, model Mo type abortableWriter struct { http.ResponseWriter cut bool + // reason is why the provider stopped the response: model_idle_timeout, model_max_duration, + // cancelled or interrupted. + reason string } func (w *abortableWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } @@ -280,16 +283,67 @@ func (w *abortableWriter) Flush() { // markCut finds the route's abortableWriter through wrappers that expose Unwrap, as // http.ResponseController does; without one there is nothing to abort. -func markCut(w http.ResponseWriter) { +func markCut(w http.ResponseWriter, reason string) { + if a := routeWriter(w); a != nil { + a.cut, a.reason = true, reason + } +} + +func routeWriter(w http.ResponseWriter) *abortableWriter { for { if a, ok := w.(*abortableWriter); ok { - a.cut = true - return + return a } u, ok := w.(interface{ Unwrap() http.ResponseWriter }) if !ok { - return + return nil + } + w = u.Unwrap() + } +} + +// CutReason reports why a provider stopped this Agent model response after its header was sent, +// or "" when it was not cut. Only the two limits are reported by name (CutIdle, CutMaxDuration); +// any other cut is "interrupted" or "cancelled". It asks the first writer in the Unwrap chain +// that knows, which on the Agent model route is the route's own response. +func CutReason(w http.ResponseWriter) string { + for { + if c, ok := w.(interface{ CutReason() string }); ok { + return c.CutReason() + } + u, ok := w.(interface{ Unwrap() http.ResponseWriter }) + if !ok { + return "" } w = u.Unwrap() } } + +// CutReason implements the lookup used by the exported CutReason. +func (w *abortableWriter) CutReason() string { + if !w.cut { + return "" + } + return w.reason +} + +// Cut reasons that end a model response at a limit. +const ( + CutIdle = "model_idle_timeout" + CutMaxDuration = "model_max_duration" +) + +// RefuseCall answers an Agent model call before any upstream work with a final, protocol-shaped +// error. Agents retry 408, 409, 429, 5xx and errors whose text reads like a network failure or a +// timeout, so the status is 422 and the message avoids those words: the Agent stops and reports +// the message instead of re-sending the same request. +func RefuseCall(w http.ResponseWriter, protocol Protocol, code, message string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusUnprocessableEntity) + text := message + "(" + code + ")" + if protocol == AnthropicMessages { + _ = json.NewEncoder(w).Encode(map[string]any{"type": "error", "error": map[string]string{"type": "invalid_request_error", "message": text}}) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"error": map[string]string{"message": text, "type": "invalid_request_error", "code": code}}) +} From 02d7eb6724d5a564f924c14a37e603bd2944461e Mon Sep 17 00:00:00 2001 From: zhangxinhao Date: Fri, 13 Nov 2026 11:17:13 +0800 Subject: [PATCH 2/5] fix(v2-server): stop Agents re-sending model calls cut at a limit Each active turn guards its model calls: after a total-limit cut every later call of the turn is refused before upstream; after an idle cut one re-send is allowed, a second stall refuses the rest. Calls cut together count as one stall. A turn Node records as cancelled whose last call was cut at a limit fails with that limit as its failure code. --- v2-server/README.md | 2 +- v2-server/internal/agentruntime/README.md | 4 + v2-server/internal/agentruntime/project.go | 19 +- .../internal/agentruntime/provider_test.go | 5 +- .../internal/agentruntime/retry_guard.go | 93 +++++++++ .../internal/agentruntime/retry_guard_test.go | 180 ++++++++++++++++++ v2-server/internal/agentruntime/session.go | 11 +- .../internal/agentruntime/session_test.go | 4 +- 8 files changed, 304 insertions(+), 14 deletions(-) create mode 100644 v2-server/internal/agentruntime/retry_guard.go create mode 100644 v2-server/internal/agentruntime/retry_guard_test.go diff --git a/v2-server/README.md b/v2-server/README.md index 9b0b442d..37321b32 100644 --- a/v2-server/README.md +++ b/v2-server/README.md @@ -35,7 +35,7 @@ go run . # 监听 127.0.0.1:8090 | `V2_NODE_ASSETS_DIR` | 未设置 | `task v2:node:prepare` 生成的固定 Node、host、guardian、Deno、OpenCode 和 Claude Code 资源;未设置时只启用纯模型路径 | | `V2_MODEL_BASE_URL` / `V2_MODEL_API_KEY` / `V2_MODEL_NAME` | 未设置 | 可选的一组 HTTPS OpenAI 兼容 provider;必须同时设置,OpenCode 与纯模型共用,清单只公开模型名称与版本 | | `V2_ANTHROPIC_MODEL_BASE_URL` / `V2_ANTHROPIC_MODEL_API_KEY` / `V2_ANTHROPIC_MODEL_NAME` | 未设置 | 可选的一组 HTTPS Anthropic 兼容 provider(Messages API,base URL 写到 `/v1`,如 `https://api.anthropic.com/v1`);必须同时提供地址、凭据和模型名,只供 Claude Code 使用。使用 Bearer 网关时以 `V2_ANTHROPIC_MODEL_AUTH_TOKEN` 代替 `V2_ANTHROPIC_MODEL_API_KEY`,二者不能同时配置;Claude Code 的 base URL 若以 `/compatible` 结尾,这里填 `/compatible/v1` | -| `V2_MODEL_IDLE_TIMEOUT` / `V2_MODEL_MAX_DURATION` | `2m` / `30m` | 单次上游模型请求的空闲超时和总上限,两种协议共用;必须为正 duration,空闲超时不能大于总上限。连续这么久收不到上游数据、或总时长到上限时中止响应,Agent 收到的是失败而不是正常结束的流 | +| `V2_MODEL_IDLE_TIMEOUT` / `V2_MODEL_MAX_DURATION` | `2m` / `30m` | 单次上游模型请求的空闲超时和总上限,两种协议共用;必须为正 duration,空闲超时不能大于总上限。连续这么久收不到上游数据、或总时长到上限时中止响应,Agent 收到的是失败而不是正常结束的流;总上限切断后这一轮不再放行重发,空闲超时切断后只放行 1 次,这一轮按失败结束并显示原因 | 测试需要 `TEST_DATABASE_URL`(任意可建 schema 的库),每个测试用独立的 schema: diff --git a/v2-server/internal/agentruntime/README.md b/v2-server/internal/agentruntime/README.md index 41640d8d..be844157 100644 --- a/v2-server/internal/agentruntime/README.md +++ b/v2-server/internal/agentruntime/README.md @@ -38,6 +38,10 @@ err = driver.Close(ctx) End 等待当前 Node incarnation 的有效 `agent_session_ended`;缺字段、未知原因、错 Node / incarnation 或传输断开不能当成停止确认。项目已 dead/fatal 时,End 使用调用方 context 走 supervisor 清理;确认旧进程树停止并完成关闭后移除原项目条目,不需要另一个 Run 触发回收。停止失败或超时返回 `ErrStopUnconfirmed`,保留原项目,供后续 End 重试;上层不能释放目录或派发后继。Close 最后通过已有 Node supervisor 确认所有自有进程退出;失败可重试。目录和工具写入在正常 End/Close 后保留。 +## 被切断后的重发 + +Agent 把被切断的模型请求当成网络失败原样重发(OpenCode 0.6.11 最多 5 次,Claude Code 0.4.5 十次以上),每次都把半截回答再流一遍。每个模型请求都套上所在轮的守卫:总上限切断后,这一轮之后的请求都在响应头前用 `RefuseCall` 拒绝;空闲超时切断后放行 1 次重发,再被空闲超时切断就同样拒绝。同时在跑、一起被切断的请求(例如标题请求)只算一次;切断之后开始并正常结束的请求清零。Node 把失败的轮次记成 `cancelled`、不带错误文字,所以这一轮最后的请求是被限值切断的时候,结果改为失败,`FailureCode` 就是切断原因,供页面显示。 + ## 首次输出前的等待 会话第一轮到首个可见输出之前的等待按五段计时:`runtime`(启动项目 Node)、`workspace`(准备工作目录和技能,用上预热副本时标 `reused`)、`plugin`(等插件安装剩余部分,安装与 clone 并行,这里只算 clone 之后还要等的时间)、`agent`(从发出 `start_agent_session` 到 Node 记录会话)、`response`(从会话记录到首个思考、回答或工具调用)。每段接着上一段开始,加起来就是这段等待。 diff --git a/v2-server/internal/agentruntime/project.go b/v2-server/internal/agentruntime/project.go index b60fbcf0..caeaade7 100644 --- a/v2-server/internal/agentruntime/project.go +++ b/v2-server/internal/agentruntime/project.go @@ -235,26 +235,31 @@ func (p *projectRuntime) start() error { // providerFor binds each observed HTTP request to the current admitted turn, not a browser ID. // Returning nil denies a request when its execution or active turn is no longer present. +// Every call of an active turn also passes the turn's retry guard. func (p *projectRuntime) providerFor(id string, binding nodeservices.Binding) nodeservices.Provider { observe := p.driver.opts.ObserveProvider - if observe == nil { - return binding.Provider - } p.mu.Lock() a := p.agents[id] p.mu.Unlock() if a == nil { + if observe == nil { + return binding.Provider + } return nil } - if !a.observeRequests { - return binding.Provider - } a.mu.Lock() defer a.mu.Unlock() if a.active == nil { + if observe == nil || !a.observeRequests { + return binding.Provider + } return nil } - return observe(execution.Ref{SessionID: a.frozen.ID, TurnID: a.active.id}, binding) + provider := binding.Provider + if observe != nil && a.observeRequests { + provider = observe(execution.Ref{SessionID: a.frozen.ID, TurnID: a.active.id}, binding) + } + return a.active.guard.provider(provider, binding.Protocol) } func (p *projectRuntime) ipc() nodeClient { diff --git a/v2-server/internal/agentruntime/provider_test.go b/v2-server/internal/agentruntime/provider_test.go index ba1cd3c5..94ffef2c 100644 --- a/v2-server/internal/agentruntime/provider_test.go +++ b/v2-server/internal/agentruntime/provider_test.go @@ -31,8 +31,9 @@ func TestProviderForCurrentObservedTurnAndPassthrough(t *testing.T) { } for _, turn := range []string{"one", "two"} { a.active = &activeTurn{id: turn} - if got := p.providerFor("execution", binding); !reflect.DeepEqual(got, observed) { - t.Fatal("observer not wired") + got, ok := p.providerFor("execution", binding).(guardedProvider) + if !ok || !reflect.DeepEqual(got.next, observed) || got.guard != &a.active.guard { + t.Fatal("observer not wired behind the turn's retry guard") } } if !reflect.DeepEqual(refs, []execution.Ref{{SessionID: "session", TurnID: "one"}, {SessionID: "session", TurnID: "two"}}) { diff --git a/v2-server/internal/agentruntime/retry_guard.go b/v2-server/internal/agentruntime/retry_guard.go new file mode 100644 index 00000000..d1e53551 --- /dev/null +++ b/v2-server/internal/agentruntime/retry_guard.go @@ -0,0 +1,93 @@ +package agentruntime + +import ( + "net/http" + "sync" + + "github.com/wanglongan587/cloud/v2-server/internal/nodeservices" +) + +// Agents re-send a model call that was cut off as if the network had failed: OpenCode up to +// five times, Claude Code ten or more, each re-send streaming its partial answer again. A cut +// at a model limit is not a network failure, so one turn allows only what the limit intends: +// none after the total limit, one after an idle cut. Every later call of the turn is refused +// before any upstream work, and the turn reports the limit as its failure. +type retryGuard struct { + mu sync.Mutex + // idleCuts counts stalls (idle cuts of calls started after the previous cut) since the last + // call that started after a cut and completed. + idleCuts int + // calls numbers the turn's calls as they start; cutAfter is the number of the newest call + // started when the latest cut happened, so only a call started later can clear it. + calls, cutAfter int + // last is the limit that cut the turn's most recent call, cleared by a later completed call. + last string + // stopped is the limit that ended the turn; every later call is refused with it. + stopped string +} + +var refusals = map[string]string{ + nodeservices.CutMaxDuration: "模型请求超过时长上限被切断,这一轮已停止", + nodeservices.CutIdle: "模型多次长时间没有输出,这一轮已停止", +} + +func (g *retryGuard) provider(next nodeservices.Provider, protocol nodeservices.Protocol) nodeservices.Provider { + return guardedProvider{g, next, protocol} +} + +// failure is the limit a turn ended at, or "" when its last call was not cut at a limit. +func (g *retryGuard) failure() string { + g.mu.Lock() + defer g.mu.Unlock() + if g.stopped != "" { + return g.stopped + } + return g.last +} + +func (g *retryGuard) begin() (stopped string, call int) { + g.mu.Lock() + defer g.mu.Unlock() + g.calls++ + return g.stopped, g.calls +} + +func (g *retryGuard) finish(call int, reason string) { + g.mu.Lock() + defer g.mu.Unlock() + switch reason { + case nodeservices.CutMaxDuration: + g.last, g.stopped, g.cutAfter = reason, reason, g.calls + case nodeservices.CutIdle: + // Calls that were already running at the previous cut stalled with it (a title request + // beside the turn's call): one stall, however many calls it cuts, earns one retry. + if call > g.cutAfter { + g.idleCuts++ + } + g.last, g.cutAfter = reason, g.calls + if g.idleCuts > 1 { + g.stopped = reason + } + case "": + // A call running alongside the cut one (a title request) proves nothing about the retry. + if call > g.cutAfter { + g.idleCuts, g.last = 0, "" + } + } +} + +type guardedProvider struct { + guard *retryGuard + next nodeservices.Provider + protocol nodeservices.Protocol +} + +func (p guardedProvider) Complete(w http.ResponseWriter, r *http.Request, m nodeservices.Model) { + stopped, n := p.guard.begin() + if stopped != "" { + nodeservices.RefuseCall(w, p.protocol, stopped, refusals[stopped]) + return + } + p.next.Complete(w, r, m) + p.guard.finish(n, nodeservices.CutReason(w)) +} diff --git a/v2-server/internal/agentruntime/retry_guard_test.go b/v2-server/internal/agentruntime/retry_guard_test.go new file mode 100644 index 00000000..0a63a72d --- /dev/null +++ b/v2-server/internal/agentruntime/retry_guard_test.go @@ -0,0 +1,180 @@ +package agentruntime + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "regexp" + "testing" + + "github.com/wanglongan587/cloud/v2-server/internal/execution" + "github.com/wanglongan587/cloud/v2-server/internal/nodehistory" + "github.com/wanglongan587/cloud/v2-server/internal/nodeservices" +) + +// cutWriter stands in for the Agent model route's response, which knows why a call was cut. +type cutWriter struct { + *httptest.ResponseRecorder + reason string +} + +func (w *cutWriter) CutReason() string { return w.reason } + +// scriptedCalls ends each call it serves with the next scripted cut reason ("" completes). +type scriptedCalls struct { + cuts []string + calls int +} + +func (s *scriptedCalls) Complete(w http.ResponseWriter, _ *http.Request, _ nodeservices.Model) { + w.WriteHeader(200) + w.(*cutWriter).reason = s.cuts[s.calls] + s.calls++ +} + +func call(t *testing.T, p nodeservices.Provider) *cutWriter { + t.Helper() + w := &cutWriter{ResponseRecorder: httptest.NewRecorder()} + p.Complete(w, httptest.NewRequest("POST", "/v1/chat/completions", nil), nodeservices.Model{}) + return w +} + +// OpenCode re-sends when the error message or response body matches one of these patterns +// (RETRYABLE_MESSAGE_PATTERNS in OpenCode v1.19.3 packages/opencode/src/session/retry.ts), +// besides the statuses its provider SDK treats as retryable. +var retryablePatterns = []*regexp.Regexp{ + regexp.MustCompile(`(?i)429|500|502|503|504|524`), + regexp.MustCompile(`(?i)rate increased too quickly|rate limit|rate-limit|rate_limit|too many requests`), + regexp.MustCompile(`(?i)overloaded|service unavailable|service_unavailable|service-unavailable|internal error|internal_error|internal server error|server error|server_error|server-error|provider returned error|provider_returned_error|provider-returned-error`), + regexp.MustCompile(`(?i)terminated|fetch failed|failed to fetch|network[-_\s]error|upstream connect|connection error|connection refused|connection lost|socket connection was closed|socket hang up|reset before headers|getaddrinfo|enotfound|eai_again|econnrefused|econnreset|etimedout`), + regexp.MustCompile(`(?i)^timeout$|\b(?:request|response|connection|network|stream|read) (?:timeout|timed out|time out)\b`), + regexp.MustCompile(`(?i)try your request again|retry your request|resource exhausted|resource_exhausted`), + regexp.MustCompile(`(?i)\btry again (?:later|in\b)|\b(?:currently|temporarily) at capacity\b`), +} + +func retryable(text string) bool { + for _, p := range retryablePatterns { + if p.MatchString(text) { + return true + } + } + return false +} + +func assertRefused(t *testing.T, w *cutWriter, protocol nodeservices.Protocol, code string) { + t.Helper() + if w.Code != http.StatusUnprocessableEntity { + t.Fatalf("refusal status %d", w.Code) + } + var body struct { + Type string `json:"type"` + Error struct { + Type string `json:"type"` + Message string `json:"message"` + Code string `json:"code"` + } `json:"error"` + } + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatal(err) + } + if protocol == nodeservices.AnthropicMessages && body.Type != "error" { + t.Fatalf("not an Anthropic error: %s", w.Body) + } + if protocol == nodeservices.OpenAICompletions && body.Error.Code != code { + t.Fatalf("code %q, want %q", body.Error.Code, code) + } + if body.Error.Message == "" || retryable(body.Error.Message) || retryable(w.Body.String()) { + t.Fatalf("refusal %s would be retried or has no message", w.Body) + } +} + +func TestRetryGuardRefusesEveryCallAfterTheTotalLimit(t *testing.T) { + for _, protocol := range []nodeservices.Protocol{nodeservices.OpenAICompletions, nodeservices.AnthropicMessages} { + var g retryGuard + upstream := &scriptedCalls{cuts: []string{nodeservices.CutMaxDuration}} + p := g.provider(upstream, protocol) + call(t, p) + for range 3 { + assertRefused(t, call(t, p), protocol, nodeservices.CutMaxDuration) + } + if upstream.calls != 1 { + t.Fatalf("refused calls reached upstream: %d", upstream.calls) + } + if g.failure() != nodeservices.CutMaxDuration { + t.Fatalf("failure %q", g.failure()) + } + } +} + +func TestRetryGuardAllowsOneRetryAfterAnIdleCut(t *testing.T) { + var g retryGuard + upstream := &scriptedCalls{cuts: []string{nodeservices.CutIdle, nodeservices.CutIdle}} + p := g.provider(upstream, nodeservices.OpenAICompletions) + call(t, p) + if g.failure() != nodeservices.CutIdle { + t.Fatalf("failure after first idle cut %q", g.failure()) + } + if w := call(t, p); w.Code != 200 { + t.Fatal("the one retry after an idle cut was refused") + } + assertRefused(t, call(t, p), nodeservices.OpenAICompletions, nodeservices.CutIdle) + if upstream.calls != 2 { + t.Fatalf("upstream calls %d, want 2", upstream.calls) + } +} + +func TestRetryGuardCompletedCallClearsIdleCuts(t *testing.T) { + var g retryGuard + upstream := &scriptedCalls{cuts: []string{nodeservices.CutIdle, "", nodeservices.CutIdle, ""}} + p := g.provider(upstream, nodeservices.OpenAICompletions) + call(t, p) + call(t, p) + if g.failure() != "" { + t.Fatalf("completed retry kept failure %q", g.failure()) + } + // A later idle cut in the same turn is a new stall and gets its own retry. + call(t, p) + if w := call(t, p); w.Code != 200 || upstream.calls != 4 { + t.Fatal("new stall after a completed call was refused") + } +} + +func TestRetryGuardIgnoresCallsStartedBeforeTheCut(t *testing.T) { + var g retryGuard + _, cut := g.begin() + _, title := g.begin() // e.g. a title request that ran alongside the cut call + g.finish(cut, nodeservices.CutIdle) + g.finish(title, "") + if g.failure() != nodeservices.CutIdle { + t.Fatalf("concurrent call cleared the cut: %q", g.failure()) + } +} + +func TestCancelledTurnCutAtLimitFailsWithTheLimit(t *testing.T) { + result := turnResult(&nodehistory.Turn{Answer: "Start. ", StopReason: "cancelled"}, nodeservices.CutMaxDuration) + if result.Status != execution.StatusFailed || result.FailureCode != nodeservices.CutMaxDuration || result.AnswerComplete { + t.Fatalf("result %+v", result) + } + if result := turnResult(&nodehistory.Turn{StopReason: "cancelled"}, ""); result.Status != execution.StatusCancelled { + t.Fatalf("plain cancel changed: %+v", result) + } + if result := turnResult(&nodehistory.Turn{Answer: "done", StopReason: "end_turn"}, nodeservices.CutIdle); result.Status != execution.StatusCompleted { + t.Fatalf("completed turn after a recovered cut: %+v", result) + } +} + +func TestRetryGuardCountsOneStallForCallsCutTogether(t *testing.T) { + var g retryGuard + _, title := g.begin() + _, turn := g.begin() + g.finish(turn, nodeservices.CutIdle) + g.finish(title, nodeservices.CutIdle) // stalled alongside: the same stall + stopped, retry := g.begin() + if stopped != "" { + t.Fatal("the one retry after a stall was refused because two calls were cut together") + } + g.finish(retry, nodeservices.CutIdle) + if stopped, _ := g.begin(); stopped != nodeservices.CutIdle { + t.Fatalf("a second stall did not stop the turn: %q", stopped) + } +} diff --git a/v2-server/internal/agentruntime/session.go b/v2-server/internal/agentruntime/session.go index 4c84c561..b11f1b89 100644 --- a/v2-server/internal/agentruntime/session.go +++ b/v2-server/internal/agentruntime/session.go @@ -30,6 +30,8 @@ type turnOutcome struct { type activeTurn struct { id string done chan turnOutcome + // guard limits the Agent's re-sends of model calls cut off at a limit during this turn. + guard retryGuard } type agentSession struct { @@ -418,7 +420,7 @@ func (a *agentSession) observe(message node.Message) (*turnOutcome, bool) { a.requestEnd() return nil, false } - result := turnResult(observation.Turn) + result := turnResult(observation.Turn, call.guard.failure()) if a.publish(execution.Event{Ref: execution.Ref{TurnID: turnID}, Kind: execution.EventTurnEnded, Result: &result}) != nil { a.fail("agent_event_write_failed", errEventStore) a.requestEnd() @@ -489,13 +491,18 @@ func (a *agentSession) flushPreview() { } } -func turnResult(turn *nodehistory.Turn) execution.Result { +// turnResult maps Node's stop reason. Node records every failed turn as cancelled, so a turn +// whose last model call was cut at a limit (cut) fails with that limit as its reason instead. +func turnResult(turn *nodehistory.Turn, cut string) execution.Result { result := execution.Result{Answer: turn.Answer, StopReason: turn.StopReason} switch turn.StopReason { case "end_turn": result.Status, result.AnswerComplete = execution.StatusCompleted, true case "cancelled": result.Status = execution.StatusCancelled + if cut != "" { + result.Status, result.FailureCode = execution.StatusFailed, cut + } case "max_tokens", "max_turn_requests", "refusal": result.Status, result.FailureCode = execution.StatusFailed, "agent_turn_incomplete" default: diff --git a/v2-server/internal/agentruntime/session_test.go b/v2-server/internal/agentruntime/session_test.go index be3a1e98..7d11aefc 100644 --- a/v2-server/internal/agentruntime/session_test.go +++ b/v2-server/internal/agentruntime/session_test.go @@ -33,7 +33,7 @@ func TestIncompleteOrForeignTerminalCannotConfirmStop(t *testing.T) { func TestOnlyNormalTurnEndCanComplete(t *testing.T) { for _, reason := range []string{"cancelled", "max_tokens", "max_turn_requests", "refusal", "unknown private provider error"} { - result := turnResult(&nodehistory.Turn{Answer: "partial", StopReason: reason}) + result := turnResult(&nodehistory.Turn{Answer: "partial", StopReason: reason}, "") if result.Status == execution.StatusCompleted || result.AnswerComplete { t.Fatalf("incomplete turn reported complete: %+v", result) } @@ -41,7 +41,7 @@ func TestOnlyNormalTurnEndCanComplete(t *testing.T) { t.Fatal("unknown reason leaked") } } - result := turnResult(&nodehistory.Turn{Answer: "complete", StopReason: "end_turn"}) + result := turnResult(&nodehistory.Turn{Answer: "complete", StopReason: "end_turn"}, "") if result.Status != execution.StatusCompleted || !result.AnswerComplete || result.Usage != nil { t.Fatalf("normal result: %+v", result) } From 9d01e0f64ddd568e6dac11867c85ba0732c8ed3e Mon Sep 17 00:00:00 2001 From: zhangxinhao Date: Fri, 13 Nov 2026 11:17:13 +0800 Subject: [PATCH 3/5] fix(frontend-v2): say which model limit stopped a turn --- .../src/components/agent-thread/model/history.test.ts | 9 +++++++++ frontend-v2/src/components/agent-thread/model/history.ts | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/frontend-v2/src/components/agent-thread/model/history.test.ts b/frontend-v2/src/components/agent-thread/model/history.test.ts index 7f9ef795..d8b0ba00 100644 --- a/frontend-v2/src/components/agent-thread/model/history.test.ts +++ b/frontend-v2/src/components/agent-thread/model/history.test.ts @@ -200,6 +200,15 @@ describe('conversationOf', () => { expect(turn.error).toContain('history_limit') }) + it('says which model limit stopped a turn that Node only recorded as cancelled', () => { + for (const [code, text] of [['model_max_duration', '超过时长上限'], ['model_idle_timeout', '长时间没有输出']]) { + const failed: ChatResult = { status: 'failed', answer: '', answerComplete: false, stopReason: 'cancelled', failureCode: code } + const [turn] = conversationOf([view({ records: [user(1, 'a'), say(2, 'Start. '), ended(3, 'cancelled')], results: [failed] })]).turns + expect(turn.status).toBe('failed') + expect(turn.error).toContain(text) + } + }) + it('marks where an ended session stops the conversation, as interrupted when chat did not end it', () => { const stopped = conversationOf([view({ id: 's1', records: [user(1, 'a'), ended(2)], results: [done], ended: true, endReason: 'stopped' })]) expect(stopped.markers).toEqual([{ kind: 'session', id: 'session-s1', afterTurn: 1, reason: 'stopped' }]) diff --git a/frontend-v2/src/components/agent-thread/model/history.ts b/frontend-v2/src/components/agent-thread/model/history.ts index f7c0a0d1..0a962284 100644 --- a/frontend-v2/src/components/agent-thread/model/history.ts +++ b/frontend-v2/src/components/agent-thread/model/history.ts @@ -214,8 +214,15 @@ function markFailures(conversation: Conversation, from: number, results: ChatRes return changed ? { ...conversation, turns } : conversation } +/** Model limits that v2-server enforces on one call; the Agent itself only reports a cancel. */ +const limitTexts: Record = { + model_max_duration: '模型请求超过时长上限被切断,这一轮已停止。', + model_idle_timeout: '模型长时间没有输出被切断,这一轮已停止。', +} + function failureText(result: ChatResult): string { if (result.status === 'interrupted') return '服务重启,这一轮被中断。' + if (limitTexts[result.failureCode]) return limitTexts[result.failureCode] if (!result.answerComplete) return `回答没能完整读取${result.failureCode ? `(${result.failureCode})` : ''},这一轮按失败处理。` return `这一轮失败了${result.failureCode ? `(${result.failureCode})` : ''}。` } From f96fe43e2e26a0af10a06981b916084273965fb3 Mon Sep 17 00:00:00 2001 From: zhangxinhao Date: Fri, 13 Nov 2026 13:21:44 +0800 Subject: [PATCH 4/5] fix(v2-server): report a model limit reached before the response header An idle or total limit that expires while waiting for the header still answers the fixed 502 provider_unavailable without aborting, but the route now records the limit. Outcome replaces CutReason and also tells whether the call completed: a 2xx answer that nothing cut. --- v2-server/internal/nodeservices/README.md | 2 +- .../internal/nodeservices/outcome_test.go | 115 ++++++++++++++++++ v2-server/internal/nodeservices/provider.go | 65 +++++++--- 3 files changed, 163 insertions(+), 19 deletions(-) create mode 100644 v2-server/internal/nodeservices/outcome_test.go diff --git a/v2-server/internal/nodeservices/README.md b/v2-server/internal/nodeservices/README.md index 1a75fd67..1cec2e85 100644 --- a/v2-server/internal/nodeservices/README.md +++ b/v2-server/internal/nodeservices/README.md @@ -38,7 +38,7 @@ renew 必须匹配原 execution,过期后不能续期。binding 和 Model 快 `NewRemoteProvider` 转发 OpenAI chat completions,`NewRemoteMessagesProvider` 转发 Anthropic Messages(后端 API key 放在 `x-api-key`,或将配置的 auth token 放在 `Authorization: Bearer …`,只透传 `anthropic-version` / `anthropic-beta`);两者都支持 JSON/SSE,强制使用已选模型,保留成功响应的 usage 等字段。真实 API key 留在后端内存,不转发 Agent token、调用者自定义 header 或上游错误正文; -禁止 HTTP 和重定向。Agent 断开、grant 撤销(`End`、删除授权、`Close`)会取消进行中的上游请求;grant 到期只拒绝新请求和续期,不取消已经开始的请求。单次远端请求有空闲超时(默认 2 分钟,每收到一段上游数据重新计时,也覆盖等待响应头)和总上限(默认 30 分钟),由 `SetLimits` 设置。响应头发出后因上述任一原因中途停止时,provider 只标记这个响应(经 `Unwrap` 穿过包装它的 writer),由 Agent 模型路由在 `Complete` 返回后以 `http.ErrAbortHandler` 中止,Agent 读到的是失败而不是正常结束的流;不经这条路由的进程内调用方(纯模型轮次)只看到被截断的正文。`CompleteObserved` 另返回 `cut` 供调用方记账。标记里带着原因(`model_idle_timeout`、`model_max_duration`、`cancelled`、`interrupted`),包装 provider 的调用方在 `Complete` 返回后用 `CutReason(w)` 读取;`RefuseCall` 在响应头前按协议回一个 422 错误,文字不含 Agent 会据以重发的字样(网络失败、超时、5xx 状态码),Agent 收到后不再重发。 +禁止 HTTP 和重定向。Agent 断开、grant 撤销(`End`、删除授权、`Close`)会取消进行中的上游请求;grant 到期只拒绝新请求和续期,不取消已经开始的请求。单次远端请求有空闲超时(默认 2 分钟,每收到一段上游数据重新计时,也覆盖等待响应头)和总上限(默认 30 分钟),由 `SetLimits` 设置。响应头发出后因上述任一原因中途停止时,provider 只标记这个响应(经 `Unwrap` 穿过包装它的 writer),由 Agent 模型路由在 `Complete` 返回后以 `http.ErrAbortHandler` 中止,Agent 读到的是失败而不是正常结束的流;不经这条路由的进程内调用方(纯模型轮次)只看到被截断的正文。`CompleteObserved` 另返回 `cut` 供调用方记账。路由的响应还记下答复的状态和停止它的模型限值:空闲超时或总上限在响应头之前到时,照旧回固定的 `502 provider_unavailable`、不中止连接,但同样记下限值。包装 provider 的调用方在 `Complete` 返回后用 `Outcome(w)` 读取:`completed` 只在上游 2xx 且没被切断时为真,`limit` 是 `model_idle_timeout` / `model_max_duration` 或空;`RefuseCall` 在响应头前按协议回一个 422 错误,文字不含 Agent 会据以重发的字样(网络失败、超时、5xx 状态码),Agent 收到后不再重发。 不支持运行时模型切换或任何隐式环境变量凭据读取。 ## 验证 diff --git a/v2-server/internal/nodeservices/outcome_test.go b/v2-server/internal/nodeservices/outcome_test.go new file mode 100644 index 00000000..31b5141c --- /dev/null +++ b/v2-server/internal/nodeservices/outcome_test.go @@ -0,0 +1,115 @@ +package nodeservices + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// outcomeProbe stands for a caller that wraps the route's provider (the per-turn retry guard): +// it reads Outcome after the provider returns, through a wrapping writer as request metrics adds. +type outcomeProbe struct { + Provider + seen chan [2]any +} + +func (p outcomeProbe) Complete(w http.ResponseWriter, r *http.Request, m Model) { + p.Provider.Complete(unwrappingWriter{w}, r, m) + completed, limit := Outcome(w) + p.seen <- [2]any{completed, limit} +} + +// stall holds an upstream call open until it is cancelled or the test is long over, so a server +// Close never waits on a handler the client side has already given up on. +func stall(r *http.Request) { + select { + case <-r.Context().Done(): + case <-time.After(3 * time.Second): + } +} + +// callRoute posts one Agent model call through the route and returns its status and body; a body +// cut off mid-stream is returned as far as it arrived. +func callRoute(t *testing.T, p Provider) (int, string) { + t.Helper() + s := testService(t, p) + c := serviceClient(t, s, true) + token, _ := issueToken(t, s, c, "execution") + req, _ := http.NewRequest("POST", s.URL+"/runtime/openai/v1/chat/completions", strings.NewReader(`{"stream":true}`)) + req.Header.Set("Authorization", "Bearer "+token) + resp, err := c.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + data, _ := io.ReadAll(resp.Body) + return resp.StatusCode, string(data) +} + +func TestOutcomeReportsLimitsBeforeAndAfterTheHeaderAndOnlyCountsA2xxAsCompleted(t *testing.T) { + cases := []struct { + name string + upstream http.HandlerFunc + idle time.Duration + total time.Duration + status int + completed bool + limit string + }{ + {"idle before the header", func(w http.ResponseWriter, r *http.Request) { stall(r) }, + 200 * time.Millisecond, time.Minute, 502, false, CutIdle}, + {"idle after the header", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(flushWriter{w}, "data: {}\n\n") + stall(r) + }, 200 * time.Millisecond, time.Minute, 200, false, CutIdle}, + {"total limit on an active stream", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + for { + if _, err := io.WriteString(flushWriter{w}, "data: {}\n\n"); err != nil { + return + } + select { + case <-r.Context().Done(): + return + case <-time.After(50 * time.Millisecond): + } + } + }, 300 * time.Millisecond, 600 * time.Millisecond, 200, false, CutMaxDuration}, + {"provider rejects", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(500) }, + time.Second, time.Minute, 502, false, ""}, + {"normal completion", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, "data: [DONE]\n\n") + }, time.Second, time.Minute, 200, true, ""}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + upstream := httptest.NewTLSServer(c.upstream) + defer upstream.Close() + p, err := NewRemoteProvider(upstream.URL, "backend-key", upstream.Client().Transport) + if err != nil { + t.Fatal(err) + } + if err := p.SetLimits(c.idle, c.total); err != nil { + t.Fatal(err) + } + probe := outcomeProbe{p, make(chan [2]any, 1)} + status, body := callRoute(t, probe) + if status != c.status { + t.Fatalf("status %d body %q, want %d", status, body, c.status) + } + select { + case got := <-probe.seen: + if got[0] != c.completed || got[1] != c.limit { + t.Fatalf("Outcome = %v %q, want %v %q", got[0], got[1], c.completed, c.limit) + } + case <-time.After(5 * time.Second): + t.Fatal("provider never returned") + } + }) + } +} diff --git a/v2-server/internal/nodeservices/provider.go b/v2-server/internal/nodeservices/provider.go index a4cc6330..f6db3fae 100644 --- a/v2-server/internal/nodeservices/provider.go +++ b/v2-server/internal/nodeservices/provider.go @@ -141,6 +141,11 @@ func (p *RemoteProvider) CompleteObserved(w http.ResponseWriter, r *http.Request defer idle.Stop() resp, err := p.client.Do(req) if err != nil { + // A limit reached before the header is still a limit: the answer keeps its fixed error + // code and is not aborted, but the route learns why (see Outcome). + if cause := context.Cause(ctx); errors.Is(cause, errIdleTimeout) || errors.Is(cause, errMaxDuration) { + markLimit(w, cause.Error()) + } fault(w, 502, "provider_unavailable") return 0, false } @@ -268,9 +273,25 @@ func (FixtureProvider) Complete(w http.ResponseWriter, r *http.Request, model Mo type abortableWriter struct { http.ResponseWriter cut bool - // reason is why the provider stopped the response: model_idle_timeout, model_max_duration, - // cancelled or interrupted. - reason string + // status is the status the route answered with; 0 until a header or body is written. + status int + // limit is the model limit that stopped the call, before or after its header: + // model_idle_timeout or model_max_duration. Other cuts leave it empty. + limit string +} + +func (w *abortableWriter) WriteHeader(status int) { + if w.status == 0 { + w.status = status + } + w.ResponseWriter.WriteHeader(status) +} + +func (w *abortableWriter) Write(p []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + return w.ResponseWriter.Write(p) } func (w *abortableWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } @@ -285,7 +306,17 @@ func (w *abortableWriter) Flush() { // http.ResponseController does; without one there is nothing to abort. func markCut(w http.ResponseWriter, reason string) { if a := routeWriter(w); a != nil { - a.cut, a.reason = true, reason + a.cut = true + if reason == CutIdle || reason == CutMaxDuration { + a.limit = reason + } + } +} + +// markLimit records a limit reached before the response header; the response is not aborted. +func markLimit(w http.ResponseWriter, reason string) { + if a := routeWriter(w); a != nil { + a.limit = reason } } @@ -302,29 +333,27 @@ func routeWriter(w http.ResponseWriter) *abortableWriter { } } -// CutReason reports why a provider stopped this Agent model response after its header was sent, -// or "" when it was not cut. Only the two limits are reported by name (CutIdle, CutMaxDuration); -// any other cut is "interrupted" or "cancelled". It asks the first writer in the Unwrap chain -// that knows, which on the Agent model route is the route's own response. -func CutReason(w http.ResponseWriter) string { +// Outcome reports how an Agent model call on this response ended: completed is true only when +// the provider answered 2xx and nothing cut the response; limit names the model limit that +// stopped it (CutIdle or CutMaxDuration), before or after the header, or is "". It asks the +// first writer in the Unwrap chain that knows, which on the Agent model route is the route's +// own response. +func Outcome(w http.ResponseWriter) (completed bool, limit string) { for { - if c, ok := w.(interface{ CutReason() string }); ok { - return c.CutReason() + if o, ok := w.(interface{ Outcome() (bool, string) }); ok { + return o.Outcome() } u, ok := w.(interface{ Unwrap() http.ResponseWriter }) if !ok { - return "" + return false, "" } w = u.Unwrap() } } -// CutReason implements the lookup used by the exported CutReason. -func (w *abortableWriter) CutReason() string { - if !w.cut { - return "" - } - return w.reason +// Outcome implements the lookup used by the exported Outcome. +func (w *abortableWriter) Outcome() (bool, string) { + return w.status >= 200 && w.status < 300 && !w.cut && w.limit == "", w.limit } // Cut reasons that end a model response at a limit. From 00c2fa52e8282f1376d84c5e98b244ccccd35d4d Mon Sep 17 00:00:00 2001 From: zhangxinhao Date: Fri, 13 Nov 2026 13:21:44 +0800 Subject: [PATCH 5/5] fix(v2-server): count header-wait stalls and clear a stall only on a normal completion The retry guard reads Outcome: a limit reached before the header counts like one after it, and only a 2xx call that nothing cut clears an earlier idle cut; provider rejections and unreachable providers leave the count unchanged. --- v2-server/internal/agentruntime/README.md | 2 +- .../internal/agentruntime/retry_guard.go | 27 ++++---- .../internal/agentruntime/retry_guard_test.go | 62 ++++++++++++++----- 3 files changed, 63 insertions(+), 28 deletions(-) diff --git a/v2-server/internal/agentruntime/README.md b/v2-server/internal/agentruntime/README.md index be844157..46f50c57 100644 --- a/v2-server/internal/agentruntime/README.md +++ b/v2-server/internal/agentruntime/README.md @@ -40,7 +40,7 @@ End 等待当前 Node incarnation 的有效 `agent_session_ended`;缺字段、 ## 被切断后的重发 -Agent 把被切断的模型请求当成网络失败原样重发(OpenCode 0.6.11 最多 5 次,Claude Code 0.4.5 十次以上),每次都把半截回答再流一遍。每个模型请求都套上所在轮的守卫:总上限切断后,这一轮之后的请求都在响应头前用 `RefuseCall` 拒绝;空闲超时切断后放行 1 次重发,再被空闲超时切断就同样拒绝。同时在跑、一起被切断的请求(例如标题请求)只算一次;切断之后开始并正常结束的请求清零。Node 把失败的轮次记成 `cancelled`、不带错误文字,所以这一轮最后的请求是被限值切断的时候,结果改为失败,`FailureCode` 就是切断原因,供页面显示。 +Agent 把被切断的模型请求当成网络失败原样重发(OpenCode 0.6.11 最多 5 次,Claude Code 0.4.5 十次以上),每次都把半截回答再流一遍。每个模型请求都套上所在轮的守卫:总上限切断后,这一轮之后的请求都在响应头前用 `RefuseCall` 拒绝;空闲超时切断后放行 1 次重发,再被空闲超时切断就同样拒绝。响应头之前和之后到时的限值一样计。同时在跑、一起被切断的请求(例如标题请求)只算一次;切断之后开始、上游 2xx 且没被切断的请求清零,其它失败(上游拒绝、连不上)不改变计数,Agent 对它们的重发归它自己的策略。Node 把失败的轮次记成 `cancelled`、不带错误文字,所以这一轮最后的请求是被限值切断的时候,结果改为失败,`FailureCode` 就是切断原因,供页面显示。 ## 首次输出前的等待 diff --git a/v2-server/internal/agentruntime/retry_guard.go b/v2-server/internal/agentruntime/retry_guard.go index d1e53551..c7fd00d3 100644 --- a/v2-server/internal/agentruntime/retry_guard.go +++ b/v2-server/internal/agentruntime/retry_guard.go @@ -15,7 +15,7 @@ import ( type retryGuard struct { mu sync.Mutex // idleCuts counts stalls (idle cuts of calls started after the previous cut) since the last - // call that started after a cut and completed. + // call that started after a cut and completed normally. idleCuts int // calls numbers the turn's calls as they start; cutAfter is the number of the newest call // started when the latest cut happened, so only a call started later can clear it. @@ -52,24 +52,28 @@ func (g *retryGuard) begin() (stopped string, call int) { return g.stopped, g.calls } -func (g *retryGuard) finish(call int, reason string) { +// finish records how call ended: completed is true only for a 2xx answer nothing cut, and limit +// is the model limit that stopped it, before or after its header. Any other failure (a provider +// rejection, an unreachable provider, a cancelled call) changes nothing: re-sending those is the +// Agent's own policy. +func (g *retryGuard) finish(call int, completed bool, limit string) { g.mu.Lock() defer g.mu.Unlock() - switch reason { - case nodeservices.CutMaxDuration: - g.last, g.stopped, g.cutAfter = reason, reason, g.calls - case nodeservices.CutIdle: + switch { + case limit == nodeservices.CutMaxDuration: + g.last, g.stopped, g.cutAfter = limit, limit, g.calls + case limit == nodeservices.CutIdle: // Calls that were already running at the previous cut stalled with it (a title request // beside the turn's call): one stall, however many calls it cuts, earns one retry. if call > g.cutAfter { g.idleCuts++ } - g.last, g.cutAfter = reason, g.calls + g.last, g.cutAfter = limit, g.calls if g.idleCuts > 1 { - g.stopped = reason + g.stopped = limit } - case "": - // A call running alongside the cut one (a title request) proves nothing about the retry. + case completed: + // A call running alongside the cut one proves nothing about the retry. if call > g.cutAfter { g.idleCuts, g.last = 0, "" } @@ -89,5 +93,6 @@ func (p guardedProvider) Complete(w http.ResponseWriter, r *http.Request, m node return } p.next.Complete(w, r, m) - p.guard.finish(n, nodeservices.CutReason(w)) + completed, limit := nodeservices.Outcome(w) + p.guard.finish(n, completed, limit) } diff --git a/v2-server/internal/agentruntime/retry_guard_test.go b/v2-server/internal/agentruntime/retry_guard_test.go index 0a63a72d..e5febbf9 100644 --- a/v2-server/internal/agentruntime/retry_guard_test.go +++ b/v2-server/internal/agentruntime/retry_guard_test.go @@ -12,24 +12,37 @@ import ( "github.com/wanglongan587/cloud/v2-server/internal/nodeservices" ) -// cutWriter stands in for the Agent model route's response, which knows why a call was cut. +// cutWriter stands in for the Agent model route's response, which knows how a call ended. type cutWriter struct { *httptest.ResponseRecorder - reason string + limit string } -func (w *cutWriter) CutReason() string { return w.reason } +func (w *cutWriter) Outcome() (bool, string) { + return w.Code >= 200 && w.Code < 300 && w.limit == "", w.limit +} + +// providerFailed is a call that ended without a limit and without a 2xx answer (502 provider_unavailable, +// provider_rejected and the like). +const providerFailed = "provider_failed" -// scriptedCalls ends each call it serves with the next scripted cut reason ("" completes). +// scriptedCalls ends each call it serves with the next scripted outcome: "" completes normally, +// providerFailed answers 502, and a limit name stops the call at that limit. type scriptedCalls struct { - cuts []string - calls int + outcomes []string + calls int } func (s *scriptedCalls) Complete(w http.ResponseWriter, _ *http.Request, _ nodeservices.Model) { - w.WriteHeader(200) - w.(*cutWriter).reason = s.cuts[s.calls] + outcome := s.outcomes[s.calls] s.calls++ + switch outcome { + case providerFailed: + w.WriteHeader(http.StatusBadGateway) + default: + w.WriteHeader(200) + w.(*cutWriter).limit = outcome + } } func call(t *testing.T, p nodeservices.Provider) *cutWriter { @@ -91,7 +104,7 @@ func assertRefused(t *testing.T, w *cutWriter, protocol nodeservices.Protocol, c func TestRetryGuardRefusesEveryCallAfterTheTotalLimit(t *testing.T) { for _, protocol := range []nodeservices.Protocol{nodeservices.OpenAICompletions, nodeservices.AnthropicMessages} { var g retryGuard - upstream := &scriptedCalls{cuts: []string{nodeservices.CutMaxDuration}} + upstream := &scriptedCalls{outcomes: []string{nodeservices.CutMaxDuration}} p := g.provider(upstream, protocol) call(t, p) for range 3 { @@ -108,7 +121,7 @@ func TestRetryGuardRefusesEveryCallAfterTheTotalLimit(t *testing.T) { func TestRetryGuardAllowsOneRetryAfterAnIdleCut(t *testing.T) { var g retryGuard - upstream := &scriptedCalls{cuts: []string{nodeservices.CutIdle, nodeservices.CutIdle}} + upstream := &scriptedCalls{outcomes: []string{nodeservices.CutIdle, nodeservices.CutIdle}} p := g.provider(upstream, nodeservices.OpenAICompletions) call(t, p) if g.failure() != nodeservices.CutIdle { @@ -125,7 +138,7 @@ func TestRetryGuardAllowsOneRetryAfterAnIdleCut(t *testing.T) { func TestRetryGuardCompletedCallClearsIdleCuts(t *testing.T) { var g retryGuard - upstream := &scriptedCalls{cuts: []string{nodeservices.CutIdle, "", nodeservices.CutIdle, ""}} + upstream := &scriptedCalls{outcomes: []string{nodeservices.CutIdle, "", nodeservices.CutIdle, ""}} p := g.provider(upstream, nodeservices.OpenAICompletions) call(t, p) call(t, p) @@ -143,8 +156,8 @@ func TestRetryGuardIgnoresCallsStartedBeforeTheCut(t *testing.T) { var g retryGuard _, cut := g.begin() _, title := g.begin() // e.g. a title request that ran alongside the cut call - g.finish(cut, nodeservices.CutIdle) - g.finish(title, "") + g.finish(cut, false, nodeservices.CutIdle) + g.finish(title, true, "") if g.failure() != nodeservices.CutIdle { t.Fatalf("concurrent call cleared the cut: %q", g.failure()) } @@ -167,14 +180,31 @@ func TestRetryGuardCountsOneStallForCallsCutTogether(t *testing.T) { var g retryGuard _, title := g.begin() _, turn := g.begin() - g.finish(turn, nodeservices.CutIdle) - g.finish(title, nodeservices.CutIdle) // stalled alongside: the same stall + g.finish(turn, false, nodeservices.CutIdle) + g.finish(title, false, nodeservices.CutIdle) // stalled alongside: the same stall stopped, retry := g.begin() if stopped != "" { t.Fatal("the one retry after a stall was refused because two calls were cut together") } - g.finish(retry, nodeservices.CutIdle) + g.finish(retry, false, nodeservices.CutIdle) if stopped, _ := g.begin(); stopped != nodeservices.CutIdle { t.Fatalf("a second stall did not stop the turn: %q", stopped) } } + +func TestRetryGuardOnlyANormalCompletionClearsAStall(t *testing.T) { + var g retryGuard + // idle cut, then the retry fails at the provider (502), then another call stalls + upstream := &scriptedCalls{outcomes: []string{nodeservices.CutIdle, providerFailed, nodeservices.CutIdle}} + p := g.provider(upstream, nodeservices.OpenAICompletions) + call(t, p) + call(t, p) + if g.failure() != nodeservices.CutIdle { + t.Fatalf("a failed call cleared the stall: %q", g.failure()) + } + call(t, p) + assertRefused(t, call(t, p), nodeservices.OpenAICompletions, nodeservices.CutIdle) + if upstream.calls != 3 { + t.Fatalf("upstream calls %d, want 3", upstream.calls) + } +}