From 1240fd70846a854e66e7828b700b37d64653f3c6 Mon Sep 17 00:00:00 2001 From: RSI smoke Date: Tue, 13 Oct 2026 20:53:40 -0600 Subject: [PATCH] Compare pilot per-request cap against visible tokens, not native (#11) Native completion counts include uncapped provider reasoning tokens (observed 884 reasoning of 1033 native while visible was 122), so the completion-exceeded-request-cap guard false-tripped and halted two dev-collection pilots. Budget accounting still uses native tokens. --- scripts/monitor-pilot/live-runner.ts | 7 ++++- src/monitoring/__tests__/live-runner.test.ts | 28 ++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/scripts/monitor-pilot/live-runner.ts b/scripts/monitor-pilot/live-runner.ts index c1346e0..9f814eb 100644 --- a/scripts/monitor-pilot/live-runner.ts +++ b/scripts/monitor-pilot/live-runner.ts @@ -452,7 +452,12 @@ async function runLiveTrajectory(options: LiveTrajectoryOptions): Promise<{ if (!metadata) throw new LivePilotGuardError("authoritative-billing-record-unavailable") assertPinnedIdentity(response, metadata) const usage = assertUsage(response, metadata) - if (usage.completion > request.maxTokens!) throw new LivePilotGuardError("completion-exceeded-request-cap") + if (metadata.tokensCompletion !== undefined && !isSafeInteger(metadata.tokensCompletion)) { + throw new LivePilotGuardError("billing-token-usage-missing") + } + // maxTokens constrains visible output; native counts include uncapped reasoning. + const visibleCompletion = metadata.tokensCompletion ?? usage.completion + if (visibleCompletion > request.maxTokens!) throw new LivePilotGuardError("completion-exceeded-request-cap") if (inputTokens + outputTokens + usage.prompt + usage.completion > LIVE_BUDGET.maxTokens) throw new LivePilotGuardError("measured-token-cap-exceeded") inputTokens += usage.prompt outputTokens += usage.completion diff --git a/src/monitoring/__tests__/live-runner.test.ts b/src/monitoring/__tests__/live-runner.test.ts index c901a55..79029cb 100644 --- a/src/monitoring/__tests__/live-runner.test.ts +++ b/src/monitoring/__tests__/live-runner.test.ts @@ -429,6 +429,33 @@ async function testCompletionOverrunIsJournaledBeforeAnyToolDispatch(): Promise< } finally { await fs.rm(directory, { recursive: true, force: true }) } } +function reasoningHeavyClient(): MockBillingClient { + const done = { name: "attempt_completion", + args: JSON.stringify({ result: "done" }) } + const usage = { promptTokens: 710, completionTokens: 1033, + totalTokens: 1743 } + const billing = { totalCostUsd: 0.0001, tokensPrompt: 710, + tokensCompletion: 122, nativeTokensPrompt: 710, + nativeTokensCompletion: 1033, nativeTokensReasoning: 884 } + return new MockBillingClient([done], LIVE_MODEL, "Meta", + { responseUsage: usage, metadata: billing }) +} + +async function testReasoningHeavyNativeOverrunIsAccepted(): Promise { + const directory = await scratch() + try { + const client = reasoningHeavyClient() + const options = { outputDir: directory, client, + maxOutputTokens: 1024 } + const summary = await collectLiveDevelopment(options) + assert.equal(summary.abortedAttempts, 0) + assert.equal(summary.completedRuns, 32) + assert.equal(summary.billing.unacceptedBillingRecords, 0) + assert.equal(summary.billing.completionTokens, 32 * 1033, + "budget accounting still uses native tokens") + } finally { await fs.rm(directory, { recursive: true, force: true }) } +} + async function testHeldoutCannotStartWithoutQualifiedDevelopment(): Promise { const directory = await scratch() try { @@ -509,6 +536,7 @@ await testUncertainRequestIsNotAutomaticallyReplayed() await testCopiedGlobalLedgerEpochsDoNotBlockResume() await testKnownPerRunAdmissionStopCompletesAndSettlesTrajectory() await testCompletionOverrunIsJournaledBeforeAnyToolDispatch() +await testReasoningHeavyNativeOverrunIsAccepted() await testHeldoutCannotStartWithoutQualifiedDevelopment() await testHeldoutRejectsDifferentFrozenOutputCapBeforeInference() await testStorageRootSymlinkIsRejectedBeforeInference()