diff --git a/scripts/monitor-pilot/live-runner.ts b/scripts/monitor-pilot/live-runner.ts index c1346e0..9f814eb 100644 --- a/scripts/monitor-pilot/live-runner.ts +++ b/scripts/monitor-pilot/live-runner.ts @@ -452,7 +452,12 @@ async function runLiveTrajectory(options: LiveTrajectoryOptions): Promise<{ if (!metadata) throw new LivePilotGuardError("authoritative-billing-record-unavailable") assertPinnedIdentity(response, metadata) const usage = assertUsage(response, metadata) - if (usage.completion > request.maxTokens!) throw new LivePilotGuardError("completion-exceeded-request-cap") + if (metadata.tokensCompletion !== undefined && !isSafeInteger(metadata.tokensCompletion)) { + throw new LivePilotGuardError("billing-token-usage-missing") + } + // maxTokens constrains visible output; native counts include uncapped reasoning. + const visibleCompletion = metadata.tokensCompletion ?? usage.completion + if (visibleCompletion > request.maxTokens!) throw new LivePilotGuardError("completion-exceeded-request-cap") if (inputTokens + outputTokens + usage.prompt + usage.completion > LIVE_BUDGET.maxTokens) throw new LivePilotGuardError("measured-token-cap-exceeded") inputTokens += usage.prompt outputTokens += usage.completion diff --git a/src/monitoring/__tests__/live-runner.test.ts b/src/monitoring/__tests__/live-runner.test.ts index c901a55..79029cb 100644 --- a/src/monitoring/__tests__/live-runner.test.ts +++ b/src/monitoring/__tests__/live-runner.test.ts @@ -429,6 +429,33 @@ async function testCompletionOverrunIsJournaledBeforeAnyToolDispatch(): Promise< } finally { await fs.rm(directory, { recursive: true, force: true }) } } +function reasoningHeavyClient(): MockBillingClient { + const done = { name: "attempt_completion", + args: JSON.stringify({ result: "done" }) } + const usage = { promptTokens: 710, completionTokens: 1033, + totalTokens: 1743 } + const billing = { totalCostUsd: 0.0001, tokensPrompt: 710, + tokensCompletion: 122, nativeTokensPrompt: 710, + nativeTokensCompletion: 1033, nativeTokensReasoning: 884 } + return new MockBillingClient([done], LIVE_MODEL, "Meta", + { responseUsage: usage, metadata: billing }) +} + +async function testReasoningHeavyNativeOverrunIsAccepted(): Promise { + const directory = await scratch() + try { + const client = reasoningHeavyClient() + const options = { outputDir: directory, client, + maxOutputTokens: 1024 } + const summary = await collectLiveDevelopment(options) + assert.equal(summary.abortedAttempts, 0) + assert.equal(summary.completedRuns, 32) + assert.equal(summary.billing.unacceptedBillingRecords, 0) + assert.equal(summary.billing.completionTokens, 32 * 1033, + "budget accounting still uses native tokens") + } finally { await fs.rm(directory, { recursive: true, force: true }) } +} + async function testHeldoutCannotStartWithoutQualifiedDevelopment(): Promise { const directory = await scratch() try { @@ -509,6 +536,7 @@ await testUncertainRequestIsNotAutomaticallyReplayed() await testCopiedGlobalLedgerEpochsDoNotBlockResume() await testKnownPerRunAdmissionStopCompletesAndSettlesTrajectory() await testCompletionOverrunIsJournaledBeforeAnyToolDispatch() +await testReasoningHeavyNativeOverrunIsAccepted() await testHeldoutCannotStartWithoutQualifiedDevelopment() await testHeldoutRejectsDifferentFrozenOutputCapBeforeInference() await testStorageRootSymlinkIsRejectedBeforeInference()