diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index aac4beeb..a588e165 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -809,6 +809,8 @@ testsprite test run --env local-dev --timeout 1800 **Login and execution.** Tunnels serve frontend tests. An explicit loopback target on a backend test fails client validation (exit 5); single backend tests keep their previous cloud path; tunnel batches skip only backend tests whose own selected environment is loopback, and include their IDs in `skipped`. Public-environment backend tests dispatch normally. The project environment's username/password are passed to the cloud agent, which logs in inline through the tunnel. OTP environments are refused **before charge** with exit 6 (`tunnel-otp-auth-unsupported`). V3 `--local` runs always use the agent path, never saved-code replay, and **do not overwrite the test's saved code**. Saved loopback environments also open tunnels for reruns. Rerun and test-list tunnel batches use one `--timeout` deadline, including capacity waits. Explicit public-environment sign-in against a local origin is announced once on stderr. Rerun/test-list responses must echo the tunnel client id; if an old server strips it, the CLI attempts to cancel what started and exits with a clear error. +When explicit test IDs mix a saved-loopback frontend with a public-environment backend, the backend runs without tunnel routing. If its run row remains non-terminal, the CLI also reads the backend test result and accepts a terminal verdict only when its run ID matches (or, for legacy results without a run ID, its finish time is no earlier than the trigger's enqueue time). Invalid legacy finish timestamps or invalid provided enqueue timestamps leave normal polling in place. This keeps JSON summaries, JUnit reports, and exit codes aligned with completed backend results instead of timing them out. Unrelated, incomplete, or unavailable results leave normal run polling in place within the original timeout. + **Timeout, cancellation, and refunds.** When a single owned `--local` run stops waiting before a terminal result, the CLI cancels it by default and closes its tunnel. This includes `--timeout` (exit 7), Ctrl-C (exit 130), and non-terminal polling/tunnel failures. In a local batch, each test has its own timeout measured from its trigger: a timed-out run is cancelled on an owned tunnel, the other tests continue, and the tunnel closes after the batch settles. Cancellation can race completion or fail; read the reported outcome instead of assuming it succeeded. **A run cancelled before it finished is refunded.** Cancelling an already-finished run does not replace its result or refund it. After an owned local timeout, start a **new** run with `testsprite test run --env --timeout 1800`; `test wait` cannot restore the closed tunnel. `--no-cancel-on-interrupt` detaches instead, but does not keep an owned tunnel alive. A borrowed `--tunnel-client` run is not cancelled on Ctrl-C/SIGTERM: the borrower detaches, never closes or deletes the adopted tunnel, and `test wait ` can resume polling while the owner keeps it alive. If liveness reports that the owner is gone while the run is non-terminal, the borrower cancels **its own run** exactly as an owned doomed run does. The message names the run id and the observed result (`cancelled`, `already finished`, or `skipped`), then points to the run read before a retry. A run cancelled before it finished is refunded. `--no-cancel-on-interrupt` skips the owner-gone cancel too, leaving the run executing and billable without a working tunnel. diff --git a/package-lock.json b/package-lock.json index 1be0c9c9..e3f82585 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3450,9 +3450,9 @@ } }, "node_modules/undici": { - "version": "7.29.1", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.29.1.tgz", - "integrity": "sha512-RYONW2MeafgYlkVOKYKkA/Ag7BmXqgIWCa8t1m0JcxrQg9pI9lEqRhAOruOBCbAohOa/gkCF+iPi9hrgvTzu6Q==", + "version": "7.30.1", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.30.1.tgz", + "integrity": "sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==", "license": "MIT", "engines": { "node": ">=20.18.1" diff --git a/src/commands/test.local-batch.spec.ts b/src/commands/test.local-batch.spec.ts index 0758ce49..85d0baec 100644 --- a/src/commands/test.local-batch.spec.ts +++ b/src/commands/test.local-batch.spec.ts @@ -9,7 +9,7 @@ import { sanitizeTelemetryExtras, takeTelemetryExtras } from '../lib/telemetry.j import type { TunnelClientOptions } from '../vendor/tunnel-client/index.js'; import { ErrCode } from '../vendor/tunnel-client/index.js'; import type { RunResponse } from '../lib/runs.types.js'; -import { createTestCommand } from './test.js'; +import { createTestCommand, type CliLatestResult } from './test.js'; type Call = { method: string; @@ -18,6 +18,10 @@ type Call = { idempotencyKey?: string | null; }; +/** + * Build an offline batch-run CLI fixture with injectable HTTP, tunnel, and timer + * behavior, exposing the command runner, captured requests and output, and test IDs. + */ function fixture( ids: string[], options: { @@ -160,6 +164,10 @@ function fixture( return { ids, calls, stdout, stderr, terminal, clientId, targetUrl, run }; } +/** + * Return a running response for testId with no finish time and an incomplete + * step, so polling fixtures can control when a terminal verdict becomes available. + */ function runningResponse(testId: string): Response { return new Response( JSON.stringify({ @@ -186,6 +194,112 @@ function runningResponse(testId: string): Response { ); } +/** Model a local frontend and a public-environment backend with an unfinalized run row. */ +function mixedBackendFixture( + result: Partial = {}, + options: { + finalRunStatus?: 'passed' | 'failed'; + terminalImmediately?: boolean; + resultUnavailable?: boolean; + remoteType?: 'frontend' | 'backend'; + backendEnqueuedAt?: string; + } = {}, +) { + let backendPolls = 0; + /** Return API-shaped JSON so the fixture exercises the client's response decoding. */ + const json = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } }); + return fixture(['fe', 'be'], { + sleep: options.finalRunStatus ? async () => {} : ms => new Promise(r => setTimeout(r, ms)), + /** + * Override trigger, run, and result responses for the mixed-backend scenario. + * Return undefined for requests that should use the base fixture's responses. + */ + onCall: async call => { + if (call.method === 'POST' && call.url.endsWith('/tests/be/runs')) + return json({ + runId: 'run_be', + status: 'queued', + enqueuedAt: options.backendEnqueuedAt ?? '2026-09-22T00:00:00.000Z', + codeVersion: 'v1', + targetUrl: 'https://example.com', + }); + if (call.method === 'POST' && call.url.endsWith('/tests/fe/runs')) + return json({ + runId: 'run_fe', + status: 'queued', + enqueuedAt: '2026-09-22T00:00:01.000Z', + codeVersion: 'v1', + targetUrl: 'http://127.0.0.1:5173', + }); + if (call.method !== 'GET') return undefined; + if (call.url.endsWith('/tests/be')) + return json({ + id: 'be', + name: 'Backend checkout', + type: options.remoteType ?? 'backend', + projectId: 'R', + }); + if (call.url.endsWith('/projects/R/env')) + return json({ + environments: [ + { name: 'default', url: 'https://example.com', isDefault: true, isTemporary: false }, + ], + }); + if (call.url.includes('/runs/run_be')) { + const run = (await runningResponse('be').json()) as RunResponse; + backendPolls++; + const status = + options.terminalImmediately || backendPolls > 1 ? options.finalRunStatus : undefined; + return json({ + ...run, + projectId: 'R', + targetUrl: null, + ...(status + ? { + status, + finishedAt: '2026-09-22T00:00:02.000Z', + stepSummary: { + total: 1, + completed: 1, + passedCount: status === 'passed' ? 1 : 0, + failedCount: status === 'failed' ? 1 : 0, + }, + } + : {}), + }); + } + if (call.url.endsWith('/tests/be/result')) { + if (options.resultUnavailable) + return json( + { error: { code: 'NOT_FOUND', message: 'Not ready', nextAction: '', requestId: 'r1' } }, + 404, + ); + return json({ + testId: 'be', + status: 'passed', + startedAt: '2026-09-22T00:00:01.000Z', + finishedAt: '2026-09-22T00:00:02.000Z', + runIdIfAvailable: 'run_be', + snapshotId: 'snapshot_be', + codeVersion: 'v1', + targetUrl: null, + failedStepIndex: result.status === 'failed' ? 0 : null, + failureKind: result.status === 'failed' ? 'assertion' : null, + videoUrl: null, + failureAnalysisUrl: null, + verdict: + result.status === 'running' ? null : result.status === 'failed' ? 'failed' : 'passed', + executionStatus: result.status === 'running' ? 'running' : 'completed', + summary: result.status ?? 'passed', + ...result, + } satisfies CliLatestResult); + } + return undefined; + }, + }); +} + afterEach(() => { vi.useRealTimers(); vi.unstubAllEnvs(); @@ -193,6 +307,159 @@ afterEach(() => { }); describe('test run local batch', () => { + describe('mixed backend verdict fallback', () => { + it.each([ + { status: 'passed', borrowed: false }, + { status: 'failed', borrowed: false }, + { status: 'passed', borrowed: true }, + { status: 'failed', borrowed: true }, + ] as const)( + 'resolves $status with borrowed=$borrowed and writes accurate reports', + async ({ status, borrowed }) => { + const f = mixedBackendFixture({ status }); + const dir = mkdtempSync(join(tmpdir(), 'mixed-backend-report-')); + const summaryFile = join(dir, 'summary.json'); + const reportFile = join(dir, 'report.xml'); + const error = await f + .run( + 'fe', + 'be', + '--wait', + '--skip-preflight', + '--timeout', + '1', + '--summary-file', + summaryFile, + '--report', + 'junit', + '--report-file', + reportFile, + ...(borrowed ? ['--tunnel-client', f.clientId] : []), + ) + .then(() => undefined) + .catch(e => e); + if (status === 'passed') expect(error).toBeUndefined(); + else expect(error).toMatchObject({ exitCode: 1 }); + const payload = JSON.parse(f.stdout.join('')); + expect(payload.results).toEqual([ + expect.objectContaining({ testId: 'fe', runId: 'run_fe', status: 'passed' }), + expect.objectContaining({ + testId: 'be', + runId: 'run_be', + status, + testTitle: 'Backend checkout', + projectId: 'R', + }), + ]); + // eslint-disable-next-line security/detect-non-literal-fs-filename -- reads summary.json from this test's own mkdtempSync-created directory; neither path component comes from user input. + const summary = JSON.parse(readFileSync(summaryFile, 'utf8')); + expect(summary).toMatchObject({ + total: 2, + passed: status === 'passed' ? 2 : 1, + failed: status === 'failed' ? 1 : 0, + timedOut: 0, + }); + // eslint-disable-next-line security/detect-non-literal-fs-filename -- reads report.xml from this test's own mkdtempSync-created directory; neither path component comes from user input. + const xml = readFileSync(reportFile, 'utf8'); + expect(xml).toContain('name="Backend checkout" testId="be" runId="run_be" time="1"'); + expect(xml).not.toContain('type="timeout"'); + if (status === 'failed') expect(xml).toContain(''); + expect(f.calls.filter(c => c.url.endsWith('/tests/be/result'))).toHaveLength(1); + expect( + f.calls.find(c => c.method === 'POST' && c.url.endsWith('/tests/be/runs'))?.body, + ).toEqual({ source: 'cli' }); + expect( + f.calls.find(c => c.method === 'POST' && c.url.endsWith('/tests/fe/runs'))?.body, + ).toMatchObject({ tunnelClientId: f.clientId, targetUrl: f.targetUrl }); + expect(f.calls.some(c => c.url.endsWith('/cancel'))).toBe(false); + expect(f.calls.filter(c => c.method === 'DELETE')).toHaveLength(borrowed ? 0 : 1); + expect(f.calls.filter(c => c.method === 'GET' && c.url.includes('/tunnel/'))).toHaveLength( + 0, + ); + }, + ); + + it('resolves a legacy result with no run ID only when it is fresh enough', async () => { + const f = mixedBackendFixture({ + runIdIfAvailable: null, + startedAt: '2026-09-22T00:00:00.100Z', + finishedAt: '2026-09-22T00:00:00.500Z', + }); + await f.run( + 'be', + 'fe', + '--skip-preflight', + '--timeout', + '1', + '--max-concurrency', + '1', + '--text-output', + ); + expect(f.stdout.join('')).toContain('passed'); + expect(f.stderr.join('\n')).toContain('2/2 passed'); + expect(f.calls.filter(c => c.url.endsWith('/tests/be/result'))).toHaveLength(1); + }); + + it.each([ + { runIdIfAvailable: 'run_other' }, + { runIdIfAvailable: null, finishedAt: '2026-09-21T23:59:59.000Z' }, + { runIdIfAvailable: null, finishedAt: 'not-a-date' }, + { status: 'running', finishedAt: null }, + ] satisfies Array>)( + 'keeps polling instead of accepting unrelated or incomplete result %j', + async result => { + const f = mixedBackendFixture(result, { finalRunStatus: 'failed' }); + await expect(f.run('fe', 'be', '--skip-preflight', '--timeout', '1')).rejects.toMatchObject( + { exitCode: 1 }, + ); + expect(JSON.parse(f.stdout.join('')).results[1]).toMatchObject({ + testId: 'be', + runId: 'run_be', + status: 'failed', + }); + expect(f.calls.filter(c => c.url.endsWith('/tests/be/result'))).toHaveLength(1); + expect(f.calls.filter(c => c.url.includes('/runs/run_be'))).toHaveLength(2); + }, + ); + + it.each(['not-a-date', ''])( + 'keeps polling when the backend enqueue timestamp is invalid: %j', + async backendEnqueuedAt => { + const f = mixedBackendFixture( + { runIdIfAvailable: null }, + { finalRunStatus: 'failed', backendEnqueuedAt }, + ); + await expect(f.run('fe', 'be', '--skip-preflight', '--timeout', '1')).rejects.toMatchObject( + { exitCode: 1 }, + ); + expect(JSON.parse(f.stdout.join('')).results[1]).toMatchObject({ + testId: 'be', + runId: 'run_be', + status: 'failed', + }); + expect(f.calls.filter(c => c.url.endsWith('/tests/be/result'))).toHaveLength(1); + expect(f.calls.filter(c => c.url.includes('/runs/run_be'))).toHaveLength(2); + }, + ); + + it('keeps the run-row verdict when the result endpoint is unavailable', async () => { + const f = mixedBackendFixture({}, { finalRunStatus: 'passed', resultUnavailable: true }); + await f.run('fe', 'be', '--skip-preflight', '--timeout', '1'); + expect(JSON.parse(f.stdout.join('')).summary.passed).toBe(2); + expect(f.calls.filter(c => c.url.endsWith('/tests/be/result'))).toHaveLength(1); + }); + + it.each([ + { finalRunStatus: 'passed', terminalImmediately: true }, + { finalRunStatus: 'passed', remoteType: 'frontend' }, + ] as const)('does not fetch an alternate result for the control %j', async options => { + const f = mixedBackendFixture({}, options); + await f.run('fe', 'be', '--skip-preflight', '--timeout', '1'); + expect(JSON.parse(f.stdout.join('')).summary.passed).toBe(2); + expect(f.calls.filter(c => c.url.endsWith('/result'))).toHaveLength(0); + }); + }); + it('keeps the paused-workspace envelope when every local trigger is refused', async () => { const f = fixture(['a', 'b'], { onCall: call => diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 97ce59e1..7903da45 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -1796,6 +1796,29 @@ describe('backendResultIsForThisRun — auto-resume stale-verdict floor (finding ), ).toBe(true); }); + + it.each([ + { finishedAt: 'not-a-date', notBefore: inFlightCreatedAt, expected: false }, + { finishedAt: 'not-a-date', notBefore: undefined, expected: false }, + { finishedAt: inFlightCreatedAt, notBefore: 'not-a-date', expected: false }, + { finishedAt: inFlightCreatedAt, notBefore: '', expected: false }, + { finishedAt: inFlightCreatedAt, notBefore: undefined, expected: true }, + { finishedAt: inFlightCreatedAt, notBefore: inFlightCreatedAt, expected: true }, + ])('validates legacy timestamp correlation: %j', ({ finishedAt, notBefore, expected }) => { + expect( + backendResultIsForThisRun({ ...staleResult, finishedAt }, 'run_inflight', notBefore), + ).toBe(expected); + }); + + it('keeps exact run ID correlation independent of malformed legacy timestamps', () => { + expect( + backendResultIsForThisRun( + { ...staleResult, runIdIfAvailable: 'run_inflight', finishedAt: 'not-a-date' }, + 'run_inflight', + 'not-a-date', + ), + ).toBe(true); + }); }); describe('backend wait fallback — testTitle overlay used for CI report titles', () => { diff --git a/src/commands/test.ts b/src/commands/test.ts index 31f3b107..78fd5f70 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -7219,7 +7219,8 @@ export function backendResultToRunResponse( * just-finished prior run could still be showing. Gate: * - result names our runId → accept (strongest signal); * - result names a different runId → reject (a different run); - * - result has no runId (legacy) → accept iff finishedAt >= notBefore. + * - result has no runId (legacy) → require a valid finishedAt, and when + * notBefore is supplied, require a valid floor and finishedAt >= notBefore. */ export function backendResultIsForThisRun( result: CliLatestResult, @@ -7231,10 +7232,11 @@ export function backendResultIsForThisRun( return result.runIdIfAvailable === runId; } if (!result.finishedAt) return false; - if (!notBefore) return true; const finished = Date.parse(result.finishedAt); + if (Number.isNaN(finished)) return false; + if (notBefore === undefined) return true; const floor = Date.parse(notBefore); - if (Number.isNaN(finished) || Number.isNaN(floor)) return true; + if (Number.isNaN(floor)) return false; return finished >= floor; } @@ -11250,6 +11252,7 @@ export async function runTestRunLocalBatch( ) stderrFn(`idempotency-key: ${idempotencyBase}`); + /** Dispatch and poll one member within its own deadline, retaining per-run fallback state. */ const runOne = async (index: number) => { const row = results[index]!; let triggerStartedAtMs: number; @@ -11360,6 +11363,28 @@ export async function runTestRunLocalBatch( const startMs = triggerStartedAtMs; const remainingSeconds = (startMs + opts.timeoutSeconds * 1000 - Date.now()) / 1000; if (remainingSeconds <= 0) throw new TimeoutError(row.runId!, opts.timeoutSeconds); + // Public-environment backend members share this pool with local frontend + // runs, but their run rows may never finalize. Reuse the ordinary wait's + // correlated fallback without adding result probes to tunnel-bound runs. + const resolveBackendAlternate = targets.has(row.testId) + ? undefined + : makeBackendWaitFallback({ + client, + resolveTestId: () => row.testId, + resolveNotBefore: () => response.enqueuedAt, + }); + const resolveAlternate = + targets.has(row.testId) && checkBorrowedLiveness + ? /** + * Check the borrowed frontend tunnel using the poller's abort signal. + * Return null after a successful check so run polling continues; + * throw when the tunnel owner is unavailable. + */ + async (_run: RunResponse, _elapsed: number, signal: AbortSignal) => { + await checkBorrowedLiveness!(signal); + return null; + } + : resolveBackendAlternate; const finalRun = await pollRunUntilTerminal(client, row.runId!, { timeoutSeconds: remainingSeconds, sleep: deps.sleep, @@ -11373,14 +11398,7 @@ export async function runTestRunLocalBatch( } ticker.update(formatRunProgressLine(run, elapsedMs, `(${row.testId})`)); }, - ...(targets.has(row.testId) && checkBorrowedLiveness - ? { - resolveAlternate: async (_run, _elapsed, signal) => { - await checkBorrowedLiveness!(signal); - return null; - }, - } - : {}), + resolveAlternate, }); row.status = finalRun.status as LocalBatchResult['status']; unsettled.delete(row);