diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index da696075..dca6a803 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -410,6 +410,45 @@ testsprite test create --plan-from ./checkout.plan.json --dry-run --output json Bulk-create frontend tests from a JSONL plan-steps file (or a directory of plan files with `--plan-from-dir`). Optional `--run --max-concurrency ` fans out triggers. Without `--wait`, each run is dispatched (`status: "queued"`) and the command exits 0 when every trigger is accepted — mirroring single `test run` without `--wait`; a trigger error still exits non-zero. With `--wait`, it polls every run to terminal and exits non-zero if any run does not pass. +**JSON output with `--run`.** The top-level `results` array remains the per-test run results. A separate additive `creation` object preserves the original batch-create receipt, including failures for individual specs that were not created: + +```json +{ + "results": [ + { + "testId": "test_xxxxxxxx", + "runId": "run_xxxxxxxx", + "status": "passed" + } + ], + "creation": { + "results": [ + { + "specIndex": 0, + "testId": "test_xxxxxxxx", + "status": "created", + "dashboardUrl": "https://..." + }, + { + "specIndex": 1, + "status": "validation_error", + "error": { + "code": "VALIDATION_ERROR", + "message": "Invalid plan" + } + } + ], + "summary": { + "total": 2, + "created": 1, + "failed": 1 + } + } +} +``` + +The `creation` object has the same `{ results, summary }` shape as create-only JSON output. Each creation result retains its `specIndex`, status, optional error and optional `dashboardUrl`; the dashboard URL is omitted when unavailable or explicitly suppressed. This receipt is also included in interrupted/partial output and `--dry-run` JSON envelopes. Existing run-result rows, text output and exit codes are unchanged. + ```bash testsprite test create-batch --plans ./plans.jsonl --run --max-concurrency 4 --output json testsprite test create-batch --plan-from-dir ./plans/ --dry-run --output json diff --git a/src/commands/test.create-batch-run.spec.ts b/src/commands/test.create-batch-run.spec.ts index be439ca8..bb3d2fad 100644 --- a/src/commands/test.create-batch-run.spec.ts +++ b/src/commands/test.create-batch-run.spec.ts @@ -190,7 +190,19 @@ describe('runCreateBatch --run --wait: happy path', () => { const fetchImpl = makeFetch(url => { // Batch create if (url.includes('/tests/batch')) { - return { body: makeBatchCreateResponse(testIds) }; + return { + body: { + results: testIds.map((testId, specIndex) => ({ + specIndex, + testId, + status: 'created' as const, + ...(specIndex === 0 + ? { dashboardUrl: 'https://portal.testsprite.com/tests/test_a1' } + : {}), + })), + summary: { total: testIds.length, created: testIds.length, failed: 0 }, + }, + }; } // Trigger POST /tests/{id}/runs const triggerMatch = /\/tests\/(test_[a-z0-9]+)\/runs$/.exec(url); @@ -238,8 +250,21 @@ describe('runCreateBatch --run --wait: happy path', () => { ); // JSON output is the batch-run envelope - const printed = JSON.parse(stdout.join('')) as { results: unknown[] }; + const printed = JSON.parse(stdout.join('')) as { + results: unknown[]; + creation: { + results: Array<{ testId?: string; status: string; dashboardUrl?: string }>; + summary: { total: number; created: number; failed: number }; + }; + }; expect(printed.results).toHaveLength(3); + expect(printed.creation.summary).toEqual({ total: 3, created: 3, failed: 0 }); + expect(printed.creation.results).toHaveLength(3); + expect(printed.creation.results[0]).toMatchObject({ + testId: 'test_a1', + status: 'created', + dashboardUrl: 'https://portal.testsprite.com/tests/test_a1', + }); const results = printed.results as Array<{ testId: string; status: string; runId: string }>; expect(results.every(r => r.status === 'passed')).toBe(true); expect(results.map(r => r.testId).sort()).toEqual(['test_a1', 'test_a2', 'test_a3'].sort()); @@ -417,10 +442,16 @@ describe('runCreateBatch --run (no --wait): exit-code contract', () => { const printed = JSON.parse(stdout.join('')) as { results: Array<{ testId: string; status: string; error?: unknown }>; + creation: { + results: Array<{ specIndex: number; testId?: string; status: string }>; + summary: { total: number; created: number; failed: number }; + }; }; expect(printed.results).toHaveLength(3); expect(printed.results.every(r => r.status === 'queued')).toBe(true); expect(printed.results.every(r => r.error === undefined)).toBe(true); + expect(printed.creation.results).toHaveLength(3); + expect(printed.creation.summary).toEqual({ total: 3, created: 3, failed: 0 }); }); it('all triggers succeed (queued) → text summary reports "3/3 triggered", exit 0 (no "0/N passed")', async () => { @@ -1068,6 +1099,89 @@ describe('runCreateBatch --run --wait: CONFLICT auto-resume', () => { }); }); +// --------------------------------------------------------------------------- +// Partial creation success — the receipt must survive normal completion with +// and without --wait, independently of the run-result status. +// --------------------------------------------------------------------------- + +describe('runCreateBatch --run: partial creation receipt', () => { + it.each([true, false])( + 'preserves created and validation_error rows on normal completion (wait=%s)', + async wait => { + const { credentialsPath } = makeCreds(); + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC]); + const creationResponse = { + results: [ + { specIndex: 0, testId: 'test_partial_created', status: 'created' }, + { + specIndex: 1, + status: 'validation_error', + error: { code: 'VALIDATION_ERROR', message: 'invalid plan', field: 'planSteps' }, + }, + ], + summary: { total: 2, created: 1, failed: 1 }, + }; + + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch')) return { body: creationResponse }; + if (url.endsWith('/tests/test_partial_created/runs')) { + return { body: makeTriggerResponse('test_partial_created', 'run_partial_created') }; + } + if (url.includes('/runs/run_partial_created')) { + return { body: makePassedRun('test_partial_created', 'run_partial_created') }; + } + return { + status: 404, + body: { + error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' }, + }, + }; + }); + + const stdout: string[] = []; + await runCreateBatch( + { + profile: 'default', + output: 'json', + debug: false, + dryRun: false, + plans: plansFile, + run: true, + wait, + maxConcurrency: 1, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + sleep: instantSleep, + }, + ); + + const printed = JSON.parse(stdout.join('')) as { + results: Array<{ testId: string; status: string }>; + creation: { + results: Array<{ specIndex: number; testId?: string; status: string; error?: unknown }>; + summary: { total: number; created: number; failed: number }; + }; + }; + expect(printed.results).toHaveLength(1); + expect(printed.results[0]?.status).toBe(wait ? 'passed' : 'queued'); + expect(printed.creation.summary).toEqual({ total: 2, created: 1, failed: 1 }); + expect(printed.creation.results).toMatchObject([ + { specIndex: 0, testId: 'test_partial_created', status: 'created' }, + { + specIndex: 1, + status: 'validation_error', + error: { code: 'VALIDATION_ERROR', field: 'planSteps' }, + }, + ]); + }, + ); +}); + // --------------------------------------------------------------------------- // --dry-run mode // --------------------------------------------------------------------------- @@ -1136,9 +1250,51 @@ describe('runCreateBatch --run --dry-run', () => { expect(envelope.dryRun).toBe(true); expect(Array.isArray(envelope.testIds)).toBe(true); expect((envelope.testIds as string[]).sort()).toEqual(testIds.sort()); - // results array is present + // Both run descriptors and the original create receipt are present. expect(Array.isArray(envelope.results)).toBe(true); expect((envelope.results as unknown[]).length).toBe(testIds.length); + expect(envelope.creation).toMatchObject({ + results: testIds.map((testId, specIndex) => ({ + specIndex, + testId, + status: 'created', + })), + summary: { total: 2, created: 2, failed: 0 }, + }); + }); + + it('text dry-run keeps the existing descriptor shape without a creation field', async () => { + const { credentialsPath } = makeCreds(); + const plansFile = writePlansJsonl([FE_SPEC]); + const fetchImpl = makeFetch(url => { + if (url.includes('/tests/batch')) return { body: makeBatchCreateResponse(['test_text_dry']) }; + return { status: 404, body: {} }; + }); + const stdout: string[] = []; + + await runCreateBatch( + { + profile: 'default', + output: 'text', + debug: false, + dryRun: true, + plans: plansFile, + run: true, + wait: false, + timeoutSeconds: 60, + }, + { + credentialsPath, + fetchImpl, + stdout: line => stdout.push(line), + stderr: () => {}, + }, + ); + + const output = stdout.join(''); + // Text mode uses the human-readable formatter, not the JSON envelope. + expect(() => JSON.parse(output)).toThrow(); + expect(output).not.toContain('creation'); }); }); @@ -1743,14 +1899,25 @@ it('a paused workspace outranks a mixed-in credits refusal from create-batch --r it('marks a dispatched batch run for recovery guidance on fan-out interruption', async () => { const { credentialsPath } = makeCreds(); - const plansFile = writePlansJsonl([FE_SPEC]); + const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC]); const interruption = new InterruptError('SIGINT'); const stdout: string[] = []; const stderr: string[] = []; + const creationResponse = { + results: [ + { specIndex: 0, testId: 'test_one', status: 'created' }, + { + specIndex: 1, + status: 'validation_error', + error: { code: 'VALIDATION_ERROR', message: 'invalid spec' }, + }, + ], + summary: { total: 2, created: 1, failed: 1 }, + }; const fetchImpl = (async (input: FetchInput) => { const url = String(input); if (url.includes('/tests/batch')) { - return new Response(JSON.stringify(makeBatchCreateResponse(['test_one']))); + return new Response(JSON.stringify(creationResponse)); } if (url.includes('/tests/test_one/runs')) { return new Response(JSON.stringify(makeTriggerResponse('test_one', 'run_one'))); @@ -1785,6 +1952,17 @@ it('marks a dispatched batch run for recovery guidance on fan-out interruption', expect((error as InterruptError).runWaitContext).toBe(true); expect(JSON.parse(stdout.at(-1)!) as unknown).toMatchObject({ results: [{ testId: 'test_one', runId: 'run_one', status: 'running' }], + creation: { + results: [ + { specIndex: 0, testId: 'test_one', status: 'created' }, + { + specIndex: 1, + status: 'validation_error', + error: { code: 'VALIDATION_ERROR', message: 'invalid spec' }, + }, + ], + summary: { total: 2, created: 1, failed: 1 }, + }, }); expect(stderr.join('\n')).toContain('testsprite test wait run_one'); }); diff --git a/src/commands/test.ts b/src/commands/test.ts index 8bf5ce5b..60ef8238 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -3016,7 +3016,22 @@ export interface CliBatchRunResult { /** Envelope emitted by `test create-batch --run` in JSON mode. */ export interface CliCreateBatchRunResponse { + /** Per-test run results; remains unchanged for existing consumers. */ results: CliBatchRunResult[]; + /** Original per-spec creation receipt, matching create-only JSON output. */ + creation: CliCreateBatchResponse; +} + +/** Dry-run variant of the `test create-batch --run` JSON envelope. */ +export interface CliCreateBatchDryRunResponse extends CliCreateBatchRunResponse { + dryRun: true; + method: 'POST'; + pathTemplate: '/api/cli/v1/tests/{testId}/runs'; + maxConcurrency: number | null; + wait: boolean; + timeoutSeconds: number; + testIds: string[]; + thenPoll?: string; } /** @@ -4325,6 +4340,17 @@ async function runBatchRun( .filter(r => r.status === 'created' && r.testId !== undefined) .map(r => r.testId as string); + // Match the create-only JSON receipt: preserve server links, normalize + // explicitly suppressed links, and use the same create-time fallback map. + const creationResults: CliBatchSpecResult[] = + !opts.dryRun && testIdToDashboardState !== undefined + ? createResponse.results.map(r => { + if (r.status !== 'created' || r.testId === undefined) return r; + const state = testIdToDashboardState.get(r.testId); + return state ? { ...r, dashboardUrl: state.dashboardUrl } : r; + }) + : createResponse.results; + if (testIds.length === 0) { // All specs failed at create time — already threw above; unreachable. return; @@ -4339,9 +4365,9 @@ async function runBatchRun( codeVersion: 'v1', })); const envelope = { - dryRun: true, - method: 'POST', - pathTemplate: '/api/cli/v1/tests/{testId}/runs', + dryRun: true as const, + method: 'POST' as const, + pathTemplate: '/api/cli/v1/tests/{testId}/runs' as const, maxConcurrency: opts.maxConcurrency ?? null, wait: opts.wait ?? false, timeoutSeconds, @@ -4349,7 +4375,19 @@ async function runBatchRun( ...(opts.wait ? { thenPoll: `/api/cli/v1/runs/?waitSeconds=25` } : {}), results: dryRunResults, }; - out.print(envelope); + if (opts.output === 'json') { + const jsonEnvelope: CliCreateBatchDryRunResponse = { + ...envelope, + creation: { + results: creationResults, + summary: createResponse.summary, + }, + }; + out.print(jsonEnvelope); + } else { + // Keep the pre-existing text-mode dry-run descriptor unchanged. + out.print(envelope); + } return; } @@ -4942,8 +4980,15 @@ async function runBatchRun( codeVersion: '', }, ); - out.print({ results: partialResults }, () => - partialResults.map(r => `${r.testId} ${r.runId || '-'} ${r.status}`).join('\n'), + out.print( + { + results: partialResults, + creation: { + results: creationResults, + summary: createResponse.summary, + }, + }, + () => partialResults.map(r => `${r.testId} ${r.runId || '-'} ${r.status}`).join('\n'), ); const unfinished = partialResults .filter(r => r.status === 'running' && r.runId) @@ -4985,7 +5030,13 @@ async function runBatchRun( return { ...r, dashboardUrl: state.dashboardUrl }; }) : batchRunResults; - out.print({ results: enrichedResults }); + out.print({ + results: enrichedResults, + creation: { + results: creationResults, + summary: createResponse.summary, + }, + }); } else if (!opts.wait) { // Text mode, no --wait: statuses are non-terminal by design ('queued'), // so a pass/fail summary would misread a successful dispatch as