Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
23 commits
Select commit Hold shift + click to select a range
2c63e10
fix(create-batch): preserve per-spec creation failures in --run JSON …
Oct 20, 2026
144e23b
chore: trigger CI entry check
Oct 22, 2026
acd284c
Merge remote-tracking branch 'upstream/main' into fix/483-create-batc…
Oct 22, 2026
a284177
Merge branch 'main' into fix/483-create-batch-run-receipt
zeshi-du Oct 27, 2026
600faa5
fix(create-batch): preserve creation receipt on partial JSON output
Lexiie Nov 2, 2026
a8e4cfb
test(create-batch): cover creation receipt on interruption
Lexiie Nov 2, 2026
f177758
style(create-batch): format creation receipt output
Lexiie Nov 2, 2026
c4c120d
style(create-batch): format interruption regression test
Lexiie Nov 2, 2026
b98e5c1
test(create-batch): use valid creation failure status
Lexiie Nov 10, 2026
86788a0
test(create-batch): assert creation failure contract
Lexiie Nov 10, 2026
116efd0
fix(create-batch): preserve enriched creation receipt in run output
Lexiie Nov 10, 2026
f343969
types(create-batch): model creation receipt in run envelopes
Lexiie Nov 10, 2026
40baea3
test(create-batch): cover creation receipt and dashboard links
Lexiie Nov 10, 2026
822b7fe
test(create-batch): cover partial creation receipts with and without …
Lexiie Nov 10, 2026
9ae648e
test(create-batch): assert creation receipt without wait
Lexiie Nov 10, 2026
c78484c
types(create-batch): type-check dry-run run envelope
Lexiie Nov 10, 2026
2cb6d13
docs(create-batch): document creation receipt in run output
Lexiie Nov 10, 2026
981054c
test(create-batch): align failed row with response contract
Lexiie Nov 10, 2026
377a6ce
test(create-batch): assert failed creation diagnostics
Lexiie Nov 10, 2026
c9d4284
fix(create-batch): keep text dry-run output unchanged
Lexiie Nov 10, 2026
8767056
test(create-batch): preserve text dry-run descriptor shape
Lexiie Nov 10, 2026
12a2a4e
docs(create-batch): format JSON example
Lexiie Nov 10, 2026
695b70e
test(create-batch): assert text dry-run output without JSON parsing
Lexiie Nov 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions DOCUMENTATION.md
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,45 @@ testsprite test create --plan-from ./checkout.plan.json --dry-run --output json

Bulk-create frontend tests from a JSONL plan-steps file (or a directory of plan files with `--plan-from-dir`). Optional `--run --max-concurrency <N>` fans out triggers. Without `--wait`, each run is dispatched (`status: "queued"`) and the command exits 0 when every trigger is accepted — mirroring single `test run` without `--wait`; a trigger error still exits non-zero. With `--wait`, it polls every run to terminal and exits non-zero if any run does not pass.

**JSON output with `--run`.** The top-level `results` array remains the per-test run results. A separate additive `creation` object preserves the original batch-create receipt, including failures for individual specs that were not created:

```json
{
"results": [
{
"testId": "test_xxxxxxxx",
"runId": "run_xxxxxxxx",
"status": "passed"
}
],
"creation": {
"results": [
{
"specIndex": 0,
"testId": "test_xxxxxxxx",
"status": "created",
"dashboardUrl": "https://..."
},
{
"specIndex": 1,
"status": "validation_error",
"error": {
"code": "VALIDATION_ERROR",
"message": "Invalid plan"
}
}
],
"summary": {
"total": 2,
"created": 1,
"failed": 1
}
}
}
```

The `creation` object has the same `{ results, summary }` shape as create-only JSON output. Each creation result retains its `specIndex`, status, optional error and optional `dashboardUrl`; the dashboard URL is omitted when unavailable or explicitly suppressed. This receipt is also included in interrupted/partial output and `--dry-run` JSON envelopes. Existing run-result rows, text output and exit codes are unchanged.

```bash
testsprite test create-batch --plans ./plans.jsonl --run --max-concurrency 4 --output json
testsprite test create-batch --plan-from-dir ./plans/ --dry-run --output json
Expand Down
188 changes: 183 additions & 5 deletions src/commands/test.create-batch-run.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,19 @@ describe('runCreateBatch --run --wait: happy path', () => {
const fetchImpl = makeFetch(url => {
// Batch create
if (url.includes('/tests/batch')) {
return { body: makeBatchCreateResponse(testIds) };
return {
body: {
results: testIds.map((testId, specIndex) => ({
specIndex,
testId,
status: 'created' as const,
...(specIndex === 0
? { dashboardUrl: 'https://portal.testsprite.com/tests/test_a1' }
: {}),
})),
summary: { total: testIds.length, created: testIds.length, failed: 0 },
},
};
}
// Trigger POST /tests/{id}/runs
const triggerMatch = /\/tests\/(test_[a-z0-9]+)\/runs$/.exec(url);
Expand Down Expand Up @@ -238,8 +250,21 @@ describe('runCreateBatch --run --wait: happy path', () => {
);

// JSON output is the batch-run envelope
const printed = JSON.parse(stdout.join('')) as { results: unknown[] };
const printed = JSON.parse(stdout.join('')) as {
results: unknown[];
creation: {
results: Array<{ testId?: string; status: string; dashboardUrl?: string }>;
summary: { total: number; created: number; failed: number };
};
};
expect(printed.results).toHaveLength(3);
expect(printed.creation.summary).toEqual({ total: 3, created: 3, failed: 0 });
expect(printed.creation.results).toHaveLength(3);
expect(printed.creation.results[0]).toMatchObject({
testId: 'test_a1',
status: 'created',
dashboardUrl: 'https://portal.testsprite.com/tests/test_a1',
});
const results = printed.results as Array<{ testId: string; status: string; runId: string }>;
expect(results.every(r => r.status === 'passed')).toBe(true);
expect(results.map(r => r.testId).sort()).toEqual(['test_a1', 'test_a2', 'test_a3'].sort());
Expand Down Expand Up @@ -417,10 +442,16 @@ describe('runCreateBatch --run (no --wait): exit-code contract', () => {

const printed = JSON.parse(stdout.join('')) as {
results: Array<{ testId: string; status: string; error?: unknown }>;
creation: {
results: Array<{ specIndex: number; testId?: string; status: string }>;
summary: { total: number; created: number; failed: number };
};
};
expect(printed.results).toHaveLength(3);
expect(printed.results.every(r => r.status === 'queued')).toBe(true);
expect(printed.results.every(r => r.error === undefined)).toBe(true);
expect(printed.creation.results).toHaveLength(3);
expect(printed.creation.summary).toEqual({ total: 3, created: 3, failed: 0 });
});

it('all triggers succeed (queued) → text summary reports "3/3 triggered", exit 0 (no "0/N passed")', async () => {
Expand Down Expand Up @@ -1068,6 +1099,89 @@ describe('runCreateBatch --run --wait: CONFLICT auto-resume', () => {
});
});

// ---------------------------------------------------------------------------
// Partial creation success — the receipt must survive normal completion with
// and without --wait, independently of the run-result status.
// ---------------------------------------------------------------------------

describe('runCreateBatch --run: partial creation receipt', () => {
it.each([true, false])(
'preserves created and validation_error rows on normal completion (wait=%s)',
async wait => {
const { credentialsPath } = makeCreds();
const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC]);
const creationResponse = {
results: [
{ specIndex: 0, testId: 'test_partial_created', status: 'created' },
{
specIndex: 1,
status: 'validation_error',
error: { code: 'VALIDATION_ERROR', message: 'invalid plan', field: 'planSteps' },
},
],
summary: { total: 2, created: 1, failed: 1 },
};

const fetchImpl = makeFetch(url => {
if (url.includes('/tests/batch')) return { body: creationResponse };
if (url.endsWith('/tests/test_partial_created/runs')) {
return { body: makeTriggerResponse('test_partial_created', 'run_partial_created') };
}
if (url.includes('/runs/run_partial_created')) {
return { body: makePassedRun('test_partial_created', 'run_partial_created') };
}
return {
status: 404,
body: {
error: { code: 'NOT_FOUND', message: 'not found', nextAction: '', requestId: 'r1' },
},
};
});

const stdout: string[] = [];
await runCreateBatch(
{
profile: 'default',
output: 'json',
debug: false,
dryRun: false,
plans: plansFile,
run: true,
wait,
maxConcurrency: 1,
timeoutSeconds: 60,
},
{
credentialsPath,
fetchImpl,
stdout: line => stdout.push(line),
stderr: () => {},
sleep: instantSleep,
},
);

const printed = JSON.parse(stdout.join('')) as {
results: Array<{ testId: string; status: string }>;
creation: {
results: Array<{ specIndex: number; testId?: string; status: string; error?: unknown }>;
summary: { total: number; created: number; failed: number };
};
};
expect(printed.results).toHaveLength(1);
expect(printed.results[0]?.status).toBe(wait ? 'passed' : 'queued');
expect(printed.creation.summary).toEqual({ total: 2, created: 1, failed: 1 });
expect(printed.creation.results).toMatchObject([
{ specIndex: 0, testId: 'test_partial_created', status: 'created' },
{
specIndex: 1,
status: 'validation_error',
error: { code: 'VALIDATION_ERROR', field: 'planSteps' },
},
]);
},
);
});

// ---------------------------------------------------------------------------
// --dry-run mode
// ---------------------------------------------------------------------------
Expand Down Expand Up @@ -1136,9 +1250,51 @@ describe('runCreateBatch --run --dry-run', () => {
expect(envelope.dryRun).toBe(true);
expect(Array.isArray(envelope.testIds)).toBe(true);
expect((envelope.testIds as string[]).sort()).toEqual(testIds.sort());
// results array is present
// Both run descriptors and the original create receipt are present.
expect(Array.isArray(envelope.results)).toBe(true);
expect((envelope.results as unknown[]).length).toBe(testIds.length);
expect(envelope.creation).toMatchObject({
results: testIds.map((testId, specIndex) => ({
specIndex,
testId,
status: 'created',
})),
summary: { total: 2, created: 2, failed: 0 },
});
});

it('text dry-run keeps the existing descriptor shape without a creation field', async () => {
const { credentialsPath } = makeCreds();
const plansFile = writePlansJsonl([FE_SPEC]);
const fetchImpl = makeFetch(url => {
if (url.includes('/tests/batch')) return { body: makeBatchCreateResponse(['test_text_dry']) };
return { status: 404, body: {} };
});
const stdout: string[] = [];

await runCreateBatch(
{
profile: 'default',
output: 'text',
debug: false,
dryRun: true,
plans: plansFile,
run: true,
wait: false,
timeoutSeconds: 60,
},
{
credentialsPath,
fetchImpl,
stdout: line => stdout.push(line),
stderr: () => {},
},
);

const output = stdout.join('');
// Text mode uses the human-readable formatter, not the JSON envelope.
expect(() => JSON.parse(output)).toThrow();
expect(output).not.toContain('creation');
});
});

Expand Down Expand Up @@ -1743,14 +1899,25 @@ it('a paused workspace outranks a mixed-in credits refusal from create-batch --r

it('marks a dispatched batch run for recovery guidance on fan-out interruption', async () => {
const { credentialsPath } = makeCreds();
const plansFile = writePlansJsonl([FE_SPEC]);
const plansFile = writePlansJsonl([FE_SPEC, FE_SPEC]);
const interruption = new InterruptError('SIGINT');
const stdout: string[] = [];
const stderr: string[] = [];
const creationResponse = {
results: [
{ specIndex: 0, testId: 'test_one', status: 'created' },
{
specIndex: 1,
status: 'validation_error',
error: { code: 'VALIDATION_ERROR', message: 'invalid spec' },
},
],
summary: { total: 2, created: 1, failed: 1 },
};
const fetchImpl = (async (input: FetchInput) => {
const url = String(input);
if (url.includes('/tests/batch')) {
return new Response(JSON.stringify(makeBatchCreateResponse(['test_one'])));
return new Response(JSON.stringify(creationResponse));
}
if (url.includes('/tests/test_one/runs')) {
return new Response(JSON.stringify(makeTriggerResponse('test_one', 'run_one')));
Expand Down Expand Up @@ -1785,6 +1952,17 @@ it('marks a dispatched batch run for recovery guidance on fan-out interruption',
expect((error as InterruptError).runWaitContext).toBe(true);
expect(JSON.parse(stdout.at(-1)!) as unknown).toMatchObject({
results: [{ testId: 'test_one', runId: 'run_one', status: 'running' }],
creation: {
results: [
{ specIndex: 0, testId: 'test_one', status: 'created' },
{
specIndex: 1,
status: 'validation_error',
error: { code: 'VALIDATION_ERROR', message: 'invalid spec' },
},
],
summary: { total: 2, created: 1, failed: 1 },
},
});
expect(stderr.join('\n')).toContain('testsprite test wait run_one');
});
Expand Down
Loading
Loading