From bc4512f5053e1158e8e152b8628b72306af9dd5d Mon Sep 17 00:00:00 2001 From: bhushan4work Date: Thu, 29 Oct 2026 00:49:43 +0530 Subject: [PATCH 1/3] feat(cli): add icontains, regex and equals text assertions (#411) --- .changeset/text-assertions.md | 5 ++ packages/cli/src/test/assertions/index.ts | 5 +- packages/cli/src/test/assertions/text.ts | 40 +++++++++++++ packages/cli/tests/test-command.test.ts | 70 +++++++++++++++++++++++ website/src/content/docs/core/testing.mdx | 9 ++- 5 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 .changeset/text-assertions.md diff --git a/.changeset/text-assertions.md b/.changeset/text-assertions.md new file mode 100644 index 00000000..f2401997 --- /dev/null +++ b/.changeset/text-assertions.md @@ -0,0 +1,5 @@ +--- +"@chatcops/cli": minor +--- + +`chatcops test` has `icontains`, `regex` and `equals` text assertions. diff --git a/packages/cli/src/test/assertions/index.ts b/packages/cli/src/test/assertions/index.ts index dc5f8f41..72c82f7b 100644 --- a/packages/cli/src/test/assertions/index.ts +++ b/packages/cli/src/test/assertions/index.ts @@ -2,7 +2,7 @@ import type { Assertion, AssertionCheck, AssertionContext, AssertionResult, Turn import { judge } from './judge.js'; import { javascript } from './javascript.js'; import { cites, leadCaptured } from './parts.js'; -import { contains } from './text.js'; +import { contains, equals, icontains, regex } from './text.js'; import { callsTool, noToolCall } from './tools.js'; import { costUnder, latencyUnder, tokensUnder } from './usage.js'; @@ -14,6 +14,9 @@ import { costUnder, latencyUnder, tokensUnder } from './usage.js'; */ export const ASSERTIONS = new Map([ ['contains', contains], + ['icontains', icontains], + ['equals', equals], + ['regex', regex], ['javascript', javascript], ['judge', judge], ['tokens-under', tokensUnder], diff --git a/packages/cli/src/test/assertions/text.ts b/packages/cli/src/test/assertions/text.ts index b8c78737..b4b4983c 100644 --- a/packages/cli/src/test/assertions/text.ts +++ b/packages/cli/src/test/assertions/text.ts @@ -9,3 +9,43 @@ export const contains: AssertionCheck = (value, turn) => { ? { pass: false, reason: `the reply doesn't contain ${quoted(missing)}` } : { pass: true, reason: `the reply contains ${quoted(wanted)}` }; }; + +/** `icontains: "team"` or `icontains: [team, "$19"]`: the reply has every one of them, case-insensitively. */ +export const icontains: AssertionCheck = (value, turn) => { + const wanted = asStrings(value, 'icontains'); + const lower = turn.text.toLowerCase(); + const missing = wanted.filter((text) => !lower.includes(text.toLowerCase())); + return missing.length + ? { pass: false, reason: `the reply doesn't contain ${quoted(missing)}` } + : { pass: true, reason: `the reply contains ${quoted(wanted)}` }; +}; + +function readRegex(value: unknown): RegExp { + if (typeof value === 'string') return new RegExp(value); + if (value !== null && typeof value === 'object' && !Array.isArray(value)) { + const keys = Object.keys(value); + if (keys.some((key) => key !== 'pattern' && key !== 'flags') || typeof (value as Record).pattern !== 'string') { + throw new Error('regex takes a pattern string or { pattern, flags }'); + } + const { pattern, flags } = value as { pattern: string; flags?: unknown }; + if (flags !== undefined && typeof flags !== 'string') throw new Error('regex flags must be a string'); + return new RegExp(pattern, flags); + } + throw new Error('regex takes a pattern string or { pattern, flags }'); +} + +/** `regex: "\\$\\d+"` or `regex: { pattern: "team", flags: "i" }`: matches the reply against a regular expression. */ +export const regex: AssertionCheck = (value, turn) => { + const re = readRegex(value); + return re.test(turn.text) + ? { pass: true, reason: `the reply matches ${re}` } + : { pass: false, reason: `the reply doesn't match ${re}` }; +}; + +/** `equals: "The Team plan is $19."`: the trimmed reply equals the exact expected string. */ +export const equals: AssertionCheck = (value, turn) => { + if (typeof value !== 'string') throw new Error('equals takes a string'); + return turn.text.trim() === value + ? { pass: true, reason: `the reply equals ${quoted([value])}` } + : { pass: false, reason: `the reply doesn't equal ${quoted([value])}` }; +}; diff --git a/packages/cli/tests/test-command.test.ts b/packages/cli/tests/test-command.test.ts index 3152f150..c015ac4f 100644 --- a/packages/cli/tests/test-command.test.ts +++ b/packages/cli/tests/test-command.test.ts @@ -99,6 +99,32 @@ describe('chatcops test', () => { expect(stdout).toContain('2 passed, 0 failed.'); }); + it('runs an eval file exercising icontains, regex, and equals assertions', async () => { + const files = { + 'evals/text-assertions.chat.yaml': `version: 1 +name: Text Assertions +tests: + - name: checks icontains, regex, and equals + turns: + - user: What's the price of Team? + expect: + - icontains: team + - icontains: ["$19", MONTH] + - not-icontains: pro + - regex: '\\$\\d+' + - regex: { pattern: "team plan", flags: "i" } + - not-regex: "^Pro" + - equals: "The Team plan is $19 a month." + - not-equals: "The Pro plan is $29 a month." +`, + }; + const { code, stdout, stderr } = await run(['test'], project(files)); + expect(stderr).toBe(''); + expect(code).toBe(EXIT.ok); + expect(stdout).toContain('✓ evals/text-assertions.chat.yaml > checks icontains, regex, and equals'); + expect(stdout).toContain('1 passed, 0 failed.'); + }); + it('exits 1 on a failed assertion and says why, with the reply', async () => { const dir = project({ 'evals/pricing.chat.yaml': `version: 1 @@ -412,6 +438,50 @@ describe('assertions', () => { expect(await check({ 'not-contains': 'Team' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply contains "Team"' }); }); + it('icontains matches case-insensitively, and not- inverts it', async () => { + expect(await check({ icontains: ['team', '$19'] }, turn, [turn])).toMatchObject({ pass: true }); + expect(await check({ icontains: 'TEAM' }, turn, [turn])).toMatchObject({ pass: true }); + expect(await check({ icontains: ['team', 'pro'] }, turn, [turn])).toMatchObject({ pass: false, reason: 'the reply doesn\'t contain "pro"' }); + expect(await check({ 'not-icontains': 'pro' }, turn, [turn])).toMatchObject({ pass: true }); + expect(await check({ 'not-icontains': 'team' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply contains "team"' }); + expect(await check({ icontains: 19 }, turn, [turn])).toMatchObject({ pass: false, reason: 'icontains takes a string or a list of strings' }); + expect(await check({ icontains: [] }, turn, [turn])).toMatchObject({ pass: false, reason: 'icontains takes a string or a list of strings' }); + expect(await check({ 'not-icontains': 19 }, turn, [turn])).toMatchObject({ pass: false, reason: 'icontains takes a string or a list of strings' }); + }); + + it('regex matches the reply with a pattern or { pattern, flags }, and not- inverts it', async () => { + expect(await check({ regex: '\\$\\d+' }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply matches /\\$\\d+/' }); + expect(await check({ regex: { pattern: 'team', flags: 'i' } }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply matches /team/i' }); + expect(await check({ regex: { pattern: '\\$\\d+' } }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply matches /\\$\\d+/' }); + expect(await check({ regex: 'pro' }, turn, [turn])).toMatchObject({ pass: false, reason: "the reply doesn't match /pro/" }); + expect(await check({ regex: { pattern: '^team' } }, turn, [turn])).toMatchObject({ pass: false, reason: "the reply doesn't match /^team/" }); + expect(await check({ 'not-regex': 'pro' }, turn, [turn])).toMatchObject({ pass: true }); + expect(await check({ 'not-regex': '\\$\\d+' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply matches /\\$\\d+/' }); + expect(await check({ 'not-regex': { pattern: 'team', flags: 'i' } }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply matches /team/i' }); + + expect(await check({ regex: '[' }, turn, [turn])).toMatchObject({ pass: false, reason: expect.stringMatching(/Invalid regular expression/) }); + expect(await check({ regex: { pattern: '[' } }, turn, [turn])).toMatchObject({ pass: false, reason: expect.stringMatching(/Invalid regular expression/) }); + expect(await check({ regex: { pattern: 'team', flags: 'invalid' } }, turn, [turn])).toMatchObject({ pass: false, reason: expect.stringMatching(/Invalid flags/) }); + expect(await check({ regex: { pattern: 'team', flags: 123 } }, turn, [turn])).toMatchObject({ pass: false, reason: 'regex flags must be a string' }); + for (const bad of [19, null, [], {}, { pattern: 19 }, { pattern: 'team', extra: 1 }]) { + expect(await check({ regex: bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'regex takes a pattern string or { pattern, flags }' }); + expect(await check({ 'not-regex': bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'regex takes a pattern string or { pattern, flags }' }); + } + }); + + it('equals compares the trimmed reply with the expected string, and not- inverts it', async () => { + expect(await check({ equals: 'The Team plan is $19.' }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply equals "The Team plan is $19."' }); + const padded: TurnResult = { ...turn, text: ' The Team plan is $19. \n' }; + expect(await check({ equals: 'The Team plan is $19.' }, padded, [padded])).toMatchObject({ pass: true }); + expect(await check({ equals: 'The Pro plan is $29.' }, turn, [turn])).toMatchObject({ pass: false, reason: 'the reply doesn\'t equal "The Pro plan is $29."' }); + expect(await check({ 'not-equals': 'The Pro plan is $29.' }, turn, [turn])).toMatchObject({ pass: true }); + expect(await check({ 'not-equals': 'The Team plan is $19.' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply equals "The Team plan is $19."' }); + for (const bad of [19, null, [], {}, true]) { + expect(await check({ equals: bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'equals takes a string' }); + expect(await check({ 'not-equals': bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'equals takes a string' }); + } + }); + it('reports bad values and unknown names as failures', async () => { expect(await check({ contains: 19 }, turn, [turn])).toMatchObject({ pass: false, reason: 'contains takes a string or a list of strings' }); expect(await check({ contains: [] }, turn, [turn])).toMatchObject({ pass: false }); diff --git a/website/src/content/docs/core/testing.mdx b/website/src/content/docs/core/testing.mdx index fe7b952a..5ebb82ab 100644 --- a/website/src/content/docs/core/testing.mdx +++ b/website/src/content/docs/core/testing.mdx @@ -60,7 +60,14 @@ tests: - contains: "$190" ``` -Each assertion has one key. `contains` takes a string or a list of strings that must all appear in the reply, `judge` has a model grade the reply ([below](#judged-by-a-model)), and every assertion also works with a `not-` prefix to say the opposite. An assertion name that doesn't exist fails the whole run before anything is sent, so a typo can't make a test pass by checking nothing. +Each assertion has one key, `judge` has a model grade the reply ([below](#judged-by-a-model)), and every assertion also works with a `not-` prefix to say the opposite. An assertion name that doesn't exist fails the whole run before anything is sent, so a typo can't make a test pass by checking nothing. + +Four assertions check what the reply says: + +- `contains`: takes a string or a list of strings that must all appear in the reply (`contains: "$19"` or `contains: [Team, "$19"]`). +- `icontains`: case-insensitive substring matching; takes a string or a list of strings (`icontains: team` or `icontains: [team, "$19"]`). +- `regex`: matches the reply against a JavaScript `RegExp`. Accepts a pattern string (`regex: "\\$\\d+"`) or `{ pattern, flags }` (`regex: { pattern: "team", flags: "i" }`). +- `equals`: compares the trimmed reply with the exact expected string (`equals: "The Team plan is $19."`). Three assertions catch a prompt change that makes replies longer, slower or more expensive: From c93332a246e7d63edaecd5d6ca32bc21e35394c0 Mon Sep 17 00:00:00 2001 From: bhushan4work Date: Thu, 29 Oct 2026 08:03:44 +0530 Subject: [PATCH 2/3] fix(cli): provide contextual error messages for invalid regex assertions (#411) --- packages/cli/src/test/assertions/text.ts | 34 ++++++++++++++++++++-- packages/cli/tests/test-command.test.ts | 37 +++++++++++++++++++++--- 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/test/assertions/text.ts b/packages/cli/src/test/assertions/text.ts index b4b4983c..6179a395 100644 --- a/packages/cli/src/test/assertions/text.ts +++ b/packages/cli/src/test/assertions/text.ts @@ -20,8 +20,22 @@ export const icontains: AssertionCheck = (value, turn) => { : { pass: true, reason: `the reply contains ${quoted(wanted)}` }; }; +function regexReason(error: SyntaxError): string { + const match = error.message.match(/:\s*([^:]+)$/); + return match ? match[1].trim() : error.message; +} + function readRegex(value: unknown): RegExp { - if (typeof value === 'string') return new RegExp(value); + if (typeof value === 'string') { + try { + return new RegExp(value); + } catch (error) { + if (error instanceof SyntaxError) { + throw new Error(`regex: ${quoted([value])} isn't a valid regular expression (${regexReason(error)})`); + } + throw error; + } + } if (value !== null && typeof value === 'object' && !Array.isArray(value)) { const keys = Object.keys(value); if (keys.some((key) => key !== 'pattern' && key !== 'flags') || typeof (value as Record).pattern !== 'string') { @@ -29,7 +43,23 @@ function readRegex(value: unknown): RegExp { } const { pattern, flags } = value as { pattern: string; flags?: unknown }; if (flags !== undefined && typeof flags !== 'string') throw new Error('regex flags must be a string'); - return new RegExp(pattern, flags); + try { + return new RegExp(pattern, flags); + } catch (error) { + if (error instanceof SyntaxError) { + if (flags !== undefined) { + try { + new RegExp('', flags); + } catch (flagsError) { + if (flagsError instanceof SyntaxError) { + throw new Error(`regex: flags ${quoted([flags])} aren't valid regular expression flags (${regexReason(flagsError)})`); + } + } + } + throw new Error(`regex: ${quoted([pattern])} isn't a valid regular expression (${regexReason(error)})`); + } + throw error; + } } throw new Error('regex takes a pattern string or { pattern, flags }'); } diff --git a/packages/cli/tests/test-command.test.ts b/packages/cli/tests/test-command.test.ts index c015ac4f..aa0d90a1 100644 --- a/packages/cli/tests/test-command.test.ts +++ b/packages/cli/tests/test-command.test.ts @@ -121,7 +121,7 @@ tests: const { code, stdout, stderr } = await run(['test'], project(files)); expect(stderr).toBe(''); expect(code).toBe(EXIT.ok); - expect(stdout).toContain('✓ evals/text-assertions.chat.yaml > checks icontains, regex, and equals'); + expect(stdout).toContain(`${join('evals', 'text-assertions.chat.yaml')}\n ✓ checks icontains, regex, and equals`); expect(stdout).toContain('1 passed, 0 failed.'); }); @@ -459,9 +459,38 @@ describe('assertions', () => { expect(await check({ 'not-regex': '\\$\\d+' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply matches /\\$\\d+/' }); expect(await check({ 'not-regex': { pattern: 'team', flags: 'i' } }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply matches /team/i' }); - expect(await check({ regex: '[' }, turn, [turn])).toMatchObject({ pass: false, reason: expect.stringMatching(/Invalid regular expression/) }); - expect(await check({ regex: { pattern: '[' } }, turn, [turn])).toMatchObject({ pass: false, reason: expect.stringMatching(/Invalid regular expression/) }); - expect(await check({ regex: { pattern: 'team', flags: 'invalid' } }, turn, [turn])).toMatchObject({ pass: false, reason: expect.stringMatching(/Invalid flags/) }); + expect(await check({ regex: '(' }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: "(" isn\'t a valid regular expression (Unterminated group)', + }); + expect(await check({ 'not-regex': '(' }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: "(" isn\'t a valid regular expression (Unterminated group)', + }); + expect(await check({ regex: { pattern: '(' } }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: "(" isn\'t a valid regular expression (Unterminated group)', + }); + expect(await check({ regex: { pattern: 'a', flags: 'zz' } }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: flags "zz" aren\'t valid regular expression flags (Invalid flags supplied to RegExp constructor \'zz\')', + }); + expect(await check({ 'not-regex': { pattern: 'a', flags: 'zz' } }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: flags "zz" aren\'t valid regular expression flags (Invalid flags supplied to RegExp constructor \'zz\')', + }); + expect(await check({ regex: '[' }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: "[" isn\'t a valid regular expression (Unterminated character class)', + }); + expect(await check({ regex: { pattern: '[' } }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: "[" isn\'t a valid regular expression (Unterminated character class)', + }); + expect(await check({ regex: { pattern: 'team', flags: 'invalid' } }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: flags "invalid" aren\'t valid regular expression flags (Invalid flags supplied to RegExp constructor \'invalid\')', + }); expect(await check({ regex: { pattern: 'team', flags: 123 } }, turn, [turn])).toMatchObject({ pass: false, reason: 'regex flags must be a string' }); for (const bad of [19, null, [], {}, { pattern: 19 }, { pattern: 'team', extra: 1 }]) { expect(await check({ regex: bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'regex takes a pattern string or { pattern, flags }' }); From f7e5c5a3a5ae918b5f5dad5d9747ba00e2f97fdd Mon Sep 17 00:00:00 2001 From: bhushan4work Date: Thu, 29 Oct 2026 21:35:29 +0530 Subject: [PATCH 3/3] fix(cli): simplify regex validation and trim equals assertion (#411) --- packages/cli/src/test/assertions/text.ts | 59 +++++++++++------------ packages/cli/tests/test-command.test.ts | 11 +++-- website/src/content/docs/core/testing.mdx | 2 +- 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/packages/cli/src/test/assertions/text.ts b/packages/cli/src/test/assertions/text.ts index 6179a395..8067e29f 100644 --- a/packages/cli/src/test/assertions/text.ts +++ b/packages/cli/src/test/assertions/text.ts @@ -20,48 +20,45 @@ export const icontains: AssertionCheck = (value, turn) => { : { pass: true, reason: `the reply contains ${quoted(wanted)}` }; }; -function regexReason(error: SyntaxError): string { +function regexReason(error: Error): string { const match = error.message.match(/:\s*([^:]+)$/); return match ? match[1].trim() : error.message; } function readRegex(value: unknown): RegExp { + let pattern: string; + let flags: string | undefined; + if (typeof value === 'string') { - try { - return new RegExp(value); - } catch (error) { - if (error instanceof SyntaxError) { - throw new Error(`regex: ${quoted([value])} isn't a valid regular expression (${regexReason(error)})`); - } - throw error; - } - } - if (value !== null && typeof value === 'object' && !Array.isArray(value)) { + pattern = value; + } else if (value !== null && typeof value === 'object' && !Array.isArray(value)) { const keys = Object.keys(value); if (keys.some((key) => key !== 'pattern' && key !== 'flags') || typeof (value as Record).pattern !== 'string') { throw new Error('regex takes a pattern string or { pattern, flags }'); } - const { pattern, flags } = value as { pattern: string; flags?: unknown }; - if (flags !== undefined && typeof flags !== 'string') throw new Error('regex flags must be a string'); + const record = value as { pattern: string; flags?: unknown }; + if (record.flags !== undefined && typeof record.flags !== 'string') { + throw new Error('regex flags must be a string'); + } + pattern = record.pattern; + flags = record.flags; + } else { + throw new Error('regex takes a pattern string or { pattern, flags }'); + } + + if (flags !== undefined) { try { - return new RegExp(pattern, flags); - } catch (error) { - if (error instanceof SyntaxError) { - if (flags !== undefined) { - try { - new RegExp('', flags); - } catch (flagsError) { - if (flagsError instanceof SyntaxError) { - throw new Error(`regex: flags ${quoted([flags])} aren't valid regular expression flags (${regexReason(flagsError)})`); - } - } - } - throw new Error(`regex: ${quoted([pattern])} isn't a valid regular expression (${regexReason(error)})`); - } - throw error; + new RegExp('', flags); + } catch { + throw new Error(`regex: flags ${quoted([flags])} aren't valid regular expression flags`); } } - throw new Error('regex takes a pattern string or { pattern, flags }'); + + try { + return new RegExp(pattern, flags); + } catch (error) { + throw new Error(`regex: ${quoted([pattern])} isn't a valid regular expression (${regexReason(error as Error)})`); + } } /** `regex: "\\$\\d+"` or `regex: { pattern: "team", flags: "i" }`: matches the reply against a regular expression. */ @@ -72,10 +69,10 @@ export const regex: AssertionCheck = (value, turn) => { : { pass: false, reason: `the reply doesn't match ${re}` }; }; -/** `equals: "The Team plan is $19."`: the trimmed reply equals the exact expected string. */ +/** `equals: "The Team plan is $19."`: the trimmed reply equals the expected string. */ export const equals: AssertionCheck = (value, turn) => { if (typeof value !== 'string') throw new Error('equals takes a string'); - return turn.text.trim() === value + return turn.text.trim() === value.trim() ? { pass: true, reason: `the reply equals ${quoted([value])}` } : { pass: false, reason: `the reply doesn't equal ${quoted([value])}` }; }; diff --git a/packages/cli/tests/test-command.test.ts b/packages/cli/tests/test-command.test.ts index aa0d90a1..057290d0 100644 --- a/packages/cli/tests/test-command.test.ts +++ b/packages/cli/tests/test-command.test.ts @@ -473,11 +473,11 @@ describe('assertions', () => { }); expect(await check({ regex: { pattern: 'a', flags: 'zz' } }, turn, [turn])).toMatchObject({ pass: false, - reason: 'regex: flags "zz" aren\'t valid regular expression flags (Invalid flags supplied to RegExp constructor \'zz\')', + reason: 'regex: flags "zz" aren\'t valid regular expression flags', }); expect(await check({ 'not-regex': { pattern: 'a', flags: 'zz' } }, turn, [turn])).toMatchObject({ pass: false, - reason: 'regex: flags "zz" aren\'t valid regular expression flags (Invalid flags supplied to RegExp constructor \'zz\')', + reason: 'regex: flags "zz" aren\'t valid regular expression flags', }); expect(await check({ regex: '[' }, turn, [turn])).toMatchObject({ pass: false, @@ -489,7 +489,7 @@ describe('assertions', () => { }); expect(await check({ regex: { pattern: 'team', flags: 'invalid' } }, turn, [turn])).toMatchObject({ pass: false, - reason: 'regex: flags "invalid" aren\'t valid regular expression flags (Invalid flags supplied to RegExp constructor \'invalid\')', + reason: 'regex: flags "invalid" aren\'t valid regular expression flags', }); expect(await check({ regex: { pattern: 'team', flags: 123 } }, turn, [turn])).toMatchObject({ pass: false, reason: 'regex flags must be a string' }); for (const bad of [19, null, [], {}, { pattern: 19 }, { pattern: 'team', extra: 1 }]) { @@ -502,9 +502,14 @@ describe('assertions', () => { expect(await check({ equals: 'The Team plan is $19.' }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply equals "The Team plan is $19."' }); const padded: TurnResult = { ...turn, text: ' The Team plan is $19. \n' }; expect(await check({ equals: 'The Team plan is $19.' }, padded, [padded])).toMatchObject({ pass: true }); + expect(await check({ equals: 'The Team plan is $19.\n' }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply equals "The Team plan is $19.\\n"' }); + expect(await check({ equals: 'The Team plan is $19.\n' }, padded, [padded])).toMatchObject({ pass: true }); expect(await check({ equals: 'The Pro plan is $29.' }, turn, [turn])).toMatchObject({ pass: false, reason: 'the reply doesn\'t equal "The Pro plan is $29."' }); + expect(await check({ equals: 'The Pro plan is $29.\n' }, turn, [turn])).toMatchObject({ pass: false, reason: 'the reply doesn\'t equal "The Pro plan is $29.\\n"' }); expect(await check({ 'not-equals': 'The Pro plan is $29.' }, turn, [turn])).toMatchObject({ pass: true }); + expect(await check({ 'not-equals': 'The Pro plan is $29.\n' }, turn, [turn])).toMatchObject({ pass: true }); expect(await check({ 'not-equals': 'The Team plan is $19.' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply equals "The Team plan is $19."' }); + expect(await check({ 'not-equals': 'The Team plan is $19.\n' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply equals "The Team plan is $19.\\n"' }); for (const bad of [19, null, [], {}, true]) { expect(await check({ equals: bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'equals takes a string' }); expect(await check({ 'not-equals': bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'equals takes a string' }); diff --git a/website/src/content/docs/core/testing.mdx b/website/src/content/docs/core/testing.mdx index 5ebb82ab..2ad402c3 100644 --- a/website/src/content/docs/core/testing.mdx +++ b/website/src/content/docs/core/testing.mdx @@ -60,7 +60,7 @@ tests: - contains: "$190" ``` -Each assertion has one key, `judge` has a model grade the reply ([below](#judged-by-a-model)), and every assertion also works with a `not-` prefix to say the opposite. An assertion name that doesn't exist fails the whole run before anything is sent, so a typo can't make a test pass by checking nothing. +Each assertion has one key. `judge` has a model grade the reply ([below](#judged-by-a-model)), and every assertion also works with a `not-` prefix to say the opposite. An assertion name that doesn't exist fails the whole run before anything is sent, so a typo can't make a test pass by checking nothing. Four assertions check what the reply says: