diff --git a/.changeset/text-assertions.md b/.changeset/text-assertions.md new file mode 100644 index 00000000..f2401997 --- /dev/null +++ b/.changeset/text-assertions.md @@ -0,0 +1,5 @@ +--- +"@chatcops/cli": minor +--- + +`chatcops test` has `icontains`, `regex` and `equals` text assertions. diff --git a/packages/cli/src/test/assertions/index.ts b/packages/cli/src/test/assertions/index.ts index dc5f8f41..72c82f7b 100644 --- a/packages/cli/src/test/assertions/index.ts +++ b/packages/cli/src/test/assertions/index.ts @@ -2,7 +2,7 @@ import type { Assertion, AssertionCheck, AssertionContext, AssertionResult, Turn import { judge } from './judge.js'; import { javascript } from './javascript.js'; import { cites, leadCaptured } from './parts.js'; -import { contains } from './text.js'; +import { contains, equals, icontains, regex } from './text.js'; import { callsTool, noToolCall } from './tools.js'; import { costUnder, latencyUnder, tokensUnder } from './usage.js'; @@ -14,6 +14,9 @@ import { costUnder, latencyUnder, tokensUnder } from './usage.js'; */ export const ASSERTIONS = new Map([ ['contains', contains], + ['icontains', icontains], + ['equals', equals], + ['regex', regex], ['javascript', javascript], ['judge', judge], ['tokens-under', tokensUnder], diff --git a/packages/cli/src/test/assertions/text.ts b/packages/cli/src/test/assertions/text.ts index b8c78737..8067e29f 100644 --- a/packages/cli/src/test/assertions/text.ts +++ b/packages/cli/src/test/assertions/text.ts @@ -9,3 +9,70 @@ export const contains: AssertionCheck = (value, turn) => { ? { pass: false, reason: `the reply doesn't contain ${quoted(missing)}` } : { pass: true, reason: `the reply contains ${quoted(wanted)}` }; }; + +/** `icontains: "team"` or `icontains: [team, "$19"]`: the reply has every one of them, case-insensitively. */ +export const icontains: AssertionCheck = (value, turn) => { + const wanted = asStrings(value, 'icontains'); + const lower = turn.text.toLowerCase(); + const missing = wanted.filter((text) => !lower.includes(text.toLowerCase())); + return missing.length + ? { pass: false, reason: `the reply doesn't contain ${quoted(missing)}` } + : { pass: true, reason: `the reply contains ${quoted(wanted)}` }; +}; + +function regexReason(error: Error): string { + const match = error.message.match(/:\s*([^:]+)$/); + return match ? match[1].trim() : error.message; +} + +function readRegex(value: unknown): RegExp { + let pattern: string; + let flags: string | undefined; + + if (typeof value === 'string') { + pattern = value; + } else if (value !== null && typeof value === 'object' && !Array.isArray(value)) { + const keys = Object.keys(value); + if (keys.some((key) => key !== 'pattern' && key !== 'flags') || typeof (value as Record).pattern !== 'string') { + throw new Error('regex takes a pattern string or { pattern, flags }'); + } + const record = value as { pattern: string; flags?: unknown }; + if (record.flags !== undefined && typeof record.flags !== 'string') { + throw new Error('regex flags must be a string'); + } + pattern = record.pattern; + flags = record.flags; + } else { + throw new Error('regex takes a pattern string or { pattern, flags }'); + } + + if (flags !== undefined) { + try { + new RegExp('', flags); + } catch { + throw new Error(`regex: flags ${quoted([flags])} aren't valid regular expression flags`); + } + } + + try { + return new RegExp(pattern, flags); + } catch (error) { + throw new Error(`regex: ${quoted([pattern])} isn't a valid regular expression (${regexReason(error as Error)})`); + } +} + +/** `regex: "\\$\\d+"` or `regex: { pattern: "team", flags: "i" }`: matches the reply against a regular expression. */ +export const regex: AssertionCheck = (value, turn) => { + const re = readRegex(value); + return re.test(turn.text) + ? { pass: true, reason: `the reply matches ${re}` } + : { pass: false, reason: `the reply doesn't match ${re}` }; +}; + +/** `equals: "The Team plan is $19."`: the trimmed reply equals the expected string. */ +export const equals: AssertionCheck = (value, turn) => { + if (typeof value !== 'string') throw new Error('equals takes a string'); + return turn.text.trim() === value.trim() + ? { pass: true, reason: `the reply equals ${quoted([value])}` } + : { pass: false, reason: `the reply doesn't equal ${quoted([value])}` }; +}; diff --git a/packages/cli/tests/test-command.test.ts b/packages/cli/tests/test-command.test.ts index 3152f150..057290d0 100644 --- a/packages/cli/tests/test-command.test.ts +++ b/packages/cli/tests/test-command.test.ts @@ -99,6 +99,32 @@ describe('chatcops test', () => { expect(stdout).toContain('2 passed, 0 failed.'); }); + it('runs an eval file exercising icontains, regex, and equals assertions', async () => { + const files = { + 'evals/text-assertions.chat.yaml': `version: 1 +name: Text Assertions +tests: + - name: checks icontains, regex, and equals + turns: + - user: What's the price of Team? + expect: + - icontains: team + - icontains: ["$19", MONTH] + - not-icontains: pro + - regex: '\\$\\d+' + - regex: { pattern: "team plan", flags: "i" } + - not-regex: "^Pro" + - equals: "The Team plan is $19 a month." + - not-equals: "The Pro plan is $29 a month." +`, + }; + const { code, stdout, stderr } = await run(['test'], project(files)); + expect(stderr).toBe(''); + expect(code).toBe(EXIT.ok); + expect(stdout).toContain(`${join('evals', 'text-assertions.chat.yaml')}\n ✓ checks icontains, regex, and equals`); + expect(stdout).toContain('1 passed, 0 failed.'); + }); + it('exits 1 on a failed assertion and says why, with the reply', async () => { const dir = project({ 'evals/pricing.chat.yaml': `version: 1 @@ -412,6 +438,84 @@ describe('assertions', () => { expect(await check({ 'not-contains': 'Team' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply contains "Team"' }); }); + it('icontains matches case-insensitively, and not- inverts it', async () => { + expect(await check({ icontains: ['team', '$19'] }, turn, [turn])).toMatchObject({ pass: true }); + expect(await check({ icontains: 'TEAM' }, turn, [turn])).toMatchObject({ pass: true }); + expect(await check({ icontains: ['team', 'pro'] }, turn, [turn])).toMatchObject({ pass: false, reason: 'the reply doesn\'t contain "pro"' }); + expect(await check({ 'not-icontains': 'pro' }, turn, [turn])).toMatchObject({ pass: true }); + expect(await check({ 'not-icontains': 'team' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply contains "team"' }); + expect(await check({ icontains: 19 }, turn, [turn])).toMatchObject({ pass: false, reason: 'icontains takes a string or a list of strings' }); + expect(await check({ icontains: [] }, turn, [turn])).toMatchObject({ pass: false, reason: 'icontains takes a string or a list of strings' }); + expect(await check({ 'not-icontains': 19 }, turn, [turn])).toMatchObject({ pass: false, reason: 'icontains takes a string or a list of strings' }); + }); + + it('regex matches the reply with a pattern or { pattern, flags }, and not- inverts it', async () => { + expect(await check({ regex: '\\$\\d+' }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply matches /\\$\\d+/' }); + expect(await check({ regex: { pattern: 'team', flags: 'i' } }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply matches /team/i' }); + expect(await check({ regex: { pattern: '\\$\\d+' } }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply matches /\\$\\d+/' }); + expect(await check({ regex: 'pro' }, turn, [turn])).toMatchObject({ pass: false, reason: "the reply doesn't match /pro/" }); + expect(await check({ regex: { pattern: '^team' } }, turn, [turn])).toMatchObject({ pass: false, reason: "the reply doesn't match /^team/" }); + expect(await check({ 'not-regex': 'pro' }, turn, [turn])).toMatchObject({ pass: true }); + expect(await check({ 'not-regex': '\\$\\d+' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply matches /\\$\\d+/' }); + expect(await check({ 'not-regex': { pattern: 'team', flags: 'i' } }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply matches /team/i' }); + + expect(await check({ regex: '(' }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: "(" isn\'t a valid regular expression (Unterminated group)', + }); + expect(await check({ 'not-regex': '(' }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: "(" isn\'t a valid regular expression (Unterminated group)', + }); + expect(await check({ regex: { pattern: '(' } }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: "(" isn\'t a valid regular expression (Unterminated group)', + }); + expect(await check({ regex: { pattern: 'a', flags: 'zz' } }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: flags "zz" aren\'t valid regular expression flags', + }); + expect(await check({ 'not-regex': { pattern: 'a', flags: 'zz' } }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: flags "zz" aren\'t valid regular expression flags', + }); + expect(await check({ regex: '[' }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: "[" isn\'t a valid regular expression (Unterminated character class)', + }); + expect(await check({ regex: { pattern: '[' } }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: "[" isn\'t a valid regular expression (Unterminated character class)', + }); + expect(await check({ regex: { pattern: 'team', flags: 'invalid' } }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'regex: flags "invalid" aren\'t valid regular expression flags', + }); + expect(await check({ regex: { pattern: 'team', flags: 123 } }, turn, [turn])).toMatchObject({ pass: false, reason: 'regex flags must be a string' }); + for (const bad of [19, null, [], {}, { pattern: 19 }, { pattern: 'team', extra: 1 }]) { + expect(await check({ regex: bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'regex takes a pattern string or { pattern, flags }' }); + expect(await check({ 'not-regex': bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'regex takes a pattern string or { pattern, flags }' }); + } + }); + + it('equals compares the trimmed reply with the expected string, and not- inverts it', async () => { + expect(await check({ equals: 'The Team plan is $19.' }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply equals "The Team plan is $19."' }); + const padded: TurnResult = { ...turn, text: ' The Team plan is $19. \n' }; + expect(await check({ equals: 'The Team plan is $19.' }, padded, [padded])).toMatchObject({ pass: true }); + expect(await check({ equals: 'The Team plan is $19.\n' }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply equals "The Team plan is $19.\\n"' }); + expect(await check({ equals: 'The Team plan is $19.\n' }, padded, [padded])).toMatchObject({ pass: true }); + expect(await check({ equals: 'The Pro plan is $29.' }, turn, [turn])).toMatchObject({ pass: false, reason: 'the reply doesn\'t equal "The Pro plan is $29."' }); + expect(await check({ equals: 'The Pro plan is $29.\n' }, turn, [turn])).toMatchObject({ pass: false, reason: 'the reply doesn\'t equal "The Pro plan is $29.\\n"' }); + expect(await check({ 'not-equals': 'The Pro plan is $29.' }, turn, [turn])).toMatchObject({ pass: true }); + expect(await check({ 'not-equals': 'The Pro plan is $29.\n' }, turn, [turn])).toMatchObject({ pass: true }); + expect(await check({ 'not-equals': 'The Team plan is $19.' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply equals "The Team plan is $19."' }); + expect(await check({ 'not-equals': 'The Team plan is $19.\n' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply equals "The Team plan is $19.\\n"' }); + for (const bad of [19, null, [], {}, true]) { + expect(await check({ equals: bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'equals takes a string' }); + expect(await check({ 'not-equals': bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'equals takes a string' }); + } + }); + it('reports bad values and unknown names as failures', async () => { expect(await check({ contains: 19 }, turn, [turn])).toMatchObject({ pass: false, reason: 'contains takes a string or a list of strings' }); expect(await check({ contains: [] }, turn, [turn])).toMatchObject({ pass: false }); diff --git a/website/src/content/docs/core/testing.mdx b/website/src/content/docs/core/testing.mdx index fe7b952a..2ad402c3 100644 --- a/website/src/content/docs/core/testing.mdx +++ b/website/src/content/docs/core/testing.mdx @@ -60,7 +60,14 @@ tests: - contains: "$190" ``` -Each assertion has one key. `contains` takes a string or a list of strings that must all appear in the reply, `judge` has a model grade the reply ([below](#judged-by-a-model)), and every assertion also works with a `not-` prefix to say the opposite. An assertion name that doesn't exist fails the whole run before anything is sent, so a typo can't make a test pass by checking nothing. +Each assertion has one key. `judge` has a model grade the reply ([below](#judged-by-a-model)), and every assertion also works with a `not-` prefix to say the opposite. An assertion name that doesn't exist fails the whole run before anything is sent, so a typo can't make a test pass by checking nothing. + +Four assertions check what the reply says: + +- `contains`: takes a string or a list of strings that must all appear in the reply (`contains: "$19"` or `contains: [Team, "$19"]`). +- `icontains`: case-insensitive substring matching; takes a string or a list of strings (`icontains: team` or `icontains: [team, "$19"]`). +- `regex`: matches the reply against a JavaScript `RegExp`. Accepts a pattern string (`regex: "\\$\\d+"`) or `{ pattern, flags }` (`regex: { pattern: "team", flags: "i" }`). +- `equals`: compares the trimmed reply with the exact expected string (`equals: "The Team plan is $19."`). Three assertions catch a prompt change that makes replies longer, slower or more expensive: