feat(cli): add icontains, regex and equals text assertions (#411)

.changeset/text-assertions.md

@@ -0,0 +1,5 @@
+---
+"@chatcops/cli": minor
+---
+
+`chatcops test` has `icontains`, `regex` and `equals` text assertions.

packages/cli/src/test/assertions/index.ts

@@ -2,7 +2,7 @@ import type { Assertion, AssertionCheck, AssertionContext, AssertionResult, Turn
 import { judge } from './judge.js';
 import { javascript } from './javascript.js';
 import { cites, leadCaptured } from './parts.js';
-import { contains } from './text.js';
+import { contains, equals, icontains, regex } from './text.js';
 import { callsTool, noToolCall } from './tools.js';
 import { costUnder, latencyUnder, tokensUnder } from './usage.js';
 
@@ -14,6 +14,9 @@ import { costUnder, latencyUnder, tokensUnder } from './usage.js';
  */
 export const ASSERTIONS = new Map<string, AssertionCheck>([
   ['contains', contains],
+  ['icontains', icontains],
+  ['equals', equals],
+  ['regex', regex],
   ['javascript', javascript],
   ['judge', judge],
   ['tokens-under', tokensUnder],

packages/cli/src/test/assertions/text.ts

@@ -9,3 +9,70 @@ export const contains: AssertionCheck = (value, turn) => {
     ? { pass: false, reason: `the reply doesn't contain ${quoted(missing)}` }
     : { pass: true, reason: `the reply contains ${quoted(wanted)}` };
 };
+
+/** `icontains: "team"` or `icontains: [team, "$19"]`: the reply has every one of them, case-insensitively. */
+export const icontains: AssertionCheck = (value, turn) => {
+  const wanted = asStrings(value, 'icontains');
+  const lower = turn.text.toLowerCase();
+  const missing = wanted.filter((text) => !lower.includes(text.toLowerCase()));
+  return missing.length
+    ? { pass: false, reason: `the reply doesn't contain ${quoted(missing)}` }
+    : { pass: true, reason: `the reply contains ${quoted(wanted)}` };
+};
+
+function regexReason(error: Error): string {
+  const match = error.message.match(/:\s*([^:]+)$/);
+  return match ? match[1].trim() : error.message;
+}
+
+function readRegex(value: unknown): RegExp {
+  let pattern: string;
+  let flags: string | undefined;
+
+  if (typeof value === 'string') {
+    pattern = value;
+  } else if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
+    const keys = Object.keys(value);
+    if (keys.some((key) => key !== 'pattern' && key !== 'flags') || typeof (value as Record<string, unknown>).pattern !== 'string') {
+      throw new Error('regex takes a pattern string or { pattern, flags }');
+    }
+    const record = value as { pattern: string; flags?: unknown };
+    if (record.flags !== undefined && typeof record.flags !== 'string') {
+      throw new Error('regex flags must be a string');
+    }
+    pattern = record.pattern;
+    flags = record.flags;
+  } else {
+    throw new Error('regex takes a pattern string or { pattern, flags }');
+  }
+
+  if (flags !== undefined) {
+    try {
+      new RegExp('', flags);
+    } catch {
+      throw new Error(`regex: flags ${quoted([flags])} aren't valid regular expression flags`);
+    }
+  }
+
+  try {
+    return new RegExp(pattern, flags);
+  } catch (error) {
+    throw new Error(`regex: ${quoted([pattern])} isn't a valid regular expression (${regexReason(error as Error)})`);
+  }
+}
+
+/** `regex: "\\$\\d+"` or `regex: { pattern: "team", flags: "i" }`: matches the reply against a regular expression. */
+export const regex: AssertionCheck = (value, turn) => {
+  const re = readRegex(value);
+  return re.test(turn.text)
+    ? { pass: true, reason: `the reply matches ${re}` }
+    : { pass: false, reason: `the reply doesn't match ${re}` };
+};
+
+/** `equals: "The Team plan is $19."`: the trimmed reply equals the expected string. */
+export const equals: AssertionCheck = (value, turn) => {
+  if (typeof value !== 'string') throw new Error('equals takes a string');
+  return turn.text.trim() === value.trim()
+    ? { pass: true, reason: `the reply equals ${quoted([value])}` }
+    : { pass: false, reason: `the reply doesn't equal ${quoted([value])}` };
+};

packages/cli/tests/test-command.test.ts

@@ -99,6 +99,32 @@ describe('chatcops test', () => {
     expect(stdout).toContain('2 passed, 0 failed.');
   });
 
+  it('runs an eval file exercising icontains, regex, and equals assertions', async () => {
+    const files = {
+      'evals/text-assertions.chat.yaml': `version: 1
+name: Text Assertions
+tests:
+  - name: checks icontains, regex, and equals
+    turns:
+      - user: What's the price of Team?
+        expect:
+          - icontains: team
+          - icontains: ["$19", MONTH]
+          - not-icontains: pro
+          - regex: '\\$\\d+'
+          - regex: { pattern: "team plan", flags: "i" }
+          - not-regex: "^Pro"
+          - equals: "The Team plan is $19 a month."
+          - not-equals: "The Pro plan is $29 a month."
+`,
+    };
+    const { code, stdout, stderr } = await run(['test'], project(files));
+    expect(stderr).toBe('');
+    expect(code).toBe(EXIT.ok);
+    expect(stdout).toContain(`${join('evals', 'text-assertions.chat.yaml')}\n  ✓ checks icontains, regex, and equals`);
+    expect(stdout).toContain('1 passed, 0 failed.');
+  });
+
   it('exits 1 on a failed assertion and says why, with the reply', async () => {
     const dir = project({
       'evals/pricing.chat.yaml': `version: 1
@@ -412,6 +438,84 @@ describe('assertions', () => {
     expect(await check({ 'not-contains': 'Team' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply contains "Team"' });
   });
 
+  it('icontains matches case-insensitively, and not- inverts it', async () => {
+    expect(await check({ icontains: ['team', '$19'] }, turn, [turn])).toMatchObject({ pass: true });
+    expect(await check({ icontains: 'TEAM' }, turn, [turn])).toMatchObject({ pass: true });
+    expect(await check({ icontains: ['team', 'pro'] }, turn, [turn])).toMatchObject({ pass: false, reason: 'the reply doesn\'t contain "pro"' });
+    expect(await check({ 'not-icontains': 'pro' }, turn, [turn])).toMatchObject({ pass: true });
+    expect(await check({ 'not-icontains': 'team' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply contains "team"' });
+    expect(await check({ icontains: 19 }, turn, [turn])).toMatchObject({ pass: false, reason: 'icontains takes a string or a list of strings' });
+    expect(await check({ icontains: [] }, turn, [turn])).toMatchObject({ pass: false, reason: 'icontains takes a string or a list of strings' });
+    expect(await check({ 'not-icontains': 19 }, turn, [turn])).toMatchObject({ pass: false, reason: 'icontains takes a string or a list of strings' });
+  });
+
+  it('regex matches the reply with a pattern or { pattern, flags }, and not- inverts it', async () => {
+    expect(await check({ regex: '\\$\\d+' }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply matches /\\$\\d+/' });
+    expect(await check({ regex: { pattern: 'team', flags: 'i' } }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply matches /team/i' });
+    expect(await check({ regex: { pattern: '\\$\\d+' } }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply matches /\\$\\d+/' });
+    expect(await check({ regex: 'pro' }, turn, [turn])).toMatchObject({ pass: false, reason: "the reply doesn't match /pro/" });
+    expect(await check({ regex: { pattern: '^team' } }, turn, [turn])).toMatchObject({ pass: false, reason: "the reply doesn't match /^team/" });
+    expect(await check({ 'not-regex': 'pro' }, turn, [turn])).toMatchObject({ pass: true });
+    expect(await check({ 'not-regex': '\\$\\d+' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply matches /\\$\\d+/' });
+    expect(await check({ 'not-regex': { pattern: 'team', flags: 'i' } }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply matches /team/i' });
+
+    expect(await check({ regex: '(' }, turn, [turn])).toMatchObject({
+      pass: false,
+      reason: 'regex: "(" isn\'t a valid regular expression (Unterminated group)',
+    });
+    expect(await check({ 'not-regex': '(' }, turn, [turn])).toMatchObject({
+      pass: false,
+      reason: 'regex: "(" isn\'t a valid regular expression (Unterminated group)',
+    });
+    expect(await check({ regex: { pattern: '(' } }, turn, [turn])).toMatchObject({
+      pass: false,
+      reason: 'regex: "(" isn\'t a valid regular expression (Unterminated group)',
+    });
+    expect(await check({ regex: { pattern: 'a', flags: 'zz' } }, turn, [turn])).toMatchObject({
+      pass: false,
+      reason: 'regex: flags "zz" aren\'t valid regular expression flags',
+    });
+    expect(await check({ 'not-regex': { pattern: 'a', flags: 'zz' } }, turn, [turn])).toMatchObject({
+      pass: false,
+      reason: 'regex: flags "zz" aren\'t valid regular expression flags',
+    });
+    expect(await check({ regex: '[' }, turn, [turn])).toMatchObject({
+      pass: false,
+      reason: 'regex: "[" isn\'t a valid regular expression (Unterminated character class)',
+    });
+    expect(await check({ regex: { pattern: '[' } }, turn, [turn])).toMatchObject({
+      pass: false,
+      reason: 'regex: "[" isn\'t a valid regular expression (Unterminated character class)',
+    });
+    expect(await check({ regex: { pattern: 'team', flags: 'invalid' } }, turn, [turn])).toMatchObject({
+      pass: false,
+      reason: 'regex: flags "invalid" aren\'t valid regular expression flags',
+    });
+    expect(await check({ regex: { pattern: 'team', flags: 123 } }, turn, [turn])).toMatchObject({ pass: false, reason: 'regex flags must be a string' });
+    for (const bad of [19, null, [], {}, { pattern: 19 }, { pattern: 'team', extra: 1 }]) {
+      expect(await check({ regex: bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'regex takes a pattern string or { pattern, flags }' });
+      expect(await check({ 'not-regex': bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'regex takes a pattern string or { pattern, flags }' });
+    }
+  });
+
+  it('equals compares the trimmed reply with the expected string, and not- inverts it', async () => {
+    expect(await check({ equals: 'The Team plan is $19.' }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply equals "The Team plan is $19."' });
+    const padded: TurnResult = { ...turn, text: '  The Team plan is $19. \n' };
+    expect(await check({ equals: 'The Team plan is $19.' }, padded, [padded])).toMatchObject({ pass: true });
+    expect(await check({ equals: 'The Team plan is $19.\n' }, turn, [turn])).toMatchObject({ pass: true, reason: 'the reply equals "The Team plan is $19.\\n"' });
+    expect(await check({ equals: 'The Team plan is $19.\n' }, padded, [padded])).toMatchObject({ pass: true });
+    expect(await check({ equals: 'The Pro plan is $29.' }, turn, [turn])).toMatchObject({ pass: false, reason: 'the reply doesn\'t equal "The Pro plan is $29."' });
+    expect(await check({ equals: 'The Pro plan is $29.\n' }, turn, [turn])).toMatchObject({ pass: false, reason: 'the reply doesn\'t equal "The Pro plan is $29.\\n"' });
+    expect(await check({ 'not-equals': 'The Pro plan is $29.' }, turn, [turn])).toMatchObject({ pass: true });
+    expect(await check({ 'not-equals': 'The Pro plan is $29.\n' }, turn, [turn])).toMatchObject({ pass: true });
+    expect(await check({ 'not-equals': 'The Team plan is $19.' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply equals "The Team plan is $19."' });
+    expect(await check({ 'not-equals': 'The Team plan is $19.\n' }, turn, [turn])).toMatchObject({ pass: false, reason: 'expected not: the reply equals "The Team plan is $19.\\n"' });
+    for (const bad of [19, null, [], {}, true]) {
+      expect(await check({ equals: bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'equals takes a string' });
+      expect(await check({ 'not-equals': bad }, turn, [turn])).toMatchObject({ pass: false, reason: 'equals takes a string' });
+    }
+  });
+
   it('reports bad values and unknown names as failures', async () => {
     expect(await check({ contains: 19 }, turn, [turn])).toMatchObject({ pass: false, reason: 'contains takes a string or a list of strings' });
     expect(await check({ contains: [] }, turn, [turn])).toMatchObject({ pass: false });

website/src/content/docs/core/testing.mdx

@@ -60,7 +60,14 @@ tests:
           - contains: "$190"
 ```
 
-Each assertion has one key. `contains` takes a string or a list of strings that must all appear in the reply, `judge` has a model grade the reply ([below](#judged-by-a-model)), and every assertion also works with a `not-` prefix to say the opposite. An assertion name that doesn't exist fails the whole run before anything is sent, so a typo can't make a test pass by checking nothing.
+Each assertion has one key. `judge` has a model grade the reply ([below](#judged-by-a-model)), and every assertion also works with a `not-` prefix to say the opposite. An assertion name that doesn't exist fails the whole run before anything is sent, so a typo can't make a test pass by checking nothing.
+
+Four assertions check what the reply says:
+
+- `contains`: takes a string or a list of strings that must all appear in the reply (`contains: "$19"` or `contains: [Team, "$19"]`).
+- `icontains`: case-insensitive substring matching; takes a string or a list of strings (`icontains: team` or `icontains: [team, "$19"]`).
+- `regex`: matches the reply against a JavaScript `RegExp`. Accepts a pattern string (`regex: "\\$\\d+"`) or `{ pattern, flags }` (`regex: { pattern: "team", flags: "i" }`).
+- `equals`: compares the trimmed reply with the exact expected string (`equals: "The Team plan is $19."`).
 
 Three assertions catch a prompt change that makes replies longer, slower or more expensive: