feat(cli): add calls-tool and no-tool-call assertions (#412)

.changeset/tool-call-assertions.md

@@ -0,0 +1,5 @@
+---
+"@chatcops/cli": minor
+---
+
+`chatcops test` has `calls-tool` and `no-tool-call` assertions, for checking that a turn calls a tool with expected arguments or makes no unexpected tool calls.

packages/cli/src/test/assertions/index.ts

@@ -3,6 +3,7 @@ import { judge } from './judge.js';
 import { javascript } from './javascript.js';
 import { cites, leadCaptured } from './parts.js';
 import { contains } from './text.js';
+import { callsTool, noToolCall } from './tools.js';
 import { costUnder, latencyUnder, tokensUnder } from './usage.js';
 
 /**
@@ -20,6 +21,8 @@ export const ASSERTIONS = new Map<string, AssertionCheck>([
   ['cost-under', costUnder],
   ['cites', cites],
   ['lead-captured', leadCaptured],
+  ['calls-tool', callsTool],
+  ['no-tool-call', noToolCall],
 ]);
 
 /** The base name of an assertion and whether it's negated: `not-contains` is `contains`, negated. */

packages/cli/src/test/assertions/tools.ts

@@ -0,0 +1,88 @@
+import { isDeepStrictEqual } from 'node:util';
+import type { AssertionCheck } from '../types.js';
+
+function isObject(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function matchesPartial(expected: unknown, actual: unknown): boolean {
+  if (Object.is(expected, actual)) return true;
+  if (Array.isArray(expected)) return isDeepStrictEqual(expected, actual);
+  if (isObject(expected)) {
+    if (!isObject(actual)) return false;
+    return Object.entries(expected).every(
+      ([key, val]) => Object.hasOwn(actual, key) && matchesPartial(val, actual[key]),
+    );
+  }
+  return false;
+}
+
+function parseCallsTool(value: unknown): { name: string; args?: Record<string, unknown> } {
+  if (typeof value === 'string' && value.length > 0) {
+    return { name: value };
+  }
+  if (isObject(value)) {
+    const keys = Object.keys(value);
+    const hasValidName = typeof value.name === 'string' && value.name.length > 0;
+    const hasValidArgs = value.args === undefined || isObject(value.args);
+    const allowed = ['name', 'args'];
+    const onlyAllowedKeys = keys.length > 0 && keys.every((key) => allowed.includes(key));
+    if (hasValidName && hasValidArgs && onlyAllowedKeys) {
+      return { name: value.name as string, args: value.args as Record<string, unknown> | undefined };
+    }
+  }
+  throw new Error('calls-tool takes a tool name, or { name, args }');
+}
+
+function parseNoToolCall(value: unknown): true | string {
+  if (value === true) return true;
+  if (typeof value === 'string' && value.length > 0) return value;
+  throw new Error('no-tool-call takes true, or a tool name');
+}
+
+function formatCall(call: { name: string; input: unknown }): string {
+  const args = call.input === undefined ? '' : JSON.stringify(call.input);
+  return `${call.name}(${args})`;
+}
+
+function formatActual(calls: Array<{ name: string; input: unknown }>): string {
+  if (!calls.length) return 'it called no tools';
+  return `it called ${calls.map(formatCall).join(', ')}`;
+}
+
+/**
+ * `calls-tool: "lookup_order"` or `calls-tool: { name: "lookup_order", args: { id: "A-1" } }`.
+ * When `args` is supplied, matches partially against `input`: extra keys are allowed,
+ * nested objects match recursively, and arrays match exactly.
+ */
+export const callsTool: AssertionCheck = (value, turn) => {
+  const { name, args } = parseCallsTool(value);
+  const calls = turn.toolCalls;
+  const matched = calls.some(
+    (call) => call.name === name && (args === undefined || matchesPartial(args, call.input)),
+  );
+  const actual = formatActual(calls);
+  const target = args !== undefined ? `"${name}" with ${JSON.stringify(args)}` : `"${name}"`;
+  return matched
+    ? { pass: true, reason: `the turn called ${target}` }
+    : { pass: false, reason: `the turn didn't call ${target} (${actual})` };
+};
+
+/**
+ * `no-tool-call: true`: passes only when no tools were called in the turn.
+ * `no-tool-call: "lookup_order"`: passes when that specific tool was not called.
+ */
+export const noToolCall: AssertionCheck = (value, turn) => {
+  const target = parseNoToolCall(value);
+  const calls = turn.toolCalls;
+  const actual = formatActual(calls);
+  if (target === true) {
+    return calls.length === 0
+      ? { pass: true, reason: 'the turn called no tools' }
+      : { pass: false, reason: `the turn called ${calls.map(formatCall).join(', ')}` };
+  }
+  const called = calls.some((call) => call.name === target);
+  return !called
+    ? { pass: true, reason: `the turn didn't call "${target}"` }
+    : { pass: false, reason: `the turn called "${target}" (${actual})` };
+};

packages/cli/tests/test-command.test.ts

@@ -177,6 +177,38 @@ tests:
     expect(turn.finishReason).toBe('stop');
   });
 
+  it('verifies tool calls with calls-tool and no-tool-call', async () => {
+    const files = {
+      'evals/tools.chat.yaml': `version: 1
+name: Tool Calls
+defaults:
+  mocks:
+    tools:
+      lookup_order: { success: true, data: { status: "shipped" } }
+tests:
+  - name: asserts tool call and arguments
+    turns:
+      - user: where is my order?
+        expect:
+          - calls-tool: lookup_order
+          - calls-tool: { name: lookup_order, args: { id: "A-1" } }
+          - no-tool-call: other_tool
+          - contains: shipped
+  - name: asserts no tool was called
+    turns:
+      - user: hi
+        expect:
+          - no-tool-call: true
+          - not-calls-tool: lookup_order
+          - contains: Hello
+`,
+    };
+    const { code, stdout } = await run(['test', '--json'], project(files));
+    expect(code).toBe(EXIT.ok);
+    const report = JSON.parse(stdout);
+    expect(report).toMatchObject({ passed: 2, failed: 0 });
+  });
+
   it('fails a test whose turn errors, and stops that run', async () => {
     const dir = project({
       'evals/down.chat.yaml': `version: 1
@@ -466,6 +498,242 @@ describe('assertions', () => {
     }
   });
 
+  it('calls-tool matches by name, partial args, nested args, and arrays exactly', async () => {
+    const withCall: TurnResult = {
+      ...turn,
+      toolCalls: [
+        {
+          name: 'lookup_order',
+          input: {
+            id: 'A-1',
+            status: 'open',
+            details: { priority: 'high', tags: ['urgent', 'billing'] },
+            meta: { extra: true },
+          },
+        },
+      ],
+    };
+
+    // Name-only matching (string or object)
+    expect(await check({ 'calls-tool': 'lookup_order' }, withCall, [withCall])).toMatchObject({
+      pass: true,
+      reason: 'the turn called "lookup_order"',
+    });
+    expect(await check({ 'calls-tool': { name: 'lookup_order' } }, withCall, [withCall])).toMatchObject({
+      pass: true,
+      reason: 'the turn called "lookup_order"',
+    });
+    expect(await check({ 'calls-tool': 'cancel_order' }, withCall, [withCall])).toMatchObject({
+      pass: false,
+    });
+
+    // Partial arguments
+    expect(await check({ 'calls-tool': { name: 'lookup_order', args: { id: 'A-1' } } }, withCall, [withCall])).toMatchObject({
+      pass: true,
+      reason: 'the turn called "lookup_order" with {"id":"A-1"}',
+    });
+    expect(await check({ 'calls-tool': { name: 'lookup_order', args: { id: 'A-1', status: 'open' } } }, withCall, [withCall])).toMatchObject({
+      pass: true,
+    });
+    expect(await check({ 'calls-tool': { name: 'lookup_order', args: { id: 'A-2' } } }, withCall, [withCall])).toMatchObject({
+      pass: false,
+    });
+
+    // Nested arguments
+    expect(
+      await check(
+        { 'calls-tool': { name: 'lookup_order', args: { details: { priority: 'high' } } } },
+        withCall,
+        [withCall],
+      ),
+    ).toMatchObject({ pass: true });
+    expect(
+      await check(
+        { 'calls-tool': { name: 'lookup_order', args: { details: { priority: 'low' } } } },
+        withCall,
+        [withCall],
+      ),
+    ).toMatchObject({ pass: false });
+
+    // Array arguments must match exactly
+    expect(
+      await check(
+        { 'calls-tool': { name: 'lookup_order', args: { details: { tags: ['urgent', 'billing'] } } } },
+        withCall,
+        [withCall],
+      ),
+    ).toMatchObject({ pass: true });
+    expect(
+      await check(
+        { 'calls-tool': { name: 'lookup_order', args: { details: { tags: ['urgent'] } } } },
+        withCall,
+        [withCall],
+      ),
+    ).toMatchObject({ pass: false });
+    expect(
+      await check(
+        { 'calls-tool': { name: 'lookup_order', args: { details: { tags: ['billing', 'urgent'] } } } },
+        withCall,
+        [withCall],
+      ),
+    ).toMatchObject({ pass: false });
+
+    // Arrays containing objects must match elements exactly, not partially
+    const orderWithItems: TurnResult = {
+      ...turn,
+      toolCalls: [{ name: 'checkout', input: { items: [{ sku: 1, qty: 2 }] } }],
+    };
+    expect(
+      await check(
+        { 'calls-tool': { name: 'checkout', args: { items: [{ sku: 1 }] } } },
+        orderWithItems,
+        [orderWithItems],
+      ),
+    ).toMatchObject({ pass: false });
+    expect(
+      await check(
+        { 'calls-tool': { name: 'checkout', args: { items: [{ sku: 1, qty: 2 }] } } },
+        orderWithItems,
+        [orderWithItems],
+      ),
+    ).toMatchObject({ pass: true });
+  });
+
+  it('calls-tool and no-tool-call report mismatches with actionable actual calls', async () => {
+    const withCall: TurnResult = {
+      ...turn,
+      toolCalls: [{ name: 'lookup_order', input: { id: 'A-1' } }],
+    };
+    const multiCall: TurnResult = {
+      ...turn,
+      toolCalls: [
+        { name: 'lookup_order', input: { id: 'A-1' } },
+        { name: 'send_email', input: { to: 'jane@acme.test' } },
+      ],
+    };
+
+    // No calls made
+    expect(await check({ 'calls-tool': 'lookup_order' }, turn, [turn])).toMatchObject({
+      pass: false,
+      reason: 'the turn didn\'t call "lookup_order" (it called no tools)',
+    });
+    expect(await check({ 'calls-tool': { name: 'lookup_order', args: { id: 'A-1' } } }, turn, [turn])).toMatchObject({
+      pass: false,
+      reason: 'the turn didn\'t call "lookup_order" with {"id":"A-1"} (it called no tools)',
+    });
+
+    // Mismatched name or args shows actual calls
+    expect(await check({ 'calls-tool': 'cancel_order' }, withCall, [withCall])).toMatchObject({
+      pass: false,
+      reason: 'the turn didn\'t call "cancel_order" (it called lookup_order({"id":"A-1"}))',
+    });
+    expect(await check({ 'calls-tool': { name: 'lookup_order', args: { id: 'A-2' } } }, withCall, [withCall])).toMatchObject({
+      pass: false,
+      reason: 'the turn didn\'t call "lookup_order" with {"id":"A-2"} (it called lookup_order({"id":"A-1"}))',
+    });
+    expect(await check({ 'calls-tool': 'cancel_order' }, multiCall, [multiCall])).toMatchObject({
+      pass: false,
+      reason: 'the turn didn\'t call "cancel_order" (it called lookup_order({"id":"A-1"}), send_email({"to":"jane@acme.test"}))',
+    });
+  });
+
+  it('no-tool-call handles boolean true and tool name string', async () => {
+    const withCall: TurnResult = {
+      ...turn,
+      toolCalls: [{ name: 'lookup_order', input: { id: 'A-1' } }],
+    };
+
+    // no-tool-call: true passes only when no tools called
+    expect(await check({ 'no-tool-call': true }, turn, [turn])).toMatchObject({
+      pass: true,
+      reason: 'the turn called no tools',
+    });
+    expect(await check({ 'no-tool-call': true }, withCall, [withCall])).toMatchObject({
+      pass: false,
+      reason: 'the turn called lookup_order({"id":"A-1"})',
+    });
+
+    // no-tool-call: string passes when specific tool not called
+    expect(await check({ 'no-tool-call': 'cancel_order' }, withCall, [withCall])).toMatchObject({
+      pass: true,
+      reason: 'the turn didn\'t call "cancel_order"',
+    });
+    expect(await check({ 'no-tool-call': 'lookup_order' }, withCall, [withCall])).toMatchObject({
+      pass: false,
+      reason: 'the turn called "lookup_order" (it called lookup_order({"id":"A-1"}))',
+    });
+    expect(await check({ 'no-tool-call': 'lookup_order' }, turn, [turn])).toMatchObject({
+      pass: true,
+      reason: 'the turn didn\'t call "lookup_order"',
+    });
+  });
+
+  it('tool assertions work with negation (not- prefix)', async () => {
+    const withCall: TurnResult = {
+      ...turn,
+      toolCalls: [{ name: 'lookup_order', input: { id: 'A-1' } }],
+    };
+
+    // not-calls-tool
+    expect(await check({ 'not-calls-tool': 'lookup_order' }, turn, [turn])).toMatchObject({
+      pass: true,
+    });
+    expect(await check({ 'not-calls-tool': 'lookup_order' }, withCall, [withCall])).toMatchObject({
+      pass: false,
+      reason: 'expected not: the turn called "lookup_order"',
+    });
+    expect(await check({ 'not-calls-tool': { name: 'lookup_order', args: { id: 'A-1' } } }, withCall, [withCall])).toMatchObject({
+      pass: false,
+      reason: 'expected not: the turn called "lookup_order" with {"id":"A-1"}',
+    });
+    expect(await check({ 'not-calls-tool': { name: 'lookup_order', args: { id: 'A-2' } } }, withCall, [withCall])).toMatchObject({
+      pass: true,
+    });
+
+    // not-no-tool-call
+    expect(await check({ 'not-no-tool-call': true }, withCall, [withCall])).toMatchObject({
+      pass: true,
+      reason: 'the turn called lookup_order({"id":"A-1"})',
+    });
+    expect(await check({ 'not-no-tool-call': true }, turn, [turn])).toMatchObject({
+      pass: false,
+      reason: 'expected not: the turn called no tools',
+    });
+    expect(await check({ 'not-no-tool-call': 'lookup_order' }, withCall, [withCall])).toMatchObject({
+      pass: true,
+    });
+    expect(await check({ 'not-no-tool-call': 'cancel_order' }, withCall, [withCall])).toMatchObject({
+      pass: false,
+      reason: 'expected not: the turn didn\'t call "cancel_order"',
+    });
+  });
+
+  it('tool assertions reject invalid values with clear errors', async () => {
+    for (const value of [123, false, true, null, [], {}, { name: 123 }, { name: '' }, { args: {} }, { name: 't', args: 123 }, { name: 't', args: [] }, { name: 't', extra: 1 }]) {
+      expect(await check({ 'calls-tool': value }, turn, [turn])).toMatchObject({
+        pass: false,
+        reason: 'calls-tool takes a tool name, or { name, args }',
+      });
+      // not- on invalid value still fails with the same error
+      expect(await check({ 'not-calls-tool': value }, turn, [turn])).toMatchObject({
+        pass: false,
+        reason: 'calls-tool takes a tool name, or { name, args }',
+      });
+    }
+
+    for (const value of [false, 123, null, [], {}, '']) {
+      expect(await check({ 'no-tool-call': value }, turn, [turn])).toMatchObject({
+        pass: false,
+        reason: 'no-tool-call takes true, or a tool name',
+      });
+      // not- on invalid value still fails with the same error
+      expect(await check({ 'not-no-tool-call': value }, turn, [turn])).toMatchObject({
+        pass: false,
+        reason: 'no-tool-call takes true, or a tool name',
+      });
+    }
+  });
+
   it('filters by tag or by name', () => {
     const test = { name: 'Quotes Price', tags: ['pricing'], turns: [{ user: 'x' }] };
     expect(matchesFilter(test, undefined)).toBe(true);