diff --git a/.changeset/tool-call-assertions.md b/.changeset/tool-call-assertions.md new file mode 100644 index 00000000..37ecdb91 --- /dev/null +++ b/.changeset/tool-call-assertions.md @@ -0,0 +1,5 @@ +--- +"@chatcops/cli": minor +--- + +`chatcops test` has `calls-tool` and `no-tool-call` assertions, for checking that a turn calls a tool with expected arguments or makes no unexpected tool calls. diff --git a/packages/cli/src/test/assertions/index.ts b/packages/cli/src/test/assertions/index.ts index 667c1dce..dc5f8f41 100644 --- a/packages/cli/src/test/assertions/index.ts +++ b/packages/cli/src/test/assertions/index.ts @@ -3,6 +3,7 @@ import { judge } from './judge.js'; import { javascript } from './javascript.js'; import { cites, leadCaptured } from './parts.js'; import { contains } from './text.js'; +import { callsTool, noToolCall } from './tools.js'; import { costUnder, latencyUnder, tokensUnder } from './usage.js'; /** @@ -20,6 +21,8 @@ export const ASSERTIONS = new Map([ ['cost-under', costUnder], ['cites', cites], ['lead-captured', leadCaptured], + ['calls-tool', callsTool], + ['no-tool-call', noToolCall], ]); /** The base name of an assertion and whether it's negated: `not-contains` is `contains`, negated. */ diff --git a/packages/cli/src/test/assertions/tools.ts b/packages/cli/src/test/assertions/tools.ts new file mode 100644 index 00000000..9286c6e2 --- /dev/null +++ b/packages/cli/src/test/assertions/tools.ts @@ -0,0 +1,88 @@ +import { isDeepStrictEqual } from 'node:util'; +import type { AssertionCheck } from '../types.js'; + +function isObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function matchesPartial(expected: unknown, actual: unknown): boolean { + if (Object.is(expected, actual)) return true; + if (Array.isArray(expected)) return isDeepStrictEqual(expected, actual); + if (isObject(expected)) { + if (!isObject(actual)) return false; + return Object.entries(expected).every( + ([key, val]) => Object.hasOwn(actual, key) && matchesPartial(val, actual[key]), + ); + } + return false; +} + +function parseCallsTool(value: unknown): { name: string; args?: Record } { + if (typeof value === 'string' && value.length > 0) { + return { name: value }; + } + if (isObject(value)) { + const keys = Object.keys(value); + const hasValidName = typeof value.name === 'string' && value.name.length > 0; + const hasValidArgs = value.args === undefined || isObject(value.args); + const allowed = ['name', 'args']; + const onlyAllowedKeys = keys.length > 0 && keys.every((key) => allowed.includes(key)); + if (hasValidName && hasValidArgs && onlyAllowedKeys) { + return { name: value.name as string, args: value.args as Record | undefined }; + } + } + throw new Error('calls-tool takes a tool name, or { name, args }'); +} + +function parseNoToolCall(value: unknown): true | string { + if (value === true) return true; + if (typeof value === 'string' && value.length > 0) return value; + throw new Error('no-tool-call takes true, or a tool name'); +} + +function formatCall(call: { name: string; input: unknown }): string { + const args = call.input === undefined ? '' : JSON.stringify(call.input); + return `${call.name}(${args})`; +} + +function formatActual(calls: Array<{ name: string; input: unknown }>): string { + if (!calls.length) return 'it called no tools'; + return `it called ${calls.map(formatCall).join(', ')}`; +} + +/** + * `calls-tool: "lookup_order"` or `calls-tool: { name: "lookup_order", args: { id: "A-1" } }`. + * When `args` is supplied, matches partially against `input`: extra keys are allowed, + * nested objects match recursively, and arrays match exactly. + */ +export const callsTool: AssertionCheck = (value, turn) => { + const { name, args } = parseCallsTool(value); + const calls = turn.toolCalls; + const matched = calls.some( + (call) => call.name === name && (args === undefined || matchesPartial(args, call.input)), + ); + const actual = formatActual(calls); + const target = args !== undefined ? `"${name}" with ${JSON.stringify(args)}` : `"${name}"`; + return matched + ? { pass: true, reason: `the turn called ${target}` } + : { pass: false, reason: `the turn didn't call ${target} (${actual})` }; +}; + +/** + * `no-tool-call: true`: passes only when no tools were called in the turn. + * `no-tool-call: "lookup_order"`: passes when that specific tool was not called. + */ +export const noToolCall: AssertionCheck = (value, turn) => { + const target = parseNoToolCall(value); + const calls = turn.toolCalls; + const actual = formatActual(calls); + if (target === true) { + return calls.length === 0 + ? { pass: true, reason: 'the turn called no tools' } + : { pass: false, reason: `the turn called ${calls.map(formatCall).join(', ')}` }; + } + const called = calls.some((call) => call.name === target); + return !called + ? { pass: true, reason: `the turn didn't call "${target}"` } + : { pass: false, reason: `the turn called "${target}" (${actual})` }; +}; diff --git a/packages/cli/tests/test-command.test.ts b/packages/cli/tests/test-command.test.ts index b0eeec74..3152f150 100644 --- a/packages/cli/tests/test-command.test.ts +++ b/packages/cli/tests/test-command.test.ts @@ -177,6 +177,38 @@ tests: expect(turn.finishReason).toBe('stop'); }); + it('verifies tool calls with calls-tool and no-tool-call', async () => { + const files = { + 'evals/tools.chat.yaml': `version: 1 +name: Tool Calls +defaults: + mocks: + tools: + lookup_order: { success: true, data: { status: "shipped" } } +tests: + - name: asserts tool call and arguments + turns: + - user: where is my order? + expect: + - calls-tool: lookup_order + - calls-tool: { name: lookup_order, args: { id: "A-1" } } + - no-tool-call: other_tool + - contains: shipped + - name: asserts no tool was called + turns: + - user: hi + expect: + - no-tool-call: true + - not-calls-tool: lookup_order + - contains: Hello +`, + }; + const { code, stdout } = await run(['test', '--json'], project(files)); + expect(code).toBe(EXIT.ok); + const report = JSON.parse(stdout); + expect(report).toMatchObject({ passed: 2, failed: 0 }); + }); + it('fails a test whose turn errors, and stops that run', async () => { const dir = project({ 'evals/down.chat.yaml': `version: 1 @@ -466,6 +498,242 @@ describe('assertions', () => { } }); + it('calls-tool matches by name, partial args, nested args, and arrays exactly', async () => { + const withCall: TurnResult = { + ...turn, + toolCalls: [ + { + name: 'lookup_order', + input: { + id: 'A-1', + status: 'open', + details: { priority: 'high', tags: ['urgent', 'billing'] }, + meta: { extra: true }, + }, + }, + ], + }; + + // Name-only matching (string or object) + expect(await check({ 'calls-tool': 'lookup_order' }, withCall, [withCall])).toMatchObject({ + pass: true, + reason: 'the turn called "lookup_order"', + }); + expect(await check({ 'calls-tool': { name: 'lookup_order' } }, withCall, [withCall])).toMatchObject({ + pass: true, + reason: 'the turn called "lookup_order"', + }); + expect(await check({ 'calls-tool': 'cancel_order' }, withCall, [withCall])).toMatchObject({ + pass: false, + }); + + // Partial arguments + expect(await check({ 'calls-tool': { name: 'lookup_order', args: { id: 'A-1' } } }, withCall, [withCall])).toMatchObject({ + pass: true, + reason: 'the turn called "lookup_order" with {"id":"A-1"}', + }); + expect(await check({ 'calls-tool': { name: 'lookup_order', args: { id: 'A-1', status: 'open' } } }, withCall, [withCall])).toMatchObject({ + pass: true, + }); + expect(await check({ 'calls-tool': { name: 'lookup_order', args: { id: 'A-2' } } }, withCall, [withCall])).toMatchObject({ + pass: false, + }); + + // Nested arguments + expect( + await check( + { 'calls-tool': { name: 'lookup_order', args: { details: { priority: 'high' } } } }, + withCall, + [withCall], + ), + ).toMatchObject({ pass: true }); + expect( + await check( + { 'calls-tool': { name: 'lookup_order', args: { details: { priority: 'low' } } } }, + withCall, + [withCall], + ), + ).toMatchObject({ pass: false }); + + // Array arguments must match exactly + expect( + await check( + { 'calls-tool': { name: 'lookup_order', args: { details: { tags: ['urgent', 'billing'] } } } }, + withCall, + [withCall], + ), + ).toMatchObject({ pass: true }); + expect( + await check( + { 'calls-tool': { name: 'lookup_order', args: { details: { tags: ['urgent'] } } } }, + withCall, + [withCall], + ), + ).toMatchObject({ pass: false }); + expect( + await check( + { 'calls-tool': { name: 'lookup_order', args: { details: { tags: ['billing', 'urgent'] } } } }, + withCall, + [withCall], + ), + ).toMatchObject({ pass: false }); + + // Arrays containing objects must match elements exactly, not partially + const orderWithItems: TurnResult = { + ...turn, + toolCalls: [{ name: 'checkout', input: { items: [{ sku: 1, qty: 2 }] } }], + }; + expect( + await check( + { 'calls-tool': { name: 'checkout', args: { items: [{ sku: 1 }] } } }, + orderWithItems, + [orderWithItems], + ), + ).toMatchObject({ pass: false }); + expect( + await check( + { 'calls-tool': { name: 'checkout', args: { items: [{ sku: 1, qty: 2 }] } } }, + orderWithItems, + [orderWithItems], + ), + ).toMatchObject({ pass: true }); + }); + + it('calls-tool and no-tool-call report mismatches with actionable actual calls', async () => { + const withCall: TurnResult = { + ...turn, + toolCalls: [{ name: 'lookup_order', input: { id: 'A-1' } }], + }; + const multiCall: TurnResult = { + ...turn, + toolCalls: [ + { name: 'lookup_order', input: { id: 'A-1' } }, + { name: 'send_email', input: { to: 'jane@acme.test' } }, + ], + }; + + // No calls made + expect(await check({ 'calls-tool': 'lookup_order' }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'the turn didn\'t call "lookup_order" (it called no tools)', + }); + expect(await check({ 'calls-tool': { name: 'lookup_order', args: { id: 'A-1' } } }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'the turn didn\'t call "lookup_order" with {"id":"A-1"} (it called no tools)', + }); + + // Mismatched name or args shows actual calls + expect(await check({ 'calls-tool': 'cancel_order' }, withCall, [withCall])).toMatchObject({ + pass: false, + reason: 'the turn didn\'t call "cancel_order" (it called lookup_order({"id":"A-1"}))', + }); + expect(await check({ 'calls-tool': { name: 'lookup_order', args: { id: 'A-2' } } }, withCall, [withCall])).toMatchObject({ + pass: false, + reason: 'the turn didn\'t call "lookup_order" with {"id":"A-2"} (it called lookup_order({"id":"A-1"}))', + }); + expect(await check({ 'calls-tool': 'cancel_order' }, multiCall, [multiCall])).toMatchObject({ + pass: false, + reason: 'the turn didn\'t call "cancel_order" (it called lookup_order({"id":"A-1"}), send_email({"to":"jane@acme.test"}))', + }); + }); + + it('no-tool-call handles boolean true and tool name string', async () => { + const withCall: TurnResult = { + ...turn, + toolCalls: [{ name: 'lookup_order', input: { id: 'A-1' } }], + }; + + // no-tool-call: true passes only when no tools called + expect(await check({ 'no-tool-call': true }, turn, [turn])).toMatchObject({ + pass: true, + reason: 'the turn called no tools', + }); + expect(await check({ 'no-tool-call': true }, withCall, [withCall])).toMatchObject({ + pass: false, + reason: 'the turn called lookup_order({"id":"A-1"})', + }); + + // no-tool-call: string passes when specific tool not called + expect(await check({ 'no-tool-call': 'cancel_order' }, withCall, [withCall])).toMatchObject({ + pass: true, + reason: 'the turn didn\'t call "cancel_order"', + }); + expect(await check({ 'no-tool-call': 'lookup_order' }, withCall, [withCall])).toMatchObject({ + pass: false, + reason: 'the turn called "lookup_order" (it called lookup_order({"id":"A-1"}))', + }); + expect(await check({ 'no-tool-call': 'lookup_order' }, turn, [turn])).toMatchObject({ + pass: true, + reason: 'the turn didn\'t call "lookup_order"', + }); + }); + + it('tool assertions work with negation (not- prefix)', async () => { + const withCall: TurnResult = { + ...turn, + toolCalls: [{ name: 'lookup_order', input: { id: 'A-1' } }], + }; + + // not-calls-tool + expect(await check({ 'not-calls-tool': 'lookup_order' }, turn, [turn])).toMatchObject({ + pass: true, + }); + expect(await check({ 'not-calls-tool': 'lookup_order' }, withCall, [withCall])).toMatchObject({ + pass: false, + reason: 'expected not: the turn called "lookup_order"', + }); + expect(await check({ 'not-calls-tool': { name: 'lookup_order', args: { id: 'A-1' } } }, withCall, [withCall])).toMatchObject({ + pass: false, + reason: 'expected not: the turn called "lookup_order" with {"id":"A-1"}', + }); + expect(await check({ 'not-calls-tool': { name: 'lookup_order', args: { id: 'A-2' } } }, withCall, [withCall])).toMatchObject({ + pass: true, + }); + + // not-no-tool-call + expect(await check({ 'not-no-tool-call': true }, withCall, [withCall])).toMatchObject({ + pass: true, + reason: 'the turn called lookup_order({"id":"A-1"})', + }); + expect(await check({ 'not-no-tool-call': true }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'expected not: the turn called no tools', + }); + expect(await check({ 'not-no-tool-call': 'lookup_order' }, withCall, [withCall])).toMatchObject({ + pass: true, + }); + expect(await check({ 'not-no-tool-call': 'cancel_order' }, withCall, [withCall])).toMatchObject({ + pass: false, + reason: 'expected not: the turn didn\'t call "cancel_order"', + }); + }); + + it('tool assertions reject invalid values with clear errors', async () => { + for (const value of [123, false, true, null, [], {}, { name: 123 }, { name: '' }, { args: {} }, { name: 't', args: 123 }, { name: 't', args: [] }, { name: 't', extra: 1 }]) { + expect(await check({ 'calls-tool': value }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'calls-tool takes a tool name, or { name, args }', + }); + // not- on invalid value still fails with the same error + expect(await check({ 'not-calls-tool': value }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'calls-tool takes a tool name, or { name, args }', + }); + } + + for (const value of [false, 123, null, [], {}, '']) { + expect(await check({ 'no-tool-call': value }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'no-tool-call takes true, or a tool name', + }); + // not- on invalid value still fails with the same error + expect(await check({ 'not-no-tool-call': value }, turn, [turn])).toMatchObject({ + pass: false, + reason: 'no-tool-call takes true, or a tool name', + }); + } + }); + it('filters by tag or by name', () => { const test = { name: 'Quotes Price', tags: ['pricing'], turns: [{ user: 'x' }] }; expect(matchesFilter(test, undefined)).toBe(true); diff --git a/website/src/content/docs/core/testing.mdx b/website/src/content/docs/core/testing.mdx index 09c48dfc..fe7b952a 100644 --- a/website/src/content/docs/core/testing.mdx +++ b/website/src/content/docs/core/testing.mdx @@ -73,6 +73,11 @@ Two assertions check what ChatCops adds to a reply: - `cites` takes a URL glob, or a list of them, and each has to match a page the reply cites (`cites: "https://acme.test/pricing*"`). Globs work the same as in `ingest.include`. `cites: true` means it cites at least one page. - `lead-captured: true` means the turn captured a lead. An object, such as `lead-captured: { email: jane@acme.test }`, also has to match those fields of the lead. +Two assertions check tool calls made during a turn: + +- `calls-tool` asserts that the bot called a tool. It takes either a tool name (`calls-tool: lookup_order`) or `{ name, args }` to also check inputs (`calls-tool: { name: lookup_order, args: { id: "A-1001" } }`). Arguments match partially: every specified key must match, nested objects follow the same rule, extra input keys are allowed, and arrays must match exactly. +- `no-tool-call: true` asserts that no tool was called during the turn. It also takes a tool name (`no-tool-call: cancel_order`) to assert that the specific tool was not called, even if other tools were called. + A test can also set: - `pageContext`: the page the visitor is on (`url`, `title`, and optionally `description` and `contentSnippet`) @@ -97,6 +102,9 @@ tests: turns: - user: Where is order A-1001? expect: + - calls-tool: lookup_order + - calls-tool: { name: lookup_order, args: { id: "A-1001" } } + - no-tool-call: cancel_order - contains: shipped ```