From 5f30f78c4a7a79e081420dd1826feb0e4d148096 Mon Sep 17 00:00:00 2001 From: Scott Lowe Date: Thu, 1 Oct 2026 16:18:17 -0700 Subject: [PATCH] feat(check): fail closed on tools and servers in inline check payloads A PR check runs the branch's agents in a real org, so nothing it sends may reach a real server by default. The payload builder's last pass now: - classifies every tool at a handled position (model.tools, tools:append in overrides, hook do[], same-org knowledge bases in model.toolIds): endCall/dtmf/voicemail/output are sent as written; function tools by function.name and apiRequest by name are mocked; transferCall becomes a mocked dead-server function; handoffs pass only to squad members or inline assistants; everything else (sms, sipRequest, code, mcp, integrations, unknown types) fails the build naming the tool; - fails any tool-bearing key (TOOL_BEARING_KEYS) outside those positions, so a new API field can't slip through unclassified; - replaces servers with https://vapi-gitops-ci.invalid on every assistant and function tool rather than deleting them (a deleted server falls back to the phone number's or org's), clears serverMessages, and dead-ends scenario webhook hooks; - adds a default error mock for every mocked tool to each scenario's toolMocks, replacing disabled ones, and warns about mocks that match no tool; - fails custom knowledge bases, model.knowledgeBaseId, personality tools with side effects, hook transfer actions, and cross-org query or knowledge-base tools. `toolMocks: off` skips the tool rules; `stripWebhooks: false` keeps assistant servers while tool servers are still replaced. Refs TEST-141 Co-Authored-By: Claude Opus 5.5 --- src/check-mocks.ts | 469 ++++++++++++++++++++++++ src/check-payload.ts | 8 + tests/check-mocks.test.ts | 688 ++++++++++++++++++++++++++++++++++++ tests/check-payload.test.ts | 15 +- 4 files changed, 1178 insertions(+), 2 deletions(-) create mode 100644 src/check-mocks.ts create mode 100644 tests/check-mocks.test.ts diff --git a/src/check-mocks.ts b/src/check-mocks.ts new file mode 100644 index 0000000..30e6e8d --- /dev/null +++ b/src/check-mocks.ts @@ -0,0 +1,469 @@ +// The fail-closed tool policy for inline PR checks: the last pass of +// checkPayloadBuild. A check runs the PR's agents in a real org with real +// conversations, so nothing in the payload may reach a real server unless +// the user turned mocks off for a dedicated CI org. +// +// - Every tool at a handled position is classified: left alone (no side +// effect), mocked by name (function, apiRequest, transferCall), or a +// build failure naming it. Unknown types fail. +// - Any tool-bearing key anywhere else fails ("unsupported tool position"), +// so a new API field that carries tools can't slip through unclassified. +// - Servers are replaced with a dead address, never deleted: a deleted +// server falls back to the phone number's or the org's server URL. +// - Default error mocks go in each scenario's toolMocks, never in assistant +// metadata, because handoffs rebuild the assistant. Scenario toolMocks +// apply on the LLM tool-call path; hook-fired tools bypass them, so for +// those the dead server is the safeguard. + +import type { CheckDefinition } from "./check-config.ts"; +import { isObject } from "./check-payload-refs.ts"; +import type { CheckRunBody } from "./check-payload.ts"; + +export const DEAD_SERVER_URL = "https://vapi-gitops-ci.invalid"; + +// Every default mock result starts with this, so a transcript scan can +// report "unmocked tool called". +export const MOCK_MARKER = "vapi-gitops-ci:"; + +// Keys that can carry tools in an assistant, squad or override. Only the +// handled positions below are allowed; anywhere else is a build error. +// Audit this list against the API's OpenAPI schema on each release. +export const TOOL_BEARING_KEYS: readonly string[] = [ + "tools", // handled: model.tools + "tools:append", // handled: inside overrides + "toolIds", // handled: model.toolIds (same-org knowledge bases only) + "toolRefs", + "functions", + "skills", + "declineToolId", + "declineTool", + "forwardingPhoneNumber", + "forwardingPhoneNumbers", + "assistantDestinations", // own rule: checked by the payload builder +]; + +// No external side effect: safe to send as written. +const SIDE_EFFECT_FREE = new Set(["endCall", "dtmf", "voicemail", "output"]); + +// Free-form data, never tool positions: function parameters, judge +// schemas, metadata, variable values, and the mocks themselves. +const DATA_KEYS = new Set([ + "parameters", + "schema", + "metadata", + "variableValues", + "toolMocks", + "evaluations", + "messages", +]); + +const ASSISTANT_KEYS = new Set(["assistant", "transferAssistant"]); +const OVERRIDE_KEYS = new Set([ + "assistantOverrides", + "membersOverrides", + "targetOverrides", +]); + +interface MockWalk { + errors: string[]; + warnings: string[]; + strict: boolean; + stripWebhooks: boolean; + crossOrg: boolean; + members: Set; + mockNames: Set; +} + +type Where = "assistant" | "overrides" | "scenario" | "other"; + +interface WalkArgs { + path: string; + where: Where; + personality: boolean; + // The key this object sits under (`model`, `members`, …). + key: string; +} + +function deadServer(): Record { + return { url: DEAD_SERVER_URL, timeoutSeconds: 1 }; +} + +function serverReplace(node: Record, always: boolean): void { + if (always || node.server !== undefined || node.serverUrl !== undefined) + node.server = deadServer(); + delete node.serverUrl; + delete node.serverUrlSecret; + if (always || node.serverMessages !== undefined) node.serverMessages = []; +} + +function toolName(tool: Record): string | undefined { + const fn = isObject(tool.function) ? tool.function : undefined; + return typeof fn?.name === "string" && fn.name !== "" ? fn.name : undefined; +} + +function handoffCheck( + w: MockWalk, + tool: Record, + at: string, +): void { + const destinations = Array.isArray(tool.destinations) + ? tool.destinations + : []; + destinations.forEach((destination, index) => { + const label = `${at}.destinations[${index}]`; + if (!isObject(destination)) return; + if (destination.type === "assistant" && isObject(destination.assistant)) + return; + // By-ID destinations are the builder's rule; it has already failed them. + if (destination.assistantId !== undefined) return; + const name = destination.assistantName; + if ( + destination.type === "assistant" && + typeof name === "string" && + w.members.has(name) + ) + return; + const kind = + destination.type === "assistant" + ? `assistant ${JSON.stringify(name ?? destination.assistantId)}` + : String(destination.type); + w.errors.push( + `${label}: hands off outside the target (${kind}); only squad members and inline assistants are allowed`, + ); + }); +} + +// Classify one tool at a handled position. Returns the tool to send. +function toolClassify( + w: MockWalk, + tool: unknown, + at: string, + personality: boolean, +): unknown { + if (!isObject(tool)) return tool; + const type = String(tool.type); + if (SIDE_EFFECT_FREE.has(type)) return tool; + if (personality) { + w.errors.push( + `${at}: a personality may only use ${[...SIDE_EFFECT_FREE].join("/")} tools, not ${type}`, + ); + return tool; + } + if (type === "query") { + if (w.crossOrg) + w.errors.push( + `${at}: query tools read the org's own knowledge bases and can't run in another org`, + ); + return tool; + } + if (type === "handoff") { + handoffCheck(w, tool, at); + return tool; + } + if (type === "function") { + const name = toolName(tool); + if (!name) { + w.errors.push(`${at}: a function tool needs function.name to be mocked`); + return tool; + } + serverReplace(tool, true); + delete tool.serverMessages; + w.mockNames.add(name); + return tool; + } + if (type === "apiRequest") { + if (typeof tool.name !== "string" || tool.name === "") { + w.errors.push( + `${at}: an apiRequest tool needs a top-level name to be mocked`, + ); + return tool; + } + tool.url = DEAD_SERVER_URL; + w.mockNames.add(tool.name); + return tool; + } + if (type === "transferCall") { + // The transferCall mock name is unverified, so the transfer becomes a + // dead-server function under its own name; it can never connect. + const name = toolName(tool) ?? "transferCall"; + w.mockNames.add(name); + return { + type: "function", + function: { + name, + description: `${MOCK_MARKER} transfer replaced for the check`, + parameters: { type: "object", properties: {} }, + }, + server: deadServer(), + }; + } + if (type === "knowledgeBase") { + w.errors.push( + `${at}: inline knowledgeBase tools aren't accepted; reference it in model.toolIds`, + ); + return tool; + } + w.errors.push( + `${at}: ${type} tools can't be mocked; remove it, or set toolMocks: off with a dedicated CI org`, + ); + return tool; +} + +function toolListProcess(w: MockWalk, tools: unknown, args: WalkArgs): unknown { + if (!Array.isArray(tools)) return tools; + return tools.map((tool, index) => { + const at = `${args.path}[${index}]`; + const next = toolClassify(w, tool, at, args.personality); + // Inline assistants inside handoff destinations are walked too. + if (isObject(next)) + walkObject(w, next, { ...args, path: at, where: "other", key: "" }); + return next; + }); +} + +function assistantHooksProcess( + w: MockWalk, + hooks: unknown[], + args: WalkArgs, +): void { + hooks.forEach((hook, hookIndex) => { + if (!isObject(hook) || !Array.isArray(hook.do)) return; + hook.do = hook.do.map((action, index) => { + const at = `${args.path}[${hookIndex}].do[${index}]`; + if (!isObject(action)) return action; + if (action.type === "say" || action.type === "message.add") return action; + if (action.type === "tool") { + if (action.toolId !== undefined) { + w.errors.push(`${at}.toolId: hook tools must be inline`); + return action; + } + action.tool = toolClassify( + w, + action.tool, + `${at}.tool`, + args.personality, + ); + return action; + } + if (action.type === "function") { + serverReplace(action, true); + delete action.serverMessages; + return action; + } + w.errors.push( + `${at}: ${String(action.type)} hook actions can't be made safe for a check; remove the hook or set toolMocks: off`, + ); + return action; + }); + }); +} + +function scenarioHooksProcess(hooks: unknown[]): void { + for (const hook of hooks) { + if (!isObject(hook) || !Array.isArray(hook.do)) continue; + for (const action of hook.do) + if (isObject(action) && action.type === "webhook") + action.server = deadServer(); + } +} + +function modelCheck( + w: MockWalk, + model: Record, + path: string, +): void { + if (model.knowledgeBaseId !== undefined) + w.errors.push( + `${path}.knowledgeBaseId: use a knowledgeBase tool in model.toolIds instead`, + ); + const kb = model.knowledgeBase; + if ( + isObject(kb) && + (kb.provider === "custom-knowledge-base" || kb.server !== undefined) + ) + w.errors.push( + `${path}.knowledgeBase: a custom knowledge base calls your server; it can't run in a check`, + ); +} + +function toolPositionHandle( + w: MockWalk, + node: Record, + key: string, + args: WalkArgs, +): void { + const at = `${args.path}.${key}`; + const child = node[key]; + if (key === "tools" && args.key === "model") { + node[key] = toolListProcess(w, child, { ...args, path: at }); + return; + } + if (key === "tools:append" && args.where === "overrides") { + node[key] = toolListProcess(w, child, { ...args, path: at }); + return; + } + if (key === "toolIds" && args.key === "model") { + // Only same-org knowledge bases survive the builder in toolIds. + if (w.crossOrg) + w.errors.push( + `${at}: knowledge base tools are org-local and can't run in another org`, + ); + else + w.warnings.push( + `${at}: knowledge base tools are kept by UUID and query the real knowledge base`, + ); + return; + } + if (key === "assistantDestinations" && args.key === "members") return; + w.errors.push( + `${at}: unsupported tool position; move these tools to model.tools`, + ); +} + +function walkValue(w: MockWalk, value: unknown, args: WalkArgs): void { + if (Array.isArray(value)) { + value.forEach((item, index) => + walkValue(w, item, { ...args, path: `${args.path}[${index}]` }), + ); + return; + } + if (isObject(value)) walkObject(w, value, args); +} + +function walkObject( + w: MockWalk, + node: Record, + args: WalkArgs, +): void { + for (const key of Object.keys(node)) { + const child = node[key]; + const at = `${args.path}.${key}`; + if (DATA_KEYS.has(key)) continue; + if (TOOL_BEARING_KEYS.includes(key)) { + if (w.strict) toolPositionHandle(w, node, key, args); + continue; + } + if (key === "hooks" && Array.isArray(child)) { + if (args.where === "scenario") scenarioHooksProcess(child); + else if (w.strict) assistantHooksProcess(w, child, { ...args, path: at }); + continue; + } + if (ASSISTANT_KEYS.has(key) && isObject(child)) { + if (w.stripWebhooks) serverReplace(child, true); + walkObject(w, child, { ...args, path: at, where: "assistant", key }); + continue; + } + if (OVERRIDE_KEYS.has(key) && isObject(child)) { + if (w.stripWebhooks) serverReplace(child, false); + walkObject(w, child, { ...args, path: at, where: "overrides", key }); + continue; + } + if (key === "model" && isObject(child) && w.strict) + modelCheck(w, child, at); + walkValue(w, child, { ...args, path: at, key }); + } +} + +function defaultMocksAdd( + w: MockWalk, + scenario: Record, + label: string, +): void { + const existing = Array.isArray(scenario.toolMocks) ? scenario.toolMocks : []; + const mocks = existing.filter( + (mock) => + !( + isObject(mock) && + mock.enabled === false && + w.mockNames.has(String(mock.toolName)) + ), + ); + const mocked = new Set( + mocks.filter(isObject).map((mock) => String(mock.toolName)), + ); + for (const name of mocked) + if (!w.mockNames.has(name)) + w.warnings.push( + `${label}.toolMocks: "${name}" matches no mocked tool in the target`, + ); + for (const name of w.mockNames) { + if (mocked.has(name)) continue; + mocks.push({ + toolName: name, + result: JSON.stringify({ + error: `${MOCK_MARKER} ${name} is not mocked in this scenario`, + }), + enabled: true, + }); + } + scenario.toolMocks = mocks; +} + +export interface CheckMocksArgs { + body: CheckRunBody; + check: CheckDefinition; + errors: string[]; + warnings: string[]; +} + +const STOCK_PERSONALITY_RE = /^a0000000-0000-4000-8000-00000000000\d$/; + +export function checkMocksApply(args: CheckMocksArgs): void { + const { body, check } = args; + const squad = body.target.type === "squad" ? body.target.squad : undefined; + const members = new Set(); + for (const member of Array.isArray(squad?.members) ? squad.members : []) + if ( + isObject(member) && + isObject(member.assistant) && + typeof member.assistant.name === "string" + ) + members.add(member.assistant.name); + const w: MockWalk = { + errors: args.errors, + warnings: args.warnings, + strict: check.toolMocks === "strict", + stripWebhooks: check.stripWebhooks, + crossOrg: check.runOrg !== check.org, + members, + mockNames: new Set(), + }; + const base: WalkArgs = { + path: "target", + where: "other", + personality: false, + key: "", + }; + walkObject(w, body.target, base); + body.simulations.forEach((entry, index) => { + const label = `simulations[${index}] (${entry.name})`; + if ("scenarioId" in entry) + w.errors.push(`${label}: scenarios must be inline, not by scenarioId`); + if ( + entry.personalityId !== undefined && + !STOCK_PERSONALITY_RE.test(entry.personalityId) + ) + w.errors.push( + `${label}: personalities must be inline or stock, not ${entry.personalityId}`, + ); + walkObject(w, entry.scenario, { + ...base, + path: `${label}.scenario`, + where: "scenario", + }); + if (entry.personality) + walkObject(w, entry.personality, { + ...base, + path: `${label}.personality`, + personality: true, + }); + }); + if (!w.strict) return; + body.simulations.forEach((entry, index) => + defaultMocksAdd( + w, + entry.scenario, + `simulations[${index}] (${entry.name}).scenario`, + ), + ); +} diff --git a/src/check-payload.ts b/src/check-payload.ts index 6f5c20e..f7f9fe4 100644 --- a/src/check-payload.ts +++ b/src/check-payload.ts @@ -27,6 +27,7 @@ import { refResolve, serverFieldsStrip, } from "./check-payload-refs.ts"; +import { checkMocksApply } from "./check-mocks.ts"; import { credentialForwardMap, replaceCredentialRefs } from "./credentials.ts"; import type { PromotionBindingsResolved } from "./promotion.ts"; import { promotionBindingsApply } from "./promotion.ts"; @@ -523,6 +524,13 @@ export function checkPayloadBuild( }, iterations: input.check.iterations, }; + // Last: the fail-closed tool and server policy (see check-mocks.ts). + checkMocksApply({ + body, + check: input.check, + errors: ctx.errors, + warnings: ctx.warnings, + }); // A backstop for reference fields the builder doesn't handle; run only // when nothing else failed, so a known problem isn't reported twice. if (ctx.errors.length === 0) leftoverReferencesCheck(ctx, body, "body"); diff --git a/tests/check-mocks.test.ts b/tests/check-mocks.test.ts new file mode 100644 index 0000000..aa40094 --- /dev/null +++ b/tests/check-mocks.test.ts @@ -0,0 +1,688 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { checksConfigParse } from "../src/check-config.ts"; +import { + checkMocksApply, + DEAD_SERVER_URL, + MOCK_MARKER, + TOOL_BEARING_KEYS, +} from "../src/check-mocks.ts"; +import type { + CheckRunBody, + CheckSimulationEntry, +} from "../src/check-payload.ts"; + +const DEAD = { url: DEAD_SERVER_URL, timeoutSeconds: 1 }; + +interface ApplyArgs { + target: CheckRunBody["target"]; + scenario?: Record; + personality?: Record; + entry?: Partial & Record; + check?: string; +} + +function apply(args: ApplyArgs): { + body: CheckRunBody; + errors: string[]; + warnings: string[]; +} { + const check = checksConfigParse( + `version: 1\nchecks:\n core:\n org: acme\n targets: [squads/s]\n suites: [core]\n${args.check ?? ""}`, + ).checks.core!; + const body: CheckRunBody = { + simulations: [ + { + type: "simulation", + name: "S1", + scenario: args.scenario ?? { name: "S1", toolMocks: [] }, + ...(args.personality + ? { personality: args.personality } + : { personalityId: "a0000000-0000-4000-8000-000000000001" }), + ...args.entry, + } as CheckSimulationEntry, + ], + target: args.target, + transport: { provider: "vapi.webchat" }, + iterations: 1, + }; + const errors: string[] = []; + const warnings: string[] = []; + checkMocksApply({ body, check, errors, warnings }); + return { body, errors, warnings }; +} + +function assistantTarget( + assistant: Record, +): CheckRunBody["target"] { + return { type: "assistant", assistant }; +} + +function withTools(tools: unknown[]): CheckRunBody["target"] { + return assistantTarget({ name: "A", model: { provider: "openai", tools } }); +} + +function toolsOf(body: CheckRunBody): unknown[] { + const target = body.target; + assert.ok(target.type === "assistant"); + return (target.assistant.model as { tools: unknown[] }).tools; +} + +function errorFor(tool: Record, check = ""): string[] { + return apply({ target: withTools([tool]), check }).errors; +} + +test("side-effect-free tools are sent as written", () => { + const tools = [ + { type: "endCall" }, + { type: "dtmf" }, + { type: "voicemail" }, + { type: "output" }, + ]; + const result = apply({ target: withTools(structuredClone(tools)) }); + assert.deepEqual([result.errors, toolsOf(result.body)], [[], tools]); +}); + +test("function tools get the dead server and a default error mock in every scenario", () => { + const result = apply({ + target: withTools([ + { + type: "function", + function: { name: "book" }, + server: { url: "https://crm.example.com/hook", secret: "s" }, + serverUrl: "https://legacy.example.com", + serverUrlSecret: "x", + }, + ]), + }); + assert.deepEqual( + [ + result.errors, + toolsOf(result.body), + result.body.simulations[0]!.scenario.toolMocks, + ], + [ + [], + [{ type: "function", function: { name: "book" }, server: DEAD }], + [ + { + toolName: "book", + result: JSON.stringify({ + error: `${MOCK_MARKER} book is not mocked in this scenario`, + }), + enabled: true, + }, + ], + ], + ); +}); + +test("scenario mocks are kept, disabled ones are replaced, and mocks for unknown tools warn", () => { + const result = apply({ + target: withTools([ + { type: "function", function: { name: "book" } }, + { type: "function", function: { name: "lookup" } }, + ]), + scenario: { + name: "S1", + toolMocks: [ + { toolName: "book", result: '{"ok":true}', enabled: true }, + { toolName: "lookup", result: '{"stale":true}', enabled: false }, + { toolName: "ghost", result: "{}" }, + ], + }, + }); + assert.deepEqual( + [result.body.simulations[0]!.scenario.toolMocks, result.warnings], + [ + [ + { toolName: "book", result: '{"ok":true}', enabled: true }, + { toolName: "ghost", result: "{}" }, + { + toolName: "lookup", + result: JSON.stringify({ + error: `${MOCK_MARKER} lookup is not mocked in this scenario`, + }), + enabled: true, + }, + ], + [ + 'simulations[0] (S1).scenario.toolMocks: "ghost" matches no mocked tool in the target', + ], + ], + ); +}); + +test("apiRequest tools are mocked by their top-level name and point at the dead host", () => { + const result = apply({ + target: withTools([ + { + type: "apiRequest", + name: "crm_lookup", + method: "POST", + url: "https://crm.example.com/api", + }, + ]), + }); + assert.deepEqual( + [ + toolsOf(result.body), + ( + result.body.simulations[0]!.scenario.toolMocks as Array<{ + toolName: string; + }> + ).map((m) => m.toolName), + ], + [ + [ + { + type: "apiRequest", + name: "crm_lookup", + method: "POST", + url: DEAD_SERVER_URL, + }, + ], + ["crm_lookup"], + ], + ); +}); + +test("transferCall becomes a mocked dead-server function under its own name", () => { + const result = apply({ + target: withTools([ + { + type: "transferCall", + function: { name: "transfer_to_billing" }, + destinations: [{ type: "number", number: "+14155550100" }], + }, + ]), + }); + assert.deepEqual(toolsOf(result.body), [ + { + type: "function", + function: { + name: "transfer_to_billing", + description: `${MOCK_MARKER} transfer replaced for the check`, + parameters: { type: "object", properties: {} }, + }, + server: DEAD, + }, + ]); +}); + +test("every tool that can't be mocked fails the build, naming it", () => { + const cases: Array<[Record, string]> = [ + [ + { type: "function" }, + "target.assistant.model.tools[0]: a function tool needs function.name to be mocked", + ], + [ + { type: "apiRequest", url: "https://x" }, + "target.assistant.model.tools[0]: an apiRequest tool needs a top-level name to be mocked", + ], + [ + { type: "knowledgeBase" }, + "target.assistant.model.tools[0]: inline knowledgeBase tools aren't accepted; reference it in model.toolIds", + ], + ...[ + "sms", + "sipRequest", + "code", + "mcp", + "bash", + "computer", + "textEditor", + "transferCancel", + "transferSuccessful", + "google.calendar.event.create", + "slack.message.send", + "gohighlevel.contact.create", + "ghl", + "make", + "brand-new-type", + ].map((type): [Record, string] => [ + { type }, + `target.assistant.model.tools[0]: ${type} tools can't be mocked; remove it, or set toolMocks: off with a dedicated CI org`, + ]), + ]; + assert.deepEqual( + cases.map(([tool]) => errorFor(tool)), + cases.map(([, message]) => [message]), + ); +}); + +test("query tools run only in their own org", () => { + assert.deepEqual( + [ + errorFor({ type: "query" }), + errorFor({ type: "query" }, " runOrg: acme-ci\n"), + ], + [ + [], + [ + "target.assistant.model.tools[0]: query tools read the org's own knowledge bases and can't run in another org", + ], + ], + ); +}); + +test("knowledge bases kept in model.toolIds warn in the same org and fail in another", () => { + const target = () => + assistantTarget({ + name: "A", + model: { + provider: "openai", + toolIds: ["44444444-4444-4444-8444-444444444444"], + }, + }); + const same = apply({ target: target() }); + const cross = apply({ target: target(), check: " runOrg: acme-ci\n" }); + assert.deepEqual( + [same.errors, same.warnings, cross.errors], + [ + [], + [ + "target.assistant.model.toolIds: knowledge base tools are kept by UUID and query the real knowledge base", + ], + [ + "target.assistant.model.toolIds: knowledge base tools are org-local and can't run in another org", + ], + ], + ); +}); + +test("a tool-bearing key outside a handled position fails, for every key", () => { + const unhandled = TOOL_BEARING_KEYS.filter( + (key) => key !== "assistantDestinations", + ); + const errors = unhandled.map( + (key) => + apply({ target: assistantTarget({ name: "A", [key]: [] }) }).errors, + ); + assert.deepEqual( + errors, + unhandled.map((key) => [ + `target.assistant.${key}: unsupported tool position; move these tools to model.tools`, + ]), + ); +}); + +test("model.functions, model.toolRefs and tools:append outside overrides fail; assistantDestinations outside a member fails", () => { + const result = apply({ + target: assistantTarget({ + name: "A", + model: { provider: "openai", functions: [], toolRefs: [] }, + assistantDestinations: [], + }), + }); + const squad = apply({ + target: { + type: "squad", + squad: { + members: [{ assistant: { name: "A" }, assistantDestinations: [] }], + }, + }, + }); + assert.deepEqual( + [result.errors, squad.errors], + [ + [ + "target.assistant.model.functions: unsupported tool position; move these tools to model.tools", + "target.assistant.model.toolRefs: unsupported tool position; move these tools to model.tools", + "target.assistant.assistantDestinations: unsupported tool position; move these tools to model.tools", + ], + [], + ], + ); +}); + +test("a parameter or schema property named tools is data, not a tool position", () => { + const result = apply({ + target: withTools([ + { + type: "function", + function: { + name: "f", + parameters: { + type: "object", + properties: { + tools: { type: "array" }, + toolIds: { type: "array" }, + }, + }, + }, + }, + ]), + scenario: { + name: "S1", + evaluations: [ + { + structuredOutput: { + name: "j", + schema: { + type: "object", + properties: { tools: { type: "string" } }, + }, + }, + }, + ], + toolMocks: [], + }, + }); + assert.deepEqual(result.errors, []); +}); + +test("handoffs: members and inline assistants pass, everything else fails, and inline assistants are walked", () => { + const handoff = (destinations: unknown[]) => ({ + type: "handoff", + destinations, + }); + const result = apply({ + target: { + type: "squad", + squad: { + members: [ + { + assistant: { + name: "A", + model: { + provider: "openai", + tools: [ + handoff([{ type: "assistant", assistantName: "B" }]), + handoff([ + { + type: "assistant", + assistant: { + name: "Inline", + model: { provider: "openai", tools: [{ type: "sms" }] }, + }, + }, + ]), + handoff([{ type: "assistant", assistantName: "Stranger" }]), + handoff([ + { + type: "dynamic", + server: { url: "https://router.example.com" }, + }, + ]), + handoff([ + { + type: "squad", + squadId: "55555555-5555-4555-8555-555555555555", + }, + ]), + ], + }, + }, + }, + { assistant: { name: "B" } }, + ], + }, + }, + }); + const squad = + result.body.target.type === "squad" ? result.body.target.squad : {}; + const inline = ( + squad.members as Array<{ + assistant: { + model: { + tools: Array<{ + destinations: Array<{ assistant: { server: unknown } }>; + }>; + }; + }; + }> + )[0]!.assistant.model.tools[1]!.destinations[0]!.assistant; + assert.deepEqual( + [result.errors, inline.server], + [ + [ + "target.squad.members[0].assistant.model.tools[1].destinations[0].assistant.model.tools[0]: sms tools can't be mocked; remove it, or set toolMocks: off with a dedicated CI org", + 'target.squad.members[0].assistant.model.tools[2].destinations[0]: hands off outside the target (assistant "Stranger"); only squad members and inline assistants are allowed', + "target.squad.members[0].assistant.model.tools[3].destinations[0]: hands off outside the target (dynamic); only squad members and inline assistants are allowed", + "target.squad.members[0].assistant.model.tools[4].destinations[0]: hands off outside the target (squad); only squad members and inline assistants are allowed", + ], + DEAD, + ], + ); +}); + +test("every assistant gets the dead server and no server messages; overrides only when they set one", () => { + const result = apply({ + target: { + type: "squad", + squad: { + members: [ + { + assistant: { + name: "A", + server: { url: "https://real.example.com" }, + serverMessages: ["tool-calls"], + }, + assistantOverrides: { + server: { url: "https://override.example.com" }, + serverUrl: "https://x", + }, + }, + { + assistant: { name: "B" }, + assistantOverrides: { variableValues: { a: 1 } }, + }, + ], + }, + }, + }); + const squad = + result.body.target.type === "squad" ? result.body.target.squad : {}; + assert.deepEqual(squad.members, [ + { + assistant: { name: "A", server: DEAD, serverMessages: [] }, + assistantOverrides: { server: DEAD }, + }, + { + assistant: { name: "B", server: DEAD, serverMessages: [] }, + assistantOverrides: { variableValues: { a: 1 } }, + }, + ]); +}); + +test("assistant hook actions: say and message.add pass, tools are classified, functions get the dead server, transfers fail", () => { + const result = apply({ + target: assistantTarget({ + name: "A", + hooks: [ + { + on: "call.ending", + do: [ + { type: "say", exact: "Bye" }, + { type: "message.add", message: { role: "system", content: "x" } }, + { + type: "tool", + tool: { + type: "function", + function: { name: "notify" }, + server: { url: "https://real" }, + }, + }, + { type: "tool", tool: { type: "sms" } }, + { type: "tool", toolId: "66666666-6666-4666-8666-666666666666" }, + { + type: "function", + function: { name: "legacy" }, + server: { url: "https://real" }, + }, + { + type: "transfer", + destination: { type: "number", number: "+14155550100" }, + }, + ], + }, + ], + }), + }); + const target = + result.body.target.type === "assistant" ? result.body.target.assistant : {}; + const actions = ( + target.hooks as Array<{ do: Array> }> + )[0]!.do; + assert.deepEqual( + [ + result.errors, + (actions[2]!.tool as { server: unknown }).server, + actions[5]!.server, + ], + [ + [ + "target.assistant.hooks[0].do[3].tool: sms tools can't be mocked; remove it, or set toolMocks: off with a dedicated CI org", + "target.assistant.hooks[0].do[4].toolId: hook tools must be inline", + "target.assistant.hooks[0].do[6]: transfer hook actions can't be made safe for a check; remove the hook or set toolMocks: off", + ], + DEAD, + DEAD, + ], + ); +}); + +test("scenario webhook hooks point at the dead server", () => { + const result = apply({ + target: withTools([]), + scenario: { + name: "S1", + hooks: [ + { + on: "call.ended", + do: [ + { type: "webhook", server: { url: "https://real.example.com" } }, + ], + }, + ], + toolMocks: [], + }, + }); + assert.deepEqual(result.body.simulations[0]!.scenario.hooks, [ + { on: "call.ended", do: [{ type: "webhook", server: DEAD }] }, + ]); +}); + +test("personalities may only use side-effect-free tools, and their assistant gets the dead server", () => { + const result = apply({ + target: withTools([]), + personality: { + name: "Caller", + assistant: { + model: { + provider: "openai", + tools: [ + { type: "endCall" }, + { type: "function", function: { name: "x" } }, + ], + }, + }, + }, + }); + const personality = result.body.simulations[0]!.personality as { + assistant: { server: unknown }; + }; + assert.deepEqual( + [result.errors, personality.assistant.server], + [ + [ + "simulations[0] (S1).personality.assistant.model.tools[1]: a personality may only use endCall/dtmf/voicemail/output tools, not function", + ], + DEAD, + ], + ); +}); + +test("knowledgeBaseId and custom knowledge bases on a model fail", () => { + const result = apply({ + target: assistantTarget({ + name: "A", + model: { + provider: "openai", + knowledgeBaseId: "77777777-7777-4777-8777-777777777777", + knowledgeBase: { + provider: "custom-knowledge-base", + server: { url: "https://kb" }, + }, + }, + }), + }); + assert.deepEqual(result.errors, [ + "target.assistant.model.knowledgeBaseId: use a knowledgeBase tool in model.toolIds instead", + "target.assistant.model.knowledgeBase: a custom knowledge base calls your server; it can't run in a check", + ]); +}); + +test("entries must carry their scenario inline and a stock or inline personality", () => { + const result = apply({ + target: withTools([]), + entry: { + scenarioId: "88888888-8888-4888-8888-888888888888", + personalityId: "99999999-9999-4999-8999-999999999999", + }, + }); + assert.deepEqual(result.errors, [ + "simulations[0] (S1): scenarios must be inline, not by scenarioId", + "simulations[0] (S1): personalities must be inline or stock, not 99999999-9999-4999-8999-999999999999", + ]); +}); + +test("toolMocks off sends tools as written; stripWebhooks still replaces assistant servers", () => { + const tools = [ + { type: "sms" }, + { + type: "function", + function: { name: "f" }, + server: { url: "https://real" }, + }, + ]; + const result = apply({ + target: assistantTarget({ + name: "A", + server: { url: "https://real" }, + model: { provider: "openai", tools: structuredClone(tools) }, + }), + check: " toolMocks: off\n", + }); + const target = + result.body.target.type === "assistant" ? result.body.target.assistant : {}; + assert.deepEqual( + [ + result.errors, + (target.model as { tools: unknown }).tools, + target.server, + result.body.simulations[0]!.scenario.toolMocks, + ], + [[], tools, DEAD, []], + ); +}); + +test("stripWebhooks false keeps assistant servers but strict mocks still replace tool servers", () => { + const result = apply({ + target: assistantTarget({ + name: "A", + server: { url: "https://real" }, + model: { + provider: "openai", + tools: [ + { + type: "function", + function: { name: "f" }, + server: { url: "https://real" }, + }, + ], + }, + }), + check: " stripWebhooks: false\n", + }); + const target = + result.body.target.type === "assistant" ? result.body.target.assistant : {}; + assert.deepEqual( + [ + target.server, + (target.model as { tools: Array<{ server: unknown }> }).tools[0]!.server, + ], + [{ url: "https://real" }, DEAD], + ); +}); diff --git a/tests/check-payload.test.ts b/tests/check-payload.test.ts index 6209e91..7c1e884 100644 --- a/tests/check-payload.test.ts +++ b/tests/check-payload.test.ts @@ -200,7 +200,11 @@ test("toolIds resolve by UUID through the source state, and server fields are st sourceState: { tools: { lookup: { uuid: UUID_A } } }, }); assert.deepEqual((assistantOf(result).model as { tools: unknown }).tools, [ - { type: "function", function: { name: "lookup" } }, + { + type: "function", + function: { name: "lookup" }, + server: { url: "https://vapi-gitops-ci.invalid", timeoutSeconds: 1 }, + }, ]); }); @@ -285,7 +289,14 @@ test("hook do[].toolId becomes an inline tool", async () => { do: [ { type: "tool", - tool: { type: "function", function: { name: "notify" } }, + tool: { + type: "function", + function: { name: "notify" }, + server: { + url: "https://vapi-gitops-ci.invalid", + timeoutSeconds: 1, + }, + }, }, ], },