diff --git a/src/check-config.ts b/src/check-config.ts new file mode 100644 index 0000000..14f0f99 --- /dev/null +++ b/src/check-config.ts @@ -0,0 +1,295 @@ +// Parser for `vapi-checks.yml`, the PR check configuration. +// +// Config-free and pure (apart from checksConfigLoad's one read), so the +// check CLI, the PR workflow and the promotion gate share it. It is a +// separate file from promotion.yml because promotionConfigParse requires a +// pipeline of two or more orgs, which single-org customers don't have. +// +// Every key is validated and unknown keys are rejected: a typo in a CI +// config should fail loudly, not silently run fewer tests. + +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { parse as parseYaml } from "yaml"; +import type { PromotionBindings } from "./promotion.ts"; +import { promotionBindingsParse, SLUG_RE } from "./promotion.ts"; + +export const CHECKS_CONFIG_FILE = "vapi-checks.yml"; + +// chat → vapi.webchat, voice → vapi.websocket. +export type CheckTransport = "chat" | "voice"; + +// strict: every tool is mocked or the build fails (see check-mocks.ts). +// off: tools are sent as written — only safe in a dedicated CI org. +export type CheckToolMocks = "strict" | "off"; + +export type CheckTargetType = "assistants" | "squads"; + +export interface CheckTarget { + type: CheckTargetType; + id: string; +} + +export interface CheckRunSettings { + transport: CheckTransport; + iterations: number; + timeoutMinutes: number; + toolMocks: CheckToolMocks; + stripWebhooks: boolean; +} + +export interface CheckDefinition extends CheckRunSettings { + name: string; + // Resources are read from resources// at the checked-out commit. + org: string; + // The org the run executes in; its state supplies credential UUIDs. + runOrg: string; + baseUrl?: string; + targets: CheckTarget[]; + suites: string[]; + simulations: string[]; + bindings: PromotionBindings; + // Extra globs (repo-relative) that mark this check affected. + paths: string[]; +} + +export interface ChecksConfig { + version: 1; + checks: Record; +} + +export const CHECK_DEFAULTS: CheckRunSettings = { + transport: "chat", + iterations: 1, + timeoutMinutes: 20, + toolMocks: "strict", + stripWebhooks: true, +}; + +const MAX_ITERATIONS = 10; +const MAX_TIMEOUT_MINUTES = 120; +const TOP_LEVEL_KEYS = ["version", "defaults", "checks"]; +const SETTINGS_KEYS = Object.keys(CHECK_DEFAULTS); +const CHECK_KEYS = [ + "org", + "runOrg", + "baseUrl", + "targets", + "suites", + "simulations", + "bindings", + "paths", + ...SETTINGS_KEYS, +]; +const TARGET_TYPES: readonly CheckTargetType[] = ["assistants", "squads"]; + +function mapping(value: unknown, label: string): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) + throw new Error(`${label} must be a mapping`); + return Object.fromEntries(Object.entries(value)); +} + +function keysAllowed( + raw: Record, + allowed: readonly string[], + label: string, +): void { + for (const key of Object.keys(raw)) { + if (key === "mode") + throw new Error( + `${label}.mode is not supported: checks always build the target from the branch's files`, + ); + if (!allowed.includes(key)) + throw new Error( + `${label} has unknown key "${key}" (allowed: ${allowed.join(", ")})`, + ); + } +} + +function slug(value: unknown, label: string): string { + if (typeof value !== "string" || !SLUG_RE.test(value)) + throw new Error(`${label} must be a lowercase slug (a-z, 0-9, -)`); + return value; +} + +// A resource ID as it appears in a file path under resources///, +// without the extension. Nested IDs (`team/intake`) are allowed. +function resourceId(value: unknown, label: string): string { + if (typeof value !== "string" || value.length === 0) + throw new Error(`${label} must be a non-empty resource ID`); + if (/\.(ya?ml|md|ts)$/.test(value)) + throw new Error(`${label} must not include a file extension: ${value}`); + const parts = value.split("/"); + if (parts.some((part) => part === "" || part === "." || part === "..")) + throw new Error(`${label} must be a relative resource ID: ${value}`); + return value; +} + +function stringList(value: unknown, label: string): unknown[] { + if (value === undefined) return []; + if (!Array.isArray(value)) throw new Error(`${label} must be a list`); + return value; +} + +function unique(values: T[], key: (value: T) => string, label: string): T[] { + const seen = new Set(); + for (const value of values) { + const id = key(value); + if (seen.has(id)) throw new Error(`${label} lists ${id} twice`); + seen.add(id); + } + return values; +} + +function target(value: unknown, label: string): CheckTarget { + if (typeof value !== "string") + throw new Error(`${label} must be "assistants/" or "squads/"`); + const slash = value.indexOf("/"); + const type = value.slice(0, slash); + if (slash < 0 || !TARGET_TYPES.includes(type as CheckTargetType)) + throw new Error( + `${label} must be "assistants/" or "squads/", got "${value}"`, + ); + return { + type: type as CheckTargetType, + id: resourceId(value.slice(slash + 1), label), + }; +} + +function settings( + raw: Record, + base: CheckRunSettings, + label: string, +): CheckRunSettings { + const result = { ...base }; + if (raw.transport !== undefined) { + if (raw.transport !== "chat" && raw.transport !== "voice") + throw new Error(`${label}.transport must be "chat" or "voice"`); + result.transport = raw.transport; + } + if (raw.iterations !== undefined) { + const iterations = raw.iterations; + if ( + typeof iterations !== "number" || + !Number.isInteger(iterations) || + iterations < 1 || + iterations > MAX_ITERATIONS + ) + throw new Error( + `${label}.iterations must be an integer from 1 to ${MAX_ITERATIONS}`, + ); + result.iterations = iterations; + } + if (raw.timeoutMinutes !== undefined) { + const timeout = raw.timeoutMinutes; + if ( + typeof timeout !== "number" || + !(timeout > 0) || + timeout > MAX_TIMEOUT_MINUTES + ) + throw new Error( + `${label}.timeoutMinutes must be a number above 0 and at most ${MAX_TIMEOUT_MINUTES}`, + ); + result.timeoutMinutes = timeout; + } + if (raw.toolMocks !== undefined) { + if (raw.toolMocks !== "strict" && raw.toolMocks !== "off") + throw new Error(`${label}.toolMocks must be "strict" or "off"`); + result.toolMocks = raw.toolMocks; + } + if (raw.stripWebhooks !== undefined) { + if (typeof raw.stripWebhooks !== "boolean") + throw new Error(`${label}.stripWebhooks must be true or false`); + result.stripWebhooks = raw.stripWebhooks; + } + return result; +} + +function baseUrl(value: unknown, label: string): string | undefined { + if (value === undefined) return undefined; + if (typeof value !== "string" || !/^https?:\/\/[^\s/]+/.test(value)) + throw new Error(`${label}.baseUrl must be an http(s) URL`); + return value.replace(/\/+$/, ""); +} + +function check( + name: string, + value: unknown, + defaults: CheckRunSettings, +): CheckDefinition { + const label = `checks.${name}`; + const raw = mapping(value, label); + keysAllowed(raw, CHECK_KEYS, label); + const org = slug(raw.org, `${label}.org`); + const runOrg = + raw.runOrg === undefined ? org : slug(raw.runOrg, `${label}.runOrg`); + const targets = unique( + stringList(raw.targets, `${label}.targets`).map((item, index) => + target(item, `${label}.targets[${index}]`), + ), + (item) => `${item.type}/${item.id}`, + `${label}.targets`, + ); + if (targets.length === 0) + throw new Error(`${label}.targets must list at least one target`); + const suites = unique( + stringList(raw.suites, `${label}.suites`).map((item, index) => + resourceId(item, `${label}.suites[${index}]`), + ), + (item) => item, + `${label}.suites`, + ); + const simulations = unique( + stringList(raw.simulations, `${label}.simulations`).map((item, index) => + resourceId(item, `${label}.simulations[${index}]`), + ), + (item) => item, + `${label}.simulations`, + ); + if (suites.length === 0 && simulations.length === 0) + throw new Error(`${label} must list at least one suite or simulation`); + const paths = stringList(raw.paths, `${label}.paths`).map((item, index) => { + if (typeof item !== "string" || item.length === 0) + throw new Error(`${label}.paths[${index}] must be a non-empty glob`); + return item; + }); + return { + name, + org, + runOrg, + baseUrl: baseUrl(raw.baseUrl, label), + targets, + suites, + simulations, + bindings: promotionBindingsParse(raw.bindings), + paths, + ...settings(raw, defaults, label), + }; +} + +export function checksConfigParse(content: string): ChecksConfig { + const raw = mapping(parseYaml(content), CHECKS_CONFIG_FILE); + keysAllowed(raw, TOP_LEVEL_KEYS, CHECKS_CONFIG_FILE); + if (raw.version !== 1) + throw new Error(`${CHECKS_CONFIG_FILE} version must be 1`); + const defaultsRaw = + raw.defaults === undefined ? {} : mapping(raw.defaults, "defaults"); + keysAllowed(defaultsRaw, SETTINGS_KEYS, "defaults"); + const defaults = settings(defaultsRaw, CHECK_DEFAULTS, "defaults"); + const checksRaw = mapping(raw.checks, "checks"); + if (Object.keys(checksRaw).length === 0) + throw new Error(`${CHECKS_CONFIG_FILE} must declare at least one check`); + const checks: Record = {}; + for (const [name, value] of Object.entries(checksRaw)) { + slug(name, `check name "${name}"`); + checks[name] = check(name, value, defaults); + } + return { version: 1, checks }; +} + +// Returns null when the repo has no vapi-checks.yml: checks are opt-in. +export function checksConfigLoad(rootDir: string): ChecksConfig | null { + const path = join(rootDir, CHECKS_CONFIG_FILE); + if (!existsSync(path)) return null; + return checksConfigParse(readFileSync(path, "utf8")); +} diff --git a/tests/check-config.test.ts b/tests/check-config.test.ts new file mode 100644 index 0000000..c5ba4ae --- /dev/null +++ b/tests/check-config.test.ts @@ -0,0 +1,282 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; +import { + CHECK_DEFAULTS, + checksConfigLoad, + checksConfigParse, +} from "../src/check-config.ts"; + +const EXAMPLE = fileURLToPath( + new URL("../vapi-checks.example.yml", import.meta.url), +); + +function errorOf(content: string): string { + try { + checksConfigParse(content); + return "no error"; + } catch (error) { + return (error as Error).message; + } +} + +const MINIMAL = `version: 1 +checks: + core: + org: acme + targets: [squads/main-squad] + suites: [core] +`; + +test("a minimal check gets the defaults, runs in its own org, and binds credentials", () => { + assert.deepEqual(checksConfigParse(MINIMAL), { + version: 1, + checks: { + core: { + name: "core", + org: "acme", + runOrg: "acme", + baseUrl: undefined, + targets: [{ type: "squads", id: "main-squad" }], + suites: ["core"], + simulations: [], + bindings: { + credentials: { default: "bind", aliases: {} }, + phoneNumbers: { default: "omit", aliases: {} }, + }, + paths: [], + ...CHECK_DEFAULTS, + }, + }, + }); +}); + +test("check settings override defaults, which override the built-in defaults", () => { + const config = checksConfigParse(`version: 1 +defaults: + transport: voice + iterations: 3 +checks: + a: + org: acme + targets: [assistants/team/intake] + simulations: [books-calm] + iterations: 2 + toolMocks: off + stripWebhooks: false + timeoutMinutes: 7.5 + b: + org: acme + runOrg: acme-ci + baseUrl: https://api.eu.vapi.ai/ + targets: [assistants/intake, squads/main] + suites: [core] + bindings: { credentials: { default: omit, crm: bind } } + paths: ["prompts/**"] +`); + const { a, b } = config.checks; + assert.deepEqual( + { + a: [ + a!.transport, + a!.iterations, + a!.toolMocks, + a!.stripWebhooks, + a!.timeoutMinutes, + a!.targets, + ], + b: [ + b!.transport, + b!.iterations, + b!.runOrg, + b!.baseUrl, + b!.targets.length, + b!.bindings.credentials, + b!.paths, + ], + }, + { + a: [ + "voice", + 2, + "off", + false, + 7.5, + [{ type: "assistants", id: "team/intake" }], + ], + b: [ + "voice", + 3, + "acme-ci", + "https://api.eu.vapi.ai", + 2, + { default: "omit", aliases: { crm: "bind" } }, + ["prompts/**"], + ], + }, + ); +}); + +test("every invalid shape is rejected with a message naming the field", () => { + const check = (body: string) => + `version: 1\nchecks:\n core:\n org: acme\n${body}`; + const withTargets = (body: string) => + check(` targets: [squads/main]\n${body}`); + const cases: Array<[string, string]> = [ + ["", "vapi-checks.yml must be a mapping"], + ["version: 2\nchecks: {}\n", "vapi-checks.yml version must be 1"], + [ + "version: 1\nchecks: {}\n", + "vapi-checks.yml must declare at least one check", + ], + ["version: 1\nchecks: []\n", "checks must be a mapping"], + [ + "version: 1\npipelines: {}\nchecks: {}\n", + 'vapi-checks.yml has unknown key "pipelines" (allowed: version, defaults, checks)', + ], + [ + "version: 1\ndefaults:\n org: acme\nchecks: {}\n", + 'defaults has unknown key "org" (allowed: transport, iterations, timeoutMinutes, toolMocks, stripWebhooks)', + ], + [ + "version: 1\nchecks:\n Core:\n org: acme\n", + 'check name "Core" must be a lowercase slug (a-z, 0-9, -)', + ], + [ + withTargets(" suites: [core]\n mode: deployed\n"), + "checks.core.mode is not supported: checks always build the target from the branch's files", + ], + [ + withTargets(" suites: [core]\n suite: core\n"), + 'checks.core has unknown key "suite" (allowed: org, runOrg, baseUrl, targets, suites, simulations, bindings, paths, transport, iterations, timeoutMinutes, toolMocks, stripWebhooks)', + ], + [ + "version: 1\nchecks:\n core:\n targets: [squads/main]\n suites: [core]\n", + "checks.core.org must be a lowercase slug (a-z, 0-9, -)", + ], + [ + withTargets(" suites: [core]\n runOrg: Acme_CI\n"), + "checks.core.runOrg must be a lowercase slug (a-z, 0-9, -)", + ], + [ + check(" suites: [core]\n"), + "checks.core.targets must list at least one target", + ], + [ + check(" targets: squads/main\n suites: [core]\n"), + "checks.core.targets must be a list", + ], + [ + check(" targets: [tools/lookup]\n suites: [core]\n"), + 'checks.core.targets[0] must be "assistants/" or "squads/", got "tools/lookup"', + ], + [ + check(" targets: [main-squad]\n suites: [core]\n"), + 'checks.core.targets[0] must be "assistants/" or "squads/", got "main-squad"', + ], + [ + check(" targets: [squads/main.yml]\n suites: [core]\n"), + "checks.core.targets[0] must not include a file extension: main.yml", + ], + [ + check(" targets: [squads/../other]\n suites: [core]\n"), + "checks.core.targets[0] must be a relative resource ID: ../other", + ], + [ + check(" targets: [squads/main, squads/main]\n suites: [core]\n"), + "checks.core.targets lists squads/main twice", + ], + [withTargets(""), "checks.core must list at least one suite or simulation"], + [ + withTargets(" suites: []\n simulations: []\n"), + "checks.core must list at least one suite or simulation", + ], + [ + withTargets(" suites: [core, core]\n"), + "checks.core.suites lists core twice", + ], + [ + withTargets(" suites: [1]\n"), + "checks.core.suites[0] must be a non-empty resource ID", + ], + [ + withTargets(" suites: [core]\n transport: phone\n"), + 'checks.core.transport must be "chat" or "voice"', + ], + [ + withTargets(" suites: [core]\n iterations: 11\n"), + "checks.core.iterations must be an integer from 1 to 10", + ], + [ + withTargets(" suites: [core]\n iterations: 1.5\n"), + "checks.core.iterations must be an integer from 1 to 10", + ], + [ + withTargets(" suites: [core]\n timeoutMinutes: 0\n"), + "checks.core.timeoutMinutes must be a number above 0 and at most 120", + ], + [ + withTargets(" suites: [core]\n toolMocks: loose\n"), + 'checks.core.toolMocks must be "strict" or "off"', + ], + [ + withTargets(" suites: [core]\n stripWebhooks: 'no'\n"), + "checks.core.stripWebhooks must be true or false", + ], + [ + withTargets(" suites: [core]\n baseUrl: api.vapi.ai\n"), + "checks.core.baseUrl must be an http(s) URL", + ], + [ + withTargets(" suites: [core]\n paths: ['']\n"), + "checks.core.paths[0] must be a non-empty glob", + ], + [ + withTargets( + " suites: [core]\n bindings: { credentials: { default: maybe } }\n", + ), + 'bindings.credentials.default must be "bind" or "omit"', + ], + ]; + assert.deepEqual( + cases.map(([content]) => errorOf(content)), + cases.map(([, message]) => message), + ); +}); + +test("the shipped example parses", () => { + const config = checksConfigParse(readFileSync(EXAMPLE, "utf8")); + assert.deepEqual(Object.keys(config.checks), ["core"]); +}); + +test("the example's commented CI-org check parses once uncommented", () => { + const example = readFileSync(EXAMPLE, "utf8"); + const start = example.indexOf(" # staging-core:"); + const uncommented = + example.slice(0, start) + example.slice(start).replace(/^ # /gm, " "); + const config = checksConfigParse(uncommented); + assert.deepEqual( + [ + config.checks["staging-core"]!.runOrg, + config.checks["staging-core"]!.toolMocks, + ], + ["example-ci", "off"], + ); +}); + +test("checksConfigLoad returns null without a vapi-checks.yml and parses one when present", () => { + const root = mkdtempSync(join(tmpdir(), "check-config-")); + try { + const missing = checksConfigLoad(root); + writeFileSync(join(root, "vapi-checks.yml"), MINIMAL); + assert.deepEqual( + [missing, Object.keys(checksConfigLoad(root)!.checks)], + [null, ["core"]], + ); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/vapi-checks.example.yml b/vapi-checks.example.yml new file mode 100644 index 0000000..5c50b9d --- /dev/null +++ b/vapi-checks.example.yml @@ -0,0 +1,43 @@ +version: 1 + +# Copy to vapi-checks.yml to turn on simulation checks. Each check builds its +# targets and tests from resources// on the PR branch and sends them +# inline in one simulation run per target: nothing is deployed, and nothing +# needs cleaning up afterwards. + +# Applies to every check; any check can override these. +defaults: + transport: chat # chat (default, cheaper) or voice + iterations: 1 # runs of each simulation, 1-10 + timeoutMinutes: 20 # per run; a run past this is canceled and reported incomplete + toolMocks: strict # strict: every tool call returns a mock, or the build fails naming the tool. + # off: tools are sent as written. Only use with a dedicated CI org (runOrg). + stripWebhooks: true # point assistant servers at a dead address; false keeps them + # (tool servers are always replaced while toolMocks is strict) + +checks: + core: + org: example-dev # reads resources/example-dev/ + targets: # one run per target + - squads/main-squad + suites: # resources/example-dev/simulations/suites/ + - core + simulations: [] # optional: extra resources/example-dev/simulations/tests/ + + # A check that runs in a separate CI org. Credentials are bound by name, + # the same way promotion.yml binds them, using .vapi-state..json. + # staging-core: + # org: example-staging + # runOrg: example-ci + # baseUrl: https://api.vapi.ai # set https://api.eu.vapi.ai for EU orgs + # targets: [assistants/support-intake] + # suites: [core] + # toolMocks: off + # bindings: + # credentials: + # default: bind + # analytics-api: omit + # phoneNumbers: + # default: omit + # paths: # extra files that make this check run when changed + # - prompts/shared/**