From a43e9d335e8b1bfbf8de413a2a8863a4a6afa6fa Mon Sep 17 00:00:00 2001 From: Kurt Overmier Date: Sun, 23 Aug 2026 10:49:53 -0500 Subject: [PATCH] feat: bind framework-native eval reports --- README.md | 17 ++++++++++++ package-lock.json | 2 +- packages/evals/package.json | 2 +- packages/evals/src/index.ts | 3 +++ packages/evals/src/receipt.test.ts | 33 +++++++++++++++++++++++ packages/evals/src/receipt.ts | 42 +++++++++++++++++++++++++----- 6 files changed, 91 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 400a7c5..1346a56 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,23 @@ const event = toEvaluationAuditEvent(artifact, { }); ``` +Frameworks with their own report contract can bind that native report without translating +or exposing its private fields: + +```typescript +const artifact = await createEvaluationReceiptFromNativeReport( + nativeReport, + { + runId, + runner: 'my-agent-framework@2.0.0', + startedAt, + finishedAt, + summary: publicAggregate, + }, + receiptOptions, +); +``` + The artifact contains aggregate metrics and content digests, not raw prompts, expected outputs, or failure transcripts. Its SHA-256 digest can also be placed in a signed Trust Bundle as `eval:`, allowing the same eval evidence to be independently verified and diff --git a/package-lock.json b/package-lock.json index 8861836..62248d6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1263,7 +1263,7 @@ }, "packages/evals": { "name": "@stackbilt/evals", - "version": "0.2.0", + "version": "0.2.1", "license": "Apache-2.0", "dependencies": { "zod": "^3.23.0" diff --git a/packages/evals/package.json b/packages/evals/package.json index e6d7145..60dfbe6 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -1,6 +1,6 @@ { "name": "@stackbilt/evals", - "version": "0.2.0", + "version": "0.2.1", "description": "Edge-native evaluation framework for AI agents and classifiers. Zero runtime dependencies beyond Zod.", "type": "module", "main": "./dist/index.js", diff --git a/packages/evals/src/index.ts b/packages/evals/src/index.ts index ce66a6b..5be59b4 100644 --- a/packages/evals/src/index.ts +++ b/packages/evals/src/index.ts @@ -20,16 +20,19 @@ export { export { DigestSchema, EvaluationGateSchema, + EvaluationReportBindingSchema, EvaluationReceiptSchema, EvaluationReceiptArtifactSchema, canonicalStringify, sha256Canonical, createEvaluationReceipt, + createEvaluationReceiptFromNativeReport, verifyEvaluationReceiptArtifact, toEvaluationAuditEvent, } from './receipt.js'; export type { EvaluationGate, + EvaluationReportBinding, EvaluationReceipt, EvaluationReceiptArtifact, CreateEvaluationReceiptOptions, diff --git a/packages/evals/src/receipt.test.ts b/packages/evals/src/receipt.test.ts index ae065b7..159bd2d 100644 --- a/packages/evals/src/receipt.test.ts +++ b/packages/evals/src/receipt.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'; import { canonicalStringify, createEvaluationReceipt, + createEvaluationReceiptFromNativeReport, sha256Canonical, toEvaluationAuditEvent, verifyEvaluationReceiptArtifact, @@ -65,6 +66,38 @@ describe('evaluation receipts', () => { expect(await verifyEvaluationReceiptArtifact(tampered)).toBe(false); }); + it('binds a framework-native report without publishing its private fields', async () => { + const nativeReport = { + evalName: 'intent-classify', + timestamp: '2026-08-23T12:00:00.000Z', + scores: [{ caseId: 'private-case', transcript: 'do not publish' }], + }; + const artifact = await createEvaluationReceiptFromNativeReport(nativeReport, { + runId: report().run_id, + runner: 'tarotscript@1.0.0', + startedAt: report().started_at, + finishedAt: report().finished_at, + summary: report().summary, + }, options); + + expect(artifact.receipt.report.digest).toBe(await sha256Canonical(nativeReport)); + expect(JSON.stringify(artifact)).not.toContain('private-case'); + expect(JSON.stringify(artifact)).not.toContain('do not publish'); + + const changed = await createEvaluationReceiptFromNativeReport( + { ...nativeReport, timestamp: '2026-08-23T12:00:03.000Z' }, + { + runId: report().run_id, + runner: 'tarotscript@1.0.0', + startedAt: report().started_at, + finishedAt: report().finished_at, + summary: report().summary, + }, + options, + ); + expect(changed.receipt.report.digest).not.toBe(artifact.receipt.report.digest); + }); + it('keeps the deployment gate explicit rather than deriving it from pass rate', async () => { const blocked = await createEvaluationReceipt(report(), { ...options, diff --git a/packages/evals/src/receipt.ts b/packages/evals/src/receipt.ts index e99d462..1e076cc 100644 --- a/packages/evals/src/receipt.ts +++ b/packages/evals/src/receipt.ts @@ -13,6 +13,15 @@ export const EvaluationGateSchema = z.object({ reasons: z.array(z.string().min(1)).default([]), }); +/** Public projection required to bind a framework-native private report. */ +export const EvaluationReportBindingSchema = z.object({ + runId: z.string().uuid(), + runner: z.string().min(1), + startedAt: z.string().datetime(), + finishedAt: z.string().datetime(), + summary: EvalRunReportSchema.shape.summary, +}); + export const EvaluationReceiptSchema = z.object({ schema_version: z.literal('evaluation-receipt.v1'), receipt_id: z.string().uuid(), @@ -50,6 +59,7 @@ export const EvaluationReceiptArtifactSchema = z.object({ }); export type EvaluationGate = z.infer; +export type EvaluationReportBinding = z.infer; export type EvaluationReceipt = z.infer; export type EvaluationReceiptArtifact = z.infer; @@ -107,7 +117,27 @@ export async function createEvaluationReceipt( options: CreateEvaluationReceiptOptions, ): Promise { const report = EvalRunReportSchema.parse(rawReport); - const reportDigest = await sha256Canonical(report); + + return createEvaluationReceiptFromNativeReport(report, { + runId: report.run_id, + runner: report.runner, + startedAt: report.started_at, + finishedAt: report.finished_at, + summary: report.summary, + }, options); +} + +/** + * Bind a framework-native private report to the standard public receipt shape. + * The native report is hashed as-is and is never copied into the receipt. + */ +export async function createEvaluationReceiptFromNativeReport( + rawReport: unknown, + publicBinding: EvaluationReportBinding, + options: CreateEvaluationReceiptOptions, +): Promise { + const binding = EvaluationReportBindingSchema.parse(publicBinding); + const reportDigest = await sha256Canonical(rawReport); const receipt = EvaluationReceiptSchema.parse({ schema_version: 'evaluation-receipt.v1', @@ -125,13 +155,13 @@ export async function createEvaluationReceipt( artifact_digest: options.subject.artifactDigest, }, report: { - run_id: report.run_id, + run_id: binding.runId, digest: reportDigest, - started_at: report.started_at, - finished_at: report.finished_at, - summary: report.summary, + started_at: binding.startedAt, + finished_at: binding.finishedAt, + summary: binding.summary, }, - runner: report.runner, + runner: binding.runner, gate: options.gate, evidence_refs: options.evidenceRefs, });