Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,23 @@ const event = toEvaluationAuditEvent(artifact, {
});
```

Frameworks with their own report contract can bind that native report without translating
or exposing its private fields:

```typescript
const artifact = await createEvaluationReceiptFromNativeReport(
nativeReport,
{
runId,
runner: 'my-agent-framework@2.0.0',
startedAt,
finishedAt,
summary: publicAggregate,
},
receiptOptions,
);
```

The artifact contains aggregate metrics and content digests, not raw prompts, expected
outputs, or failure transcripts. Its SHA-256 digest can also be placed in a signed Trust
Bundle as `eval:<name>`, allowing the same eval evidence to be independently verified and
Expand Down
2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion packages/evals/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "@stackbilt/evals",
"version": "0.2.0",
"version": "0.2.1",
"description": "Edge-native evaluation framework for AI agents and classifiers. Zero runtime dependencies beyond Zod.",
"type": "module",
"main": "./dist/index.js",
Expand Down
3 changes: 3 additions & 0 deletions packages/evals/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,16 +20,19 @@ export {
export {
DigestSchema,
EvaluationGateSchema,
EvaluationReportBindingSchema,
EvaluationReceiptSchema,
EvaluationReceiptArtifactSchema,
canonicalStringify,
sha256Canonical,
createEvaluationReceipt,
createEvaluationReceiptFromNativeReport,
verifyEvaluationReceiptArtifact,
toEvaluationAuditEvent,
} from './receipt.js';
export type {
EvaluationGate,
EvaluationReportBinding,
EvaluationReceipt,
EvaluationReceiptArtifact,
CreateEvaluationReceiptOptions,
Expand Down
33 changes: 33 additions & 0 deletions packages/evals/src/receipt.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest';
import {
canonicalStringify,
createEvaluationReceipt,
createEvaluationReceiptFromNativeReport,
sha256Canonical,
toEvaluationAuditEvent,
verifyEvaluationReceiptArtifact,
Expand Down Expand Up @@ -65,6 +66,38 @@ describe('evaluation receipts', () => {
expect(await verifyEvaluationReceiptArtifact(tampered)).toBe(false);
});

it('binds a framework-native report without publishing its private fields', async () => {
const nativeReport = {
evalName: 'intent-classify',
timestamp: '2026-08-23T12:00:00.000Z',
scores: [{ caseId: 'private-case', transcript: 'do not publish' }],
};
const artifact = await createEvaluationReceiptFromNativeReport(nativeReport, {
runId: report().run_id,
runner: 'tarotscript@1.0.0',
startedAt: report().started_at,
finishedAt: report().finished_at,
summary: report().summary,
}, options);

expect(artifact.receipt.report.digest).toBe(await sha256Canonical(nativeReport));
expect(JSON.stringify(artifact)).not.toContain('private-case');
expect(JSON.stringify(artifact)).not.toContain('do not publish');

const changed = await createEvaluationReceiptFromNativeReport(
{ ...nativeReport, timestamp: '2026-08-23T12:00:03.000Z' },
{
runId: report().run_id,
runner: 'tarotscript@1.0.0',
startedAt: report().started_at,
finishedAt: report().finished_at,
summary: report().summary,
},
options,
);
expect(changed.receipt.report.digest).not.toBe(artifact.receipt.report.digest);
});

it('keeps the deployment gate explicit rather than deriving it from pass rate', async () => {
const blocked = await createEvaluationReceipt(report(), {
...options,
Expand Down
42 changes: 36 additions & 6 deletions packages/evals/src/receipt.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@ export const EvaluationGateSchema = z.object({
reasons: z.array(z.string().min(1)).default([]),
});

/** Public projection required to bind a framework-native private report. */
export const EvaluationReportBindingSchema = z.object({
runId: z.string().uuid(),
runner: z.string().min(1),
startedAt: z.string().datetime(),
finishedAt: z.string().datetime(),
summary: EvalRunReportSchema.shape.summary,
});

export const EvaluationReceiptSchema = z.object({
schema_version: z.literal('evaluation-receipt.v1'),
receipt_id: z.string().uuid(),
Expand Down Expand Up @@ -50,6 +59,7 @@ export const EvaluationReceiptArtifactSchema = z.object({
});

export type EvaluationGate = z.infer<typeof EvaluationGateSchema>;
export type EvaluationReportBinding = z.infer<typeof EvaluationReportBindingSchema>;
export type EvaluationReceipt = z.infer<typeof EvaluationReceiptSchema>;
export type EvaluationReceiptArtifact = z.infer<typeof EvaluationReceiptArtifactSchema>;

Expand Down Expand Up @@ -107,7 +117,27 @@ export async function createEvaluationReceipt(
options: CreateEvaluationReceiptOptions,
): Promise<EvaluationReceiptArtifact> {
const report = EvalRunReportSchema.parse(rawReport);
const reportDigest = await sha256Canonical(report);

return createEvaluationReceiptFromNativeReport(report, {
runId: report.run_id,
runner: report.runner,
startedAt: report.started_at,
finishedAt: report.finished_at,
summary: report.summary,
}, options);
}

/**
* Bind a framework-native private report to the standard public receipt shape.
* The native report is hashed as-is and is never copied into the receipt.
*/
export async function createEvaluationReceiptFromNativeReport(
rawReport: unknown,
publicBinding: EvaluationReportBinding,
options: CreateEvaluationReceiptOptions,
): Promise<EvaluationReceiptArtifact> {
const binding = EvaluationReportBindingSchema.parse(publicBinding);
const reportDigest = await sha256Canonical(rawReport);

const receipt = EvaluationReceiptSchema.parse({
schema_version: 'evaluation-receipt.v1',
Expand All @@ -125,13 +155,13 @@ export async function createEvaluationReceipt(
artifact_digest: options.subject.artifactDigest,
},
report: {
run_id: report.run_id,
run_id: binding.runId,
digest: reportDigest,
started_at: report.started_at,
finished_at: report.finished_at,
summary: report.summary,
started_at: binding.startedAt,
finished_at: binding.finishedAt,
summary: binding.summary,
},
runner: report.runner,
runner: binding.runner,
gate: options.gate,
evidence_refs: options.evidenceRefs,
});
Expand Down
Loading