diff --git a/README.md b/README.md index 1346a56..6264eeb 100644 --- a/README.md +++ b/README.md @@ -148,6 +148,9 @@ const artifact = await createEvaluationReceiptFromNativeReport( ); ``` +Latency percentiles are optional in a receipt projection. Omit measurements the native +framework did not collect; do not use zero as a stand-in for unknown evidence. + The artifact contains aggregate metrics and content digests, not raw prompts, expected outputs, or failure transcripts. Its SHA-256 digest can also be placed in a signed Trust Bundle as `eval:`, allowing the same eval evidence to be independently verified and diff --git a/package-lock.json b/package-lock.json index 62248d6..aa9eb05 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1263,7 +1263,7 @@ }, "packages/evals": { "name": "@stackbilt/evals", - "version": "0.2.1", + "version": "0.2.2", "license": "Apache-2.0", "dependencies": { "zod": "^3.23.0" diff --git a/packages/evals/package.json b/packages/evals/package.json index 60dfbe6..df3b80c 100644 --- a/packages/evals/package.json +++ b/packages/evals/package.json @@ -1,6 +1,6 @@ { "name": "@stackbilt/evals", - "version": "0.2.1", + "version": "0.2.2", "description": "Edge-native evaluation framework for AI agents and classifiers. Zero runtime dependencies beyond Zod.", "type": "module", "main": "./dist/index.js", diff --git a/packages/evals/src/index.ts b/packages/evals/src/index.ts index 5be59b4..f5703b0 100644 --- a/packages/evals/src/index.ts +++ b/packages/evals/src/index.ts @@ -20,6 +20,7 @@ export { export { DigestSchema, EvaluationGateSchema, + EvaluationReceiptSummarySchema, EvaluationReportBindingSchema, EvaluationReceiptSchema, EvaluationReceiptArtifactSchema, @@ -32,6 +33,7 @@ export { } from './receipt.js'; export type { EvaluationGate, + EvaluationReceiptSummary, EvaluationReportBinding, EvaluationReceipt, EvaluationReceiptArtifact, diff --git a/packages/evals/src/receipt.test.ts b/packages/evals/src/receipt.test.ts index 159bd2d..a97b585 100644 --- a/packages/evals/src/receipt.test.ts +++ b/packages/evals/src/receipt.test.ts @@ -98,6 +98,24 @@ describe('evaluation receipts', () => { expect(changed.receipt.report.digest).not.toBe(artifact.receipt.report.digest); }); + it('omits latency rather than fabricating unmeasured percentiles', async () => { + const { p50_latency_ms, p90_latency_ms, p95_latency_ms, ...summaryWithoutLatency } = report().summary; + const artifact = await createEvaluationReceiptFromNativeReport( + { framework: 'native', result: 'pass' }, + { + runId: report().run_id, + runner: 'native-framework@1.0.0', + startedAt: report().started_at, + finishedAt: report().finished_at, + summary: summaryWithoutLatency, + }, + options, + ); + + expect(artifact.receipt.report.summary).not.toHaveProperty('p50_latency_ms'); + expect(p50_latency_ms + p90_latency_ms + p95_latency_ms).toBeGreaterThan(0); + }); + it('keeps the deployment gate explicit rather than deriving it from pass rate', async () => { const blocked = await createEvaluationReceipt(report(), { ...options, diff --git a/packages/evals/src/receipt.ts b/packages/evals/src/receipt.ts index 1e076cc..5051180 100644 --- a/packages/evals/src/receipt.ts +++ b/packages/evals/src/receipt.ts @@ -13,13 +13,23 @@ export const EvaluationGateSchema = z.object({ reasons: z.array(z.string().min(1)).default([]), }); +/** + * Public aggregate shared across eval frameworks. Latency percentiles are + * optional because a producer must omit measurements it did not collect. + */ +export const EvaluationReceiptSummarySchema = EvalRunReportSchema.shape.summary.extend({ + p50_latency_ms: z.number().nonnegative().optional(), + p90_latency_ms: z.number().nonnegative().optional(), + p95_latency_ms: z.number().nonnegative().optional(), +}); + /** Public projection required to bind a framework-native private report. */ export const EvaluationReportBindingSchema = z.object({ runId: z.string().uuid(), runner: z.string().min(1), startedAt: z.string().datetime(), finishedAt: z.string().datetime(), - summary: EvalRunReportSchema.shape.summary, + summary: EvaluationReceiptSummarySchema, }); export const EvaluationReceiptSchema = z.object({ @@ -42,7 +52,7 @@ export const EvaluationReceiptSchema = z.object({ digest: DigestSchema, started_at: z.string().datetime(), finished_at: z.string().datetime(), - summary: EvalRunReportSchema.shape.summary, + summary: EvaluationReceiptSummarySchema, }), runner: z.string().min(1), gate: EvaluationGateSchema, @@ -59,6 +69,7 @@ export const EvaluationReceiptArtifactSchema = z.object({ }); export type EvaluationGate = z.infer; +export type EvaluationReceiptSummary = z.infer; export type EvaluationReportBinding = z.infer; export type EvaluationReceipt = z.infer; export type EvaluationReceiptArtifact = z.infer;