diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 2109c95..85cbb6c 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -2,6 +2,12 @@ name: exact external content cache on: pull_request: branches: [dev] + paths-ignore: + - 'README.md' + - 'evidence/**' + - '.github/workflows/report.yml' + - 'scripts/markdown-review/**' + - 'scripts/publish-review.ts' workflow_dispatch: permissions: contents: read diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index c7ce9c9..900f2ba 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -5,6 +5,8 @@ on: paths-ignore: - 'README.md' - 'evidence/**' + - 'scripts/markdown-review/**' + - 'scripts/publish-review.ts' - '.github/workflows/cache.yml' - '.github/workflows/report.yml' - 'scripts/assert-cache.mjs' @@ -49,7 +51,7 @@ jobs: persist-credentials: false - name: Autonomous analysis and verified installation id: analyze - uses: rootform-dev/action/analyze@aa619a35a674a0be901650379db3b0551ee645ad + uses: rootform-dev/action/analyze@cc457702223d887c9db5da52ba36099ad3e1b4e0 with: version: 0.1.0-pr.117.1 input: fixtures/rootform/examples/playground/shared-data-platform/base/plan.json @@ -62,7 +64,7 @@ jobs: run: node scripts/assert-output.mjs analyze - name: Optional setup reuses exact verified bytes id: setup - uses: rootform-dev/action/setup@aa619a35a674a0be901650379db3b0551ee645ad + uses: rootform-dev/action/setup@cc457702223d887c9db5da52ba36099ad3e1b4e0 - name: Verify CLI version and executable digest shell: bash env: @@ -78,7 +80,7 @@ jobs: run: node -e "const fs=require('node:fs'),crypto=require('node:crypto');fs.appendFileSync(process.env.GITHUB_ENV,'ROOTFORM_PREVIOUS_SHA='+crypto.createHash('sha256').update(fs.readFileSync(process.env.ORIGINAL_FORM)).digest('hex')+'\n')" - name: Reopen the provided Form id: reopen - uses: rootform-dev/action/analyze@aa619a35a674a0be901650379db3b0551ee645ad + uses: rootform-dev/action/analyze@cc457702223d887c9db5da52ba36099ad3e1b4e0 with: input: ${{ steps.analyze.outputs.form }} upload-artifact: 'false' @@ -90,7 +92,7 @@ jobs: run: node scripts/assert-output.mjs analyze - name: Compare Form with independently paired plan id: compare - uses: rootform-dev/action/compare@aa619a35a674a0be901650379db3b0551ee645ad + uses: rootform-dev/action/compare@cc457702223d887c9db5da52ba36099ad3e1b4e0 with: before: ${{ steps.analyze.outputs.form }} after: fixtures/rootform/examples/playground/shared-data-platform/head/plan.json @@ -102,7 +104,7 @@ jobs: run: node scripts/assert-output.mjs compare - name: Evaluate both saved comparison sides id: check - uses: rootform-dev/action/check@aa619a35a674a0be901650379db3b0551ee645ad + uses: rootform-dev/action/check@cc457702223d887c9db5da52ba36099ad3e1b4e0 with: input: ${{ steps.compare.outputs.form }} side: both @@ -115,7 +117,7 @@ jobs: - name: Genuine negative verdict retains all evidence id: negative continue-on-error: true - uses: rootform-dev/action/check@aa619a35a674a0be901650379db3b0551ee645ad + uses: rootform-dev/action/check@cc457702223d887c9db5da52ba36099ad3e1b4e0 with: input: ${{ steps.analyze.outputs.form }} policy-pack: fixtures/negative diff --git a/.github/workflows/report.yml b/.github/workflows/report.yml index 666daba..94ea121 100644 --- a/.github/workflows/report.yml +++ b/.github/workflows/report.yml @@ -27,30 +27,81 @@ jobs: path: fixtures/rootform sparse-checkout: examples/playground/shared-data-platform persist-credentials: false - - name: Integrated analysis and genuine negative gate - id: rootform - continue-on-error: true - uses: rootform-dev/action@aa619a35a674a0be901650379db3b0551ee645ad + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: rootform-dev/action + ref: cc457702223d887c9db5da52ba36099ad3e1b4e0 + path: fixtures/action + sparse-checkout: src + persist-credentials: false + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: '1.26.7' + cache: false + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version: '1.4.0' + - name: Generate actual Form with the published release + id: analyze + uses: rootform-dev/action/analyze@cc457702223d887c9db5da52ba36099ad3e1b4e0 with: version: 0.1.0-pr.117.1 input: fixtures/rootform/examples/playground/shared-data-platform/base/plan.json plan-file: fixtures/rootform/examples/playground/shared-data-platform/base/plan.tfplan + upload-artifact: 'false' + summary: 'false' + - name: Reopen actual Form with the modified public CLI + env: + INPUT_FORM: ${{ steps.analyze.outputs.form }} + GOWORK: 'off' + GOTOOLCHAIN: go1.26.7 + run: | + mkdir -p rendered + go -C scripts/markdown-review run . run "$INPUT_FORM" "$GITHUB_WORKSPACE/rendered/architecture.md" "$GITHUB_WORKSPACE/rendered/form.json" + cmp "$INPUT_FORM" rendered/form.json + - name: Produce actual negative Policy result + id: check + continue-on-error: true + uses: rootform-dev/action/check@cc457702223d887c9db5da52ba36099ad3e1b4e0 + with: + input: ${{ steps.analyze.outputs.form }} policy-pack: fixtures/negative - comment: 'true' - artifact-name: integrated-evidence - - name: Verify integrated union and failed gate evidence + upload-artifact: 'false' + summary: 'false' + - name: Render recorded Policy evidence with the modified public CLI env: - ROOTFORM_OUTPUTS: ${{ toJSON(steps.rootform.outputs) }} - ROOTFORM_EXPECT_ARTIFACT: 'true' - STEP_OUTCOME: ${{ steps.rootform.outcome }} + INPUT_FORM: ${{ steps.analyze.outputs.form }} + POLICY_RESULT: ${{ steps.check.outputs.result }} + GOWORK: 'off' + GOTOOLCHAIN: go1.26.7 + run: | + go -C scripts/markdown-review run . check "$INPUT_FORM" "$POLICY_RESULT" "$GITHUB_WORKSPACE/rendered/policies.md" "$GITHUB_WORKSPACE/rendered/result.json" + cmp "$POLICY_RESULT" rendered/result.json + - name: Inspect and assemble actual outputs + env: + ROOTFORM_OUTPUTS: ${{ toJSON(steps.check.outputs) }} + ROOTFORM_ACTION_SOURCE: fixtures/action + ROOTFORM_INPUT_FORM: ${{ steps.analyze.outputs.form }} + ROOTFORM_ARCHITECTURE: rendered/architecture.md + ROOTFORM_POLICY_REPORT: rendered/policies.md + ROOTFORM_POLICY_RESULT: ${{ steps.check.outputs.result }} + ROOTFORM_SARIF: ${{ steps.check.outputs.sarif }} + ROOTFORM_HTML: ${{ steps.analyze.outputs.html }} + STEP_OUTCOME: ${{ steps.check.outcome }} run: | - node -e "require('node:assert/strict').equal(process.env.STEP_OUTCOME,'failure')" - node scripts/assert-output.mjs analyze node scripts/assert-output.mjs check 1 - - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + bun scripts/publish-review.ts assemble + - name: Retain generated outputs + id: artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - artifact-ids: ${{ steps.rootform.outputs.artifact-id }} - path: downloaded-integrated - merge-multiple: true - - name: Verify exact integrated artifact inventory - run: node scripts/verify-artifact.mjs downloaded-integrated form.json,report.md,explorer.html,result.json,results.sarif + name: actual-markdown-review + path: rendered/ + retention-days: 7 + if-no-files-found: error + - name: Publish exact report to Summary and the single PR comment + env: + ROOTFORM_ACTION_SOURCE: fixtures/action + ROOTFORM_ARTIFACT_URL: ${{ steps.artifact.outputs.artifact-url }} + GITHUB_TOKEN: ${{ github.token }} + run: bun scripts/publish-review.ts publish diff --git a/README.md b/README.md index e58339f..d21b40a 100644 --- a/README.md +++ b/README.md @@ -20,3 +20,14 @@ one serialized root reporter per PR; no fork or privileged-target comment. Fork qualification uses a pull request into the public upstream repository: the integrated Action keeps its Form, reports, artifacts and Summary with the fork's reduced token, while PR commenting is skipped. + +The PR report currently qualifies the public CLI Markdown source at +`a1adfafa66a548b270727cfbce913325e5980f77`. A published binary produces the +actual plan Form and negative Policy result, then a small public CLI embedding +reopens that saved evidence and exports complete Markdown/JSON. Compilation and +Policy evaluation are unavailable in the embedding. A recorded Policy answer +is reused only for its exact Form digest, stage and selection; Form and Policy +result bytes must remain identical. Shared reporting functions +from the exact Action commit publish the generated report to Summary and the +single protected bot comment. This qualifies source presentation, not a new +distributed candidate. The artifact records both sources and report digests. diff --git a/scripts/assert-output.mjs b/scripts/assert-output.mjs index 9c8fa7d..395e94b 100644 --- a/scripts/assert-output.mjs +++ b/scripts/assert-output.mjs @@ -14,6 +14,9 @@ for (const field of expected) { assert.ok(statSync(outputs[field]).size > 0, `${field} file empty`); } execFileSync('rootform', ['validate', 'form', outputs.form], { stdio: 'pipe' }); +const markdown = readFileSync(outputs.report, 'utf8'); +assert.ok(!markdown.includes('Each list above shows at most'), 'Action must request complete Markdown'); +assert.ok(!/\d+ of \d+ (?:entries|evaluations|evidence lines) shown|\(\d+ of \d+ shown\)/.test(markdown), 'Action Markdown must not contain compact previews'); assert.equal(outputs.version, '0.1.0-pr.117.1'); if (expectedCode !== undefined) assert.equal(outputs['exit-code'], expectedCode); if (process.env.ROOTFORM_EXPECT_ARTIFACT === 'true') { diff --git a/scripts/markdown-review/go.mod b/scripts/markdown-review/go.mod new file mode 100644 index 0000000..7f22ccd --- /dev/null +++ b/scripts/markdown-review/go.mod @@ -0,0 +1,11 @@ +module github.com/rootform-dev/action-qualification/markdown-review + +go 1.26.7 + +require github.com/rootform-dev/rootform/cli v0.0.0-20261002160814-a1adfafa66a5 + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/cobra v1.10.2 // indirect + github.com/spf13/pflag v1.0.10 // indirect +) diff --git a/scripts/markdown-review/go.sum b/scripts/markdown-review/go.sum new file mode 100644 index 0000000..8b21ea0 --- /dev/null +++ b/scripts/markdown-review/go.sum @@ -0,0 +1,13 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/rootform-dev/rootform/cli v0.0.0-20261002160814-a1adfafa66a5 h1:asbp+U1A78/NJFmEl3tlWBhakHyZCXZZo7AJfl93bZo= +github.com/rootform-dev/rootform/cli v0.0.0-20261002160814-a1adfafa66a5/go.mod h1:3mpH3CHzXAPAIutuAhZS7ENGerHCZvyDvZDOJ+3RRLk= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/scripts/markdown-review/main.go b/scripts/markdown-review/main.go new file mode 100644 index 0000000..5926916 --- /dev/null +++ b/scripts/markdown-review/main.go @@ -0,0 +1,109 @@ +package main + +import ( + "context" + "fmt" + "io" + "os" + "reflect" + + "github.com/rootform-dev/rootform/cli" + "github.com/rootform-dev/rootform/cli/backend" + "github.com/rootform-dev/rootform/cli/form" + "github.com/rootform-dev/rootform/cli/policyresult" +) + +// Presentation qualification reopens actual saved evidence. It has no compiler +// or evaluator: a recorded Policy answer is reusable only for its exact Form, +// stage and selection. The CLI remains the sole Markdown formatter. +type savedFormBackend struct { + backend.Backend + recorded *policyresult.Result +} + +type savedFormSession struct { + backend.Session + recorded *policyresult.Result +} + +func (b savedFormBackend) Open(context.Context, backend.Selection, io.Writer) backend.Session { + return savedFormSession{recorded: b.recorded} +} + +func (savedFormSession) Compile(context.Context, backend.Export) (backend.Compiled, error) { + return backend.Compiled{}, fmt.Errorf("qualification requires a saved Form; compilation is unavailable") +} + +func (savedFormSession) Presentation(context.Context, *form.InputForm) backend.Presentation { + return backend.Presentation{} +} + +func (s savedFormSession) Policies(context.Context, []string) (backend.PolicySet, error) { + if s.recorded == nil { + return nil, fmt.Errorf("qualification requires an existing Policy result") + } + return recordedPolicies{result: *s.recorded}, nil +} + +type recordedPolicies struct{ result policyresult.Result } + +func (p recordedPolicies) Packs() []backend.Pack { + return []backend.Pack{{Record: p.result.Architectures[0].PolicyPacks[0], Policies: p.result.Selection.Policies}} +} + +func (p recordedPolicies) Evaluate(_ context.Context, input *form.InputForm, stage form.Stage, selected []string) policyresult.Architecture { + digest, err := form.Digest(form.Form{Input: input}) + a := p.result.Architectures[0] + if err != nil || digest != p.result.Form.Digest || a.Digest != digest || stage != a.Stage || !reflect.DeepEqual(selected, p.result.Selection.Policies) { + return policyresult.ArchitectureUnavailable(len(selected), "EVIDENCE_MISMATCH", "recorded Policy answer does not match the Form, stage or selected Policies") + } + return a +} + +func execute() error { + if len(os.Args) < 2 { + return fmt.Errorf("usage: markdown-review run FORM MARKDOWN REOPENED_FORM | check FORM RESULT MARKDOWN REOPENED_RESULT") + } + b := savedFormBackend{} + version := "0.1.0-pr.117.1" + var args []string + expected := 0 + switch { + case os.Args[1] == "run" && len(os.Args) == 5: + args = []string{"run", os.Args[2], "--no-serve", "--details", "-o", os.Args[3], "-o", os.Args[4]} + case os.Args[1] == "check" && len(os.Args) == 6: + data, err := os.ReadFile(os.Args[3]) + if err != nil { + return err + } + r, err := policyresult.Decode(data) + if err != nil { + return err + } + if r.Form == nil || r.Scope != policyresult.ScopeInput || r.Form.Origin != "saved" || len(r.Architectures) != 1 || len(r.Architectures[0].PolicyPacks) != 1 || r.Architectures[0].Side != "" { + return fmt.Errorf("qualification requires one recorded single-architecture check with one Policy Pack") + } + b.recorded, version = &r, r.Generator.Version + expected = policyresult.ExitCode(r.Status) + args = []string{"check", os.Args[2], "--stage", string(r.Architectures[0].Stage), "--details", "-o", os.Args[4], "-o", os.Args[5]} + default: + return fmt.Errorf("invalid presentation qualification arguments") + } + code := cli.Run(cli.Env{ + Args: args, + Stdin: os.Stdin, Stdout: io.Discard, Stderr: os.Stderr, + Getwd: os.Getwd, Getenv: os.Getenv, LookupEnv: os.LookupEnv, + Backend: b, Version: version, + }) + if code != expected { + return fmt.Errorf("CLI returned %d; recorded evidence requires %d", code, expected) + } + return nil +} + +func main() { + if err := execute(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } +} diff --git a/scripts/publish-review.ts b/scripts/publish-review.ts new file mode 100644 index 0000000..56947d3 --- /dev/null +++ b/scripts/publish-review.ts @@ -0,0 +1,71 @@ +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { appendFileSync, copyFileSync, readFileSync, writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const cliSource = 'a1adfafa66a548b270727cfbce913325e5980f77'; +const actionSource = 'cc457702223d887c9db5da52ba36099ad3e1b4e0'; +const source = resolve(process.env.ROOTFORM_ACTION_SOURCE || 'fixtures/action'); +const { combineReports, renderReport, REPORT_MARKER } = await import(pathToFileURL(resolve(source, 'src/report.ts')).href); +const { readGitHubContext, upsertPullRequestComment } = await import(pathToFileURL(resolve(source, 'src/pull-request.ts')).href); +const required = (name: string) => { + const value = process.env[name]; + assert.ok(value, `${name} is required`); + return value; +}; +const digest = (path: string) => createHash('sha256').update(readFileSync(path)).digest('hex'); + +if (process.argv[2] === 'assemble') { + assert.equal(process.env.STEP_OUTCOME, 'failure', 'the negative Policy gate must fail'); + const original = required('ROOTFORM_INPUT_FORM'); + assert.deepEqual(readFileSync('rendered/form.json'), readFileSync(original), 'reopening must preserve exact Form bytes'); + assert.ok(readFileSync('scripts/markdown-review/go.mod', 'utf8').includes(`-${cliSource.slice(0, 12)}`)); + assert.deepEqual(readFileSync('rendered/result.json'), readFileSync(required('ROOTFORM_POLICY_RESULT')), 'Policy rendering must preserve the actual recorded result bytes'); + const report = combineReports([required('ROOTFORM_ARCHITECTURE'), required('ROOTFORM_POLICY_REPORT')], 'rendered'); + assert.equal(report, 'rendered/report.md'); + const markdown = readFileSync(report, 'utf8'); + assert.equal((markdown.match(/^## Rootform$/gm) || []).length, 1); + assert.deepEqual([...markdown.matchAll(/^### (.+)$/gm)].map(match => match[1]), ['Architecture', 'Policies', 'Details']); + assert.equal((markdown.match(/^---$/gm) || []).length, 2); + assert.ok(!/^# |^#####|^#### (?:Uncertainty|Incomplete coverage|Without target|`)/m.test(markdown)); + assert.ok(!markdown.includes('Each list above shows at most')); + assert.ok(!/\d+ of \d+ (?:entries|evaluations|evidence lines) shown|\(\d+ of \d+ shown\)/.test(markdown)); + assert.ok(markdown.includes('> [!CAUTION]\n> **Verdict: VIOLATED**')); + assert.ok(markdown.includes('Instances: 76 added')); + assert.ok(markdown.includes('#### Δ Net change')); + for (const [variable, filename] of [['ROOTFORM_SARIF', 'results.sarif'], ['ROOTFORM_HTML', 'explorer.html']]) { + copyFileSync(required(variable), `rendered/${filename}`); + } + writeFileSync('rendered/provenance.json', JSON.stringify({ + format_version: '1', + review_cli_source: cliSource, + action_source: actionSource, + semantic_cli_release: '0.1.0-pr.117.1', + architecture_report_sha256: digest(required('ROOTFORM_ARCHITECTURE')), + form_sha256: digest('rendered/form.json'), + report_sha256: digest('rendered/report.md'), + policy_report_sha256: digest(required('ROOTFORM_POLICY_REPORT')), + policy_result_sha256: digest('rendered/result.json'), + note: 'Review Markdown is generated by the exact public CLI source from the actual released-binary Form and recorded Policy answer. Form and Policy result bytes remain identical; no compilation or new Policy evaluation occurs in the presentation embedding. This is source report qualification, not a new distributed candidate.' + }, null, 2) + '\n'); +} else if (process.argv[2] === 'publish') { + const context = readGitHubContext(); + const body = renderReport({ + version: `source-${cliSource.slice(0, 7)}`, + markdown: readFileSync('rendered/report.md', 'utf8'), + workflowUrl: context.workflowUrl, + artifactUrl: required('ROOTFORM_ARTIFACT_URL') + }); + appendFileSync(required('GITHUB_STEP_SUMMARY'), `${body}\n`); + if (context.eventName === 'pull_request' && context.pullRequest?.sameRepository) { + const result = await upsertPullRequestComment({ + token: required('GITHUB_TOKEN'), identity: context.pullRequest, + body: `${REPORT_MARKER}\n${body}`, + runId: context.runId, runAttempt: context.runAttempt + }); + console.log(JSON.stringify(result)); + } +} else { + throw new Error('usage: publish-review.ts assemble|publish'); +}