From dffae7ad89e308d47719f29766f50671f7e67f76 Mon Sep 17 00:00:00 2001 From: Adam Getchell Date: Wed, 28 Oct 2026 18:58:26 -0700 Subject: [PATCH] refactor: adopt shared release orchestration Replace the three release adapters with published research-repo-tools 0.2.4 commands and declarative policies. Run evidence and asset commands in isolated Python outside the observed checkout, preserving frozen post-approval revalidation before OIDC. Make performance release consume saved preparation; tag fetching is explicit and README publication remains independently retryable. Preparation positional tags move to the shared declaration, and evidence --identity JSON becomes --expected PATH. Retain native policy, workflow transport and scientific publication regressions. Closes #304 --- .github/workflows/ci.yml | 17 +- .github/workflows/publish-crates.yml | 3 +- docs/BENCHMARKING.md | 26 +- docs/RELEASING.md | 49 ++- docs/code_organization.md | 19 +- docs/dev/performance-tooling-adoption.md | 28 +- justfile | 49 ++- pyproject.toml | 2 + tests/tooling/test_changelog_integration.py | 12 +- .../tooling/test_justfile_discoverability.py | 2 +- tests/tooling/test_performance_workflow.py | 52 ++- tests/tooling/test_release_changelog.py | 115 ++---- tests/tooling/test_release_evidence.py | 381 ++++++++++-------- tests/tooling/test_release_performance.py | 190 ++++----- tests/tooling/test_release_publishing.py | 95 +++-- tests/tooling/test_toolchain_integration.py | 30 ++ tooling/README.md | 42 +- tooling/performance-release.toml | 11 + tooling/release-assets.toml | 3 + tooling/release-evidence.toml | 31 ++ tooling/release_changelog.py | 70 ---- tooling/release_evidence.py | 261 ------------ tooling/release_performance.py | 129 ------ 23 files changed, 666 insertions(+), 951 deletions(-) create mode 100644 tooling/performance-release.toml create mode 100644 tooling/release-assets.toml create mode 100644 tooling/release-evidence.toml delete mode 100644 tooling/release_changelog.py delete mode 100644 tooling/release_evidence.py delete mode 100644 tooling/release_performance.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 62b7546..e8ff4a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,7 +90,22 @@ jobs: - name: Capture successful native validation identities env: GH_TOKEN: ${{ github.token }} - run: uv run --locked python -m tooling.release_evidence seal --output target/release-evidence/validation.json + run: | + uvx --python 3.15 --from research-repo-tools==0.2.4 python -I - <<'PY' + import os + import sys + from pathlib import Path + from research_repo_tools.cli import main + from research_repo_tools.release_metadata import read_package_info + root = Path.cwd() + sys.exit(main([ + "release", "evidence", "create", "tooling/release-evidence.toml", + "--checkout", str(root), "--commit", os.environ["GITHUB_SHA"], + "--version", read_package_info(root).version, + "--run-id", os.environ["GITHUB_RUN_ID"], "--attempt", os.environ["GITHUB_RUN_ATTEMPT"], + "--output", "target/release-evidence/validation.json", + ])) + PY - name: Sign original main validation evidence id: validation uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index c28fffc..af7698a 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -87,7 +87,8 @@ jobs: set -euo pipefail just release-gate "$RELEASE_TAG" test -n "$CI_EVIDENCE" - just release-evidence --identity "$CI_EVIDENCE" + printf '%s\n' "$CI_EVIDENCE" > "$RUNNER_TEMP/approved-evidence.json" + just release-evidence --expected "$RUNNER_TEMP/approved-evidence.json" uv run --locked --no-sync research-repo-tools release registry "$RELEASE_TAG" --expect absent - name: Obtain a short-lived crates.io token diff --git a/docs/BENCHMARKING.md b/docs/BENCHMARKING.md index 7f82f3c..90bd54d 100644 --- a/docs/BENCHMARKING.md +++ b/docs/BENCHMARKING.md @@ -24,13 +24,15 @@ the commands measure and where their outputs go. ## Start Here -Run `just performance-prepare` before measurements to fetch the release tags. -This explicit Git operation and the measurement recipes' temporary worktrees -require authorization. Both labels of new measurements must be v0.5.0 or newer. +Run `just performance-prepare` before measurements using reviewed local tags. +Add `--fetch-tags --allow-git-mutations` to fetch tags explicitly. Fetching and +measurement recipes' temporary worktrees require authorization. Both labels of +new measurements must be v0.5.0 or newer. Preparation also saves independent release tags and commits for a later `just performance-readme` invocation in ignored -`target/bench-reports/release-inputs.json`. For an explicit release pair, use -`just performance-prepare "$CURRENT_TAG" "$PREVIOUS_TAG"`. +`target/bench-reports/release-inputs.json`. For an explicit prepared pair, set +`mode = "explicit"` and both `current` and `baseline` in +`tooling/performance-release.toml`. The default is publication chronology. | Goal | Recipe | | --- | --- | @@ -437,19 +439,21 @@ still apply. README publication additionally requires live source freshness. ### Create The Release Performance Report -After updating the package version and fetching tags: +After updating the package version and reviewing local tags: ```bash +just performance-prepare just performance-release ``` -With no labels, the installed CLI infers the current package's release pair -using publication chronology. A prospective version compares the working tree -with the latest release; a published version compares existing tags. Explicit -labels select existing tags: +Preparation uses publication chronology. A prospective version compares the +working tree with the latest release; a published version compares existing +tags. The release command consumes that saved selection. For ad hoc measurement +of two existing tags followed by separate promotion: ```bash -just performance-release "$CURRENT_TAG" "$PREVIOUS_TAG" +just performance-measure "$CURRENT_TAG" "$PREVIOUS_TAG" +just performance-promote ``` `performance-prepare` captures both source commits before measurement and prints diff --git a/docs/RELEASING.md b/docs/RELEASING.md index c504227..f49fb6b 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -35,8 +35,9 @@ including when a rerun happens after UTC midnight. ## Reusable CI evidence The publishing process uses the installed `research-repo-tools==0.2.4` -validation helpers. `tooling/release_evidence.py` supplies the Cargo-specific -policy; it does not use the shared package's wheel/sdist release commands. +release evidence commands. `tooling/release-evidence.toml` declares the repository, +package, workflow, native jobs/steps and signer policy. The guarantee is +`signed-source`; it does not promise promotion of an identical Cargo archive. Every successful protected `main` push produces signed evidence after all three native `just ci` jobs and the default/exact Cargo packaging job succeed. The `release-evidence` job signs their exact commit, run, attempt, job IDs and @@ -50,27 +51,32 @@ nonexpired artifact retrieved by immutable ID. The shared downloader checks its authoritative ZIP size and SHA-256 digest and exact file inventory. GitHub CLI then verifies the signed manifest's workflow, source commit, `main` ref, hosted runner, push event, run and attempt. The manifest must match the live job and -package identities. Pending matching runs receive a bounded wait; missing, +package identities. Pending, failed, ambiguous or expired evidence stops publication. PR, manual dispatch, fork, local and partial-rerun results cannot replace it. -Local verification prints the validated identity as deterministic single-line -JSON. In Actions, `just release-evidence --github-output "$GITHUB_OUTPUT"` -uses the installed `research_repo_tools.ci.export_environment` helper to append -the `identity` output only after verification succeeds. The destination must +Local verification prints a success message; pass `--output ` to retain +the full approval selection. In Actions, +`just release-evidence --github-output "$GITHUB_OUTPUT"` uses the installed +transactional output writer to append `identity` and `commit` only after +verification succeeds. The destination must already exist; export errors fail the command. Existing output records survive, and normal or quiet Just execution transports the same JSON without capturing -stdout. Non-finite JSON values are rejected. The shared helper owns command-file -transport; this adapter retains the Cargo-specific evidence policy. -The Just recipe launches uv and Python directly so JSON arguments never pass -through the native Windows-to-MSYS shell boundary. Omitting `--identity` selects -fresh evidence; supplying an empty identity is an error. +stdout. The shared command owns JSON validation and command-file transport. +The Just recipe launches the exact published package with `uvx` outside the +observed checkout, using native Python argument transport on Windows. +Omitting `--expected` selects fresh evidence; a supplied empty or malformed +selection file is an error. Existing selection files can only be reused with +identical bytes. The installed downloader bounds archives at 32 MiB and the +signed manifest at 1 MiB; repository, digest and signature checks still apply. The publishing workflow reuses these completed checks and passes the run/attempt and artifact ID/digest through `steps.evidence.outputs.identity` and `needs.verify.outputs.ci-evidence` to `CI_EVIDENCE`. After deployment approval, -it requires a nonempty value and revalidates that exact identity with -`just release-evidence --identity "$CI_EVIDENCE"` before token acquisition. +it requires a nonempty value, writes it to `RUNNER_TEMP/approved-evidence.json`, +and revalidates it with `just release-evidence --expected ` before token +acquisition. A new run, attempt, artifact, manifest or policy cannot replace +the approved selection. It also retains the independent release metadata, exact-commit SARIF, durable Criterion asset, immutable GitHub release and registry-absence gates. Release benchmarks remain a separate scientific measurement workflow; CI @@ -82,7 +88,7 @@ dry-run `.crate` archive. The shared tooling's Python distribution promotion process has a different artifact contract. Use a current GitHub CLI with `gh attestation verify` and the signer/source -constraints used by the adapter. See GitHub's +constraints used by the installed verifier. See GitHub's [attestation verification options](https://cli.github.com/manual/gh_attestation_verify). ## Prepare the environment @@ -213,7 +219,7 @@ After updating the package version, fetch the tags and capture the independent source assertions before measurement: ```bash -just performance-prepare +just performance-prepare --fetch-tags --allow-git-mutations ``` Preparation selects the release pair using GitHub's publication history, @@ -224,6 +230,13 @@ This Git-ignored file carries the tags and commits forward to Review the printed tags and commits. A failed release lookup or missing tag stops preparation with a diagnostic. +The plain `performance-prepare` command uses local tags without changing refs. +The shared preparation record also binds the package version and measurement +declaration. If you have an old adapter-format `release-inputs.json`, remove +that ignored scratch file once and prepare again. It is deliberately not +converted from measurement evidence. Moved refs or changed package/declaration +inputs require a new preparation before measurement or README publication. + No shell variables or exports are needed. `performance-prepare` does not set `PREVIOUS_TAG`, `CURRENT_COMMIT`, or `PREVIOUS_COMMIT` in your shell; those variables may remain unset. Do not run the old manual `git rev-parse` @@ -235,8 +248,8 @@ Then run the long measurement command: just performance-release ``` -The no-argument recipe compares the prospective package version with the -latest stable published release. It measures and promotes the release report; +The no-argument recipe measures the independently prepared pair and promotes +the release report; README publication remains a separate, retryable step below. Review `docs/performance.md` and the complete diff --git a/docs/code_organization.md b/docs/code_organization.md index 5306b59..7b8ae55 100644 --- a/docs/code_organization.md +++ b/docs/code_organization.md @@ -171,9 +171,9 @@ and tag preparation through its CLI. Consumer identity and dependency-body polic stay in `[tool.research-repo-tools.changelog]` in `pyproject.toml`, with formatting in `changelog-rumdl.toml`; no common template is copied locally. Focused integration checks live in `tests/tooling/test_changelog_integration.py`. -`tooling/release_changelog.py` temporarily resolves omitted prospective release -dates from the prepared citation metadata before delegating generation and -preview to the installed CLI. It does not update citation metadata or use a clock. +The installed CLI resolves omitted prospective dates from `CITATION.cff`, as +declared by `changelog.date-source`. Generation and preview do not update +citation metadata or use a clock. The same dependency owns CodeRabbit review orchestration through thin Just wrappers. `tests/tooling/test_review_integration.py` owns consumer wiring checks with local stubs; the [contributor review workflow](../CONTRIBUTING.md#coderabbit-review) @@ -262,9 +262,9 @@ freshness checks and tagged evidence links. Static named configurations under linear/log choices. `tooling/plots/non-exact/` retains eight diagnostic metrics. All views require explicit input ownership. -`tooling/release_performance.py` temporarily captures independent release tags and -commits during `performance-prepare` and supplies them to the separate -`performance-readme` publication step, pending equivalent published upstream commands. +`tooling/performance-release.toml` declares independent preparation storage and +the measurement, report and README policies. Installed `performance prepare`, +`release` and `readme` commands own those separately retriable phases. The former measurement/publication implementations and root `scripts/` directory are retired. The installed CLI owns orchestration, validation and publication transactions. Working-tree measurements now include nonignored untracked regular files. @@ -291,9 +291,12 @@ outputs remain outside production scans. input validation, and inventory under the release workflow's shared setup timeout. `.github/workflows/ci.yml` owns native checks, default/exact Cargo dry-runs and -signed main-push validation evidence. `tooling/release_evidence.py` supplies the -Cargo-specific policy around shared run/job validation and artifact retrieval; +signed main-push validation evidence. `tooling/release-evidence.toml` supplies +native/Cargo and signer policy to the installed verifier; `tests/tooling/test_release_evidence.py` owns its provider-boundary regressions. +`tooling/release-assets.toml` declares the draft-target identity for the installed +`release-assets` recipe. Privileged benchmark jobs continue to use installed +asset commands from `RUNNER_TEMP`, without executing checkout code. `.github/workflows/publish-crates.yml` owns the reviewed-release event, reuse of that exact CI attempt and artifact, deployment approval and OIDC upload. The publishing declaration in `pyproject.toml` selects `la-stack`, native checks and the Criterion asset; diff --git a/docs/dev/performance-tooling-adoption.md b/docs/dev/performance-tooling-adoption.md index d254f8f..eaa03a8 100644 --- a/docs/dev/performance-tooling-adoption.md +++ b/docs/dev/performance-tooling-adoption.md @@ -57,10 +57,11 @@ or shell implementations of the shared workflow. | `performance-local` | `performance measure`, `measurements/full.toml`, current versus latest published release | | `performance-local-non-exact` | `performance measure`, `measurements/non-exact.toml`, current versus latest published release | | `performance-measure` | Explicit current/baseline labels and a named full/non-exact declaration | -| `performance-release` | `performance measure`, followed by explicit `performance promote` | +| `performance-prepare` | `performance prepare` with `performance-release.toml`; independent saved assertions | +| `performance-release` | `performance release` with `performance-release.toml`; measure the prepared pair and promote | | `performance-promote` | Explicit scratch payload and envelope, checked against `performance-report.toml` | | `performance-doc` | Offline retained history through `performance promote` and the independent full policy | -| `performance-readme` | `performance publish` with `performance-readme.toml`, independent identities and transactional formatting | +| `performance-readme` | `performance readme` with `performance-release.toml`, prepared or explicit independent identities and transactional formatting | | `plot-vs-linalg` | `performance publish --artifacts-only` with named full scratch, retained, or non-exact scratch views | | `bench-release-check` | `performance verify-saved` with saved name, Criterion directory and `criterion-saved.toml` | | `bench-release-check-suite` | `performance verify-saved --suite` with the complete independent policy and selected feature suite | @@ -68,6 +69,29 @@ or shell implementations of the shared workflow. | `bench-compare` | Existing installed saved-Criterion comparison CLI | Paths in the table are relative to `tooling/`. + +### Release adapter migration + +Published `research-repo-tools==0.2.4` now owns the former +`release_changelog.py`, `release_evidence.py` and `release_performance.py` +adapters. Citation dates are declarative in `pyproject.toml`; evidence, draft +assets and performance phases have dedicated TOML policies. The native Just +launchers observe checkout identity and forward arguments to the installed +commands. Evidence and asset verification use an isolated runtime outside the +checkout. Privileged benchmark jobs retain their existing installed asset +commands and captured target without checking out or executing repository code. + +The upstream installed-consumer suites +[`public_evidence_consumer.py`](https://github.com/acgetchell/research-repo-tools/blob/v0.2.4/tests/releases/public_evidence_consumer.py) +and +[`public_dates_consumer.py`](https://github.com/acgetchell/research-repo-tools/blob/v0.2.4/tests/changelog/public_dates_consumer.py) +own generic output rollback, immutable handoffs, malformed metadata, signature +and date parsing regressions. Local tests retain the actual native/Cargo policy, +workflow approval barriers, real Just output transport, installed runtime +identity, stale preparation rejection and scientific publication fixtures. +The shared downloader's archive bound is 32 MiB; signed manifests remain +bounded at 1 MiB. Recovery uses the shared selection schema; legacy preparation +files must be replaced through a new independent preparation. [Benchmarking](../BENCHMARKING.md) owns arguments and examples. ## Consumer contracts diff --git a/justfile b/justfile index 23b2c44..c654266 100644 --- a/justfile +++ b/justfile @@ -215,11 +215,11 @@ changelog-check: tools-check python-sync changelog-preview *args: tools-check python-sync #!/usr/bin/env bash set -euo pipefail - {{ _run }} uv run --locked --no-sync python -m tooling.release_changelog --dry-run "$@" + {{ _run }} uv run --locked --no-sync research-repo-tools changelog generate --dry-run "$@" # Generate a prospective release using the citation date unless an ISO date is supplied. changelog-release tag date="": tools-check python-sync - {{ _run }} uv run --locked --no-sync python -m tooling.release_changelog --tag {{ quote(tag) }} --date {{ quote(date) }} + {{ _run }} uv run --locked --no-sync research-repo-tools changelog generate --tag {{ quote(tag) }} {{ if date == "" { "" } else { "--date " + quote(date) } }} alias changelog-unreleased := changelog-release @@ -356,10 +356,12 @@ performance-local-non-exact: _ensure-gh python-sync performance-measure current_tag baseline_tag view="full": python-sync {{ _run }} uv run --locked research-repo-tools performance measure {{ quote("tooling/measurements/" + view + ".toml") }} {{ quote(current_tag) }} {{ quote(baseline_tag) }} --allow-git-mutations --payload {{ quote("target/bench-reports/" + view + "/run.json") }} --manifest {{ quote("target/bench-reports/" + view + "/evidence.json") }} --report {{ quote("target/bench-reports/" + view + "/report.md") }} -# Fetch tags and save independent README source assertions before measurement. -performance-prepare current_tag="" baseline_tag="": _ensure-gh python-sync - git fetch origin --tags - uv run --locked --no-sync python -m tooling.release_performance capture {{ quote(current_tag) }} {{ quote(baseline_tag) }} +# Save independent source assertions; opt in to fetching with --fetch-tags --allow-git-mutations. +[positional-arguments] +performance-prepare *args: _ensure-gh python-sync + #!/usr/bin/env bash + set -euo pipefail + uv run --locked --no-sync research-repo-tools performance prepare tooling/performance-release.toml "$@" # Promote the explicitly selected full scratch run; missing inputs never fall back. performance-promote: python-sync @@ -367,17 +369,16 @@ performance-promote: python-sync # Publish the README using prepared source assertions, or four explicit assertions. performance-readme current_tag="" current_revision="" baseline_tag="" baseline_revision="": python-sync - {{ _run }} uv run --locked --no-sync python -m tooling.release_performance publish {{ quote(current_tag) }} {{ quote(current_revision) }} {{ quote(baseline_tag) }} {{ quote(baseline_revision) }} + {{ _run }} uv run --locked --no-sync research-repo-tools performance readme tooling/performance-release.toml {{ if current_tag + current_revision + baseline_tag + baseline_revision == "" { "" } else { "--current-release " + quote(current_tag) + " --current-revision " + quote(current_revision) + " --baseline-release " + quote(baseline_tag) + " --baseline-revision " + quote(baseline_revision) } }} # Measure, retain an immutable complete run, and promote distinct-release documentation. -performance-release current_tag="" baseline_tag="": _ensure-gh python-sync +performance-release: _ensure-gh python-sync #!/usr/bin/env bash set -euo pipefail mkdir -p target/bench-reports/full started=$(date +%s) trap 'status=$?; elapsed=$(($(date +%s) - started)); printf "performance-release: %s seconds; exit %s\\n" "$elapsed" "$status" | tee target/bench-reports/full/runtime.txt' EXIT - {{ _run }} uv run --locked research-repo-tools performance measure tooling/measurements/full.toml {{ quote(current_tag) }} {{ quote(baseline_tag) }} --repository acgetchell/la-stack --allow-git-mutations --payload target/bench-reports/full/run.json --manifest target/bench-reports/full/evidence.json --report target/bench-reports/full/report.md - {{ _run }} uv run --locked research-repo-tools performance promote tooling/performance-report.toml --payload target/bench-reports/full/run.json --manifest target/bench-reports/full/evidence.json + {{ _run }} uv run --locked research-repo-tools performance release tooling/performance-release.toml --allow-git-mutations # Render a named metric/statistic/axis view from explicitly retained or scratch inputs. plot-vs-linalg current_tag current_revision baseline_tag baseline_revision view="lu_solve-median-linear" input="retained": python-sync @@ -415,14 +416,32 @@ python-sync: _ensure-uv python-typecheck: python-sync uv run --locked research-repo-tools python typecheck -# Verify signed Cargo/native CI evidence; accept --identity JSON and --github-output PATH. +# Preflight, upload or verify a captured draft target with an isolated trusted runtime. +[positional-arguments] +[script("uvx", "--python", "3.15", "--from", "research-repo-tools==0.2.4", "python", "-I")] +release-assets operation tag *args: + import sys + from pathlib import Path + from research_repo_tools.cli import main + from research_repo_tools.process import run_git_command + root = Path.cwd() + commit = run_git_command(["--no-pager", "rev-parse", "HEAD"], cwd=root, timeout=30).stdout.strip() + sys.exit(main(["release", "assets", sys.argv[1], "tooling/release-assets.toml", "--checkout", str(root), "--commit", commit, "--tag", sys.argv[2], *sys.argv[3:]])) + +# Verify signed native/Cargo evidence; accept --expected PATH and --github-output PATH. [positional-arguments] -[script("uv", "run", "--locked", "--no-sync", "python")] +[script("uvx", "--python", "3.15", "--from", "research-repo-tools==0.2.4", "python", "-I")] release-evidence *args: - import runpy import sys - sys.argv[1:1] = ["verify"] - runpy.run_path("tooling/release_evidence.py", run_name="__main__") + import tempfile + from pathlib import Path + from research_repo_tools.cli import main + from research_repo_tools.process import run_git_command + from research_repo_tools.release_metadata import read_package_info + root = Path.cwd() + commit = run_git_command(["--no-pager", "rev-parse", "HEAD"], cwd=root, timeout=30).stdout.strip() + with tempfile.TemporaryDirectory(prefix="la-stack-release-evidence-") as temporary: + sys.exit(main(["release", "evidence", "verify", "tooling/release-evidence.toml", "--checkout", str(root), "--commit", commit, "--version", read_package_info(root).version, "--output", str(Path(temporary).resolve() / "selection.json"), *sys.argv[1:]])) # Require the release event, reviewed commit, native checks, assets, and notes. release-gate tag: diff --git a/pyproject.toml b/pyproject.toml index 3da84b7..4943a7b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -141,6 +141,8 @@ owner = "acgetchell" repository = "la-stack" formatter = "changelog-rumdl.toml" dependency-bodies = "preserve" +date-source = "citation" +citation = "CITATION.cff" [tool.research-repo-tools.zizmor] persona = "regular" diff --git a/tests/tooling/test_changelog_integration.py b/tests/tooling/test_changelog_integration.py index 9e2c6b8..ac1e9d9 100644 --- a/tests/tooling/test_changelog_integration.py +++ b/tests/tooling/test_changelog_integration.py @@ -9,8 +9,6 @@ from research_repo_tools.just_inspect import dry_run, inspect_justfile from research_repo_tools.process import run_command -from tooling.release_changelog import main as release_changelog - if TYPE_CHECKING: import pytest @@ -25,6 +23,8 @@ def test_consumer_changelog_policy_uses_the_shared_formatter() -> None: "repository": "la-stack", "formatter": "changelog-rumdl.toml", "dependency-bodies": "preserve", + "date-source": "citation", + "citation": "CITATION.cff", } assert not (ROOT / "cliff.toml").exists() @@ -45,16 +45,16 @@ def test_consumer_generation_preserves_an_authored_dependency_note(tmp_path: Pat citation.write_text('version: "0.4.6"\ndate-released: 2000-02-29\n', encoding="utf-8") original_citation = citation.read_bytes() monkeypatch.chdir(tmp_path) - assert release_changelog(["--tag", "v0.4.6"]) == 0 + assert main(["changelog", "generate", "--tag", "v0.4.6"]) == 0 generated = (tmp_path / "CHANGELOG.md").read_text(encoding="utf-8") assert "### Dependencies" in generated assert note in generated assert "## [0.4.6] - 2000-02-29" in generated assert "## [0.4.5] - 2000-01-02" in generated - assert release_changelog(["--tag", "v0.4.6"]) == 0 + assert main(["changelog", "generate", "--tag", "v0.4.6"]) == 0 assert (tmp_path / "CHANGELOG.md").read_text(encoding="utf-8") == generated assert citation.read_bytes() == original_citation - assert release_changelog(["--tag", "v0.4.6", "--dry-run"]) == 0 + assert main(["changelog", "generate", "--tag", "v0.4.6", "--dry-run"]) == 0 assert (tmp_path / "CHANGELOG.md").read_text(encoding="utf-8") == generated @@ -63,7 +63,7 @@ def test_consumer_recipes_and_retained_history(capsys: pytest.CaptureFixture[str recipes = inspect_justfile(ROOT) assert recipes.aliases["changelog-unreleased"] == "changelog-release" result = dry_run(ROOT, "changelog-preview", ["--tag", "v0.4.7", "--date", "2026-10-09"]) - assert 'python -m tooling.release_changelog --dry-run "$@"' in result.stderr + assert 'research-repo-tools changelog generate --dry-run "$@"' in result.stderr assert main(["--root", str(ROOT), "changelog", "check"]) == 0 for tag in ("v0.1.0", "v0.2.0", "v0.3.0", "v0.4.6"): assert main(["--root", str(ROOT), "changelog", "notes", tag]) == 0 diff --git a/tests/tooling/test_justfile_discoverability.py b/tests/tooling/test_justfile_discoverability.py index d05c841..cb8afd6 100644 --- a/tests/tooling/test_justfile_discoverability.py +++ b/tests/tooling/test_justfile_discoverability.py @@ -84,7 +84,7 @@ def test_ci_enforces_full_python_fixture_lint_policy() -> None: assert "research-repo-tools python check" in python_body assert "--select" not in python_body inventory = set(select_files(REPO_ROOT, include=("*.py", "*.pyi"))) - assert {"tooling/release_evidence.py", "tests/tooling/test_release_evidence.py", "tests/semgrep/scripts/python_portability.py"} <= inventory + assert {"tooling/shared_tooling.py", "tests/tooling/test_release_evidence.py", "tests/semgrep/scripts/python_portability.py"} <= inventory assert not any(".venv" in Path(name).parts for name in inventory) diff --git a/tests/tooling/test_performance_workflow.py b/tests/tooling/test_performance_workflow.py index 82f70dd..bbd0202 100644 --- a/tests/tooling/test_performance_workflow.py +++ b/tests/tooling/test_performance_workflow.py @@ -16,6 +16,7 @@ from research_repo_tools.evidence import load_evidence, publish_evidence from research_repo_tools.just_inspect import dry_run from research_repo_tools.measurement import load_measurement +from research_repo_tools.performance_release import Preparation, serialize_preparation from research_repo_tools.process import run_command from research_repo_tools.release_pairs import ReleasePair from research_repo_tools.worktrees import capture_snapshot @@ -239,9 +240,10 @@ def test_native_recipes_preserve_scientific_gates_and_peer_reuse() -> None: assert "research-repo-tools performance measure" in command assert "--mode current-vs-latest" in command assert "--allow-git-mutations" in command - command = dry_run(ROOT, "performance-release", [PAIR.current, PAIR.baseline]).stderr - assert command.count("performance measure") == command.count("performance promote") == 1 - assert "--payload target/bench-reports/full/run.json" in command + command = dry_run(ROOT, "performance-release").stderr + assert command.count("performance release tooling/performance-release.toml") == 1 + plan = tomllib.loads((ROOT / "tooling/performance-release.toml").read_text(encoding="utf-8")) + assert plan["payload"] == "target/bench-reports/full/run.json" assert "scripts/" not in command @@ -269,8 +271,6 @@ def prepared(tmp_path: Path) -> tuple[Path, Path]: destination = root / source.relative_to(ROOT) destination.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(source, destination) - for name in ("__init__.py", "release_performance.py"): - shutil.copyfile(ROOT / "tooling" / name, root / "tooling" / name) (root / "src").mkdir() (root / "src/lib.rs").write_text(f"// {phase}\n", encoding="utf-8", newline="\n") (root / ".gitignore").write_text("/target/\n", encoding="utf-8", newline="\n") @@ -322,6 +322,9 @@ def prepared(tmp_path: Path) -> tuple[Path, Path]: @pytest.fixture(scope="module") def measured_run(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Evidence]: baseline, current = prepared(tmp_path_factory.mktemp("performance")) + # Retain the independently captured baseline commit for preparation freshness. + run_command("git", ["fetch", str(baseline), "HEAD"], cwd=current) + run_command("git", ["-c", "tag.gpgsign=false", "tag", PAIR.baseline, "FETCH_HEAD"], cwd=current) asserted = [ PAIR.current, run_command("git", ["--no-pager", "rev-parse", "HEAD"], cwd=current).stdout.strip(), @@ -419,7 +422,21 @@ def test_actual_retention_offline_plot_and_readme_recipes(complete_run: tuple[Pa if source == "retained": prepared_inputs = root / "target/bench-reports/release-inputs.json" prepared_inputs.parent.mkdir(parents=True) - prepared_inputs.write_text(json.dumps({"schema": 1, "assertions": identities(root)}), encoding="utf-8", newline="\n") + current_tag, current_revision, baseline_tag, baseline_revision = identities(root) + prepared_inputs.write_bytes( + serialize_preparation( + Preparation( + "acgetchell/la-stack", + "tooling/measurements/full.toml", + hashlib.sha256((root / "tooling/measurements/full.toml").read_bytes()).hexdigest(), + current_tag.removeprefix("v"), + ReleasePair(current_tag, baseline_tag), + current_revision, + baseline_revision, + working_tree=True, + ) + ) + ) result = recipe(root, "performance-readme", *([] if source == "retained" else identities(root))) assert result.returncode == 0, result.stdout + result.stderr document = (root / "README.md").read_text(encoding="utf-8") @@ -483,11 +500,19 @@ def test_publication_boundaries_leave_all_outputs_unchanged(complete_run: tuple[ assert result.returncode != 0 assert {path: path.read_bytes() for path in before} == before assert not (root / "docs/assets").exists() + if failure == "formatter": + # Retry publication after repairing its formatter. Archived measurements + # remain byte-identical; no timing or discovery command is involved. + retained = {path: path.read_bytes() for path in (root / "docs/performance-runs").rglob("*") if path.is_file()} + config = root / "tooling/performance-readme.toml" + config.write_bytes((ROOT / "tooling/performance-readme.toml").read_bytes()) + result = recipe(root, "performance-readme", *identities(root)) + assert result.returncode == 0, result.stdout + result.stderr + assert {path: path.read_bytes() for path in retained} == retained @pytest.mark.parametrize("phase", ["baseline", "current"]) -@pytest.mark.parametrize("command", ["performance-measure", "performance-release"]) -def test_actual_measurement_recipe_rejects_old_releases_before_git(phase: str, command: str, tmp_path: Path) -> None: +def test_actual_measurement_recipe_rejects_old_releases_before_git(phase: str, tmp_path: Path) -> None: root = tmp_path / "consumer" root.mkdir() for name in ("pyproject.toml", "uv.lock", ".python-version", "Cargo.toml", "rust-toolchain.toml"): @@ -499,18 +524,11 @@ def test_actual_measurement_recipe_rejects_old_releases_before_git(phase: str, c shutil.copyfile(source, destination) (root / "src").mkdir() (root / "src/lib.rs").write_text("// fixture\n", encoding="utf-8", newline="\n") - result = recipe(root, command, "v0.4.6" if phase == "current" else PAIR.current, "v0.4.6" if phase == "baseline" else PAIR.baseline) + result = recipe(root, "performance-measure", "v0.4.6" if phase == "current" else PAIR.current, "v0.4.6" if phase == "baseline" else PAIR.baseline) assert result.returncode != 0 assert "requires v0.5.0 or newer" in result.stderr assert not (root / ".git").exists() - if command == "performance-release": - runtime = (root / "target/bench-reports/full/runtime.txt").read_text(encoding="utf-8") - assert runtime.startswith("performance-release: ") - assert runtime.endswith(f" seconds; exit {result.returncode}\n") - assert not (root / "target/bench-reports/full/run.json").exists() - assert not (root / "docs/performance-runs").exists() - else: - assert not (root / "target").exists() + assert not (root / "target").exists() def test_legacy_bytes_hashes_are_unchanged() -> None: diff --git a/tests/tooling/test_release_changelog.py b/tests/tooling/test_release_changelog.py index 52f057a..cedff2f 100644 --- a/tests/tooling/test_release_changelog.py +++ b/tests/tooling/test_release_changelog.py @@ -1,106 +1,71 @@ -"""Recorded citation dates make prospective changelog generation repeatable.""" +"""Consumer citation policy uses the installed deterministic date resolver.""" from pathlib import Path import pytest +from research_repo_tools.changelog import resolve_date +from research_repo_tools.config import load from research_repo_tools.just_inspect import dry_run -from tooling import release_changelog as adapter - ROOT = Path(__file__).resolve().parents[2] +@pytest.fixture +def consumer(tmp_path: Path) -> Path: + (tmp_path / "pyproject.toml").write_bytes((ROOT / "pyproject.toml").read_bytes()) + return tmp_path + + @pytest.mark.parametrize("quotation", ["", "'", '"']) -def test_omitted_date_uses_recorded_citation_date_for_generation_and_preview( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - quotation: str, -) -> None: - citation = tmp_path / "CITATION.cff" +def test_omitted_date_uses_recorded_citation_date(consumer: Path, quotation: str) -> None: + citation = consumer / "CITATION.cff" contents = f"version: 0.5.1\ndate-released: {quotation}2000-02-29{quotation}\n" citation.write_text(contents, encoding="utf-8") - monkeypatch.chdir(tmp_path) - calls: list[list[str]] = [] - - def shared_cli(argv: list[str]) -> int: - calls.append(argv) - return 0 - - monkeypatch.setattr(adapter, "tooling_main", shared_cli) - for extra in ([], ["--date", ""], ["--dry-run"]): - assert adapter.main(["--tag", "v0.5.1", *extra]) == 0 - expected = ["changelog", "generate", "--tag", "v0.5.1", "--date", "2000-02-29"] - if extra == ["--dry-run"]: - expected.append("--dry-run") - assert calls[-1] == expected + assert resolve_date(load(root=consumer), tag="v0.5.1") == "2000-02-29" assert citation.read_text(encoding="utf-8") == contents -@pytest.mark.parametrize("arguments", [["--tag", "v0.5.1", "--date", "2026-10-09"], []]) -def test_explicit_date_and_untagged_preview_do_not_require_citation( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - arguments: list[str], -) -> None: - monkeypatch.chdir(tmp_path) - calls: list[list[str]] = [] - - def shared_cli(argv: list[str]) -> int: - calls.append(argv) - return 7 - - monkeypatch.setattr(adapter, "tooling_main", shared_cli) - assert adapter.main([*arguments, "--dry-run"]) == 7 - assert calls == [["changelog", "generate", *arguments, "--dry-run"]] - assert not (tmp_path / "CITATION.cff").exists() +def test_explicit_date_and_untagged_preview_need_no_citation(consumer: Path) -> None: + settings = load(root=consumer) + assert resolve_date(settings, tag="v0.5.1", released="2026-10-09") == "2026-10-09" + assert resolve_date(settings) is None + assert not (consumer / "CITATION.cff").exists() @pytest.mark.parametrize( - ("contents", "diagnostic"), + "contents", [ - (None, "CITATION.cff"), - ("[", "while parsing"), - ("- item\n", "top-level mapping"), - ("version: 0.5.1\n", "top-level date-released"), - ("date-released: 2000-02-29\n", "top-level version"), - ("version: 0.5.0\ndate-released: 2000-02-29\n", "does not match v0.5.1"), - ("version: 0.5.1\ndate-released: 2001-02-29\n", "day"), - ("version: 0.5.1\ndate-released: 20000229\n", "YYYY-MM-DD"), - ("version: 0.5.1\ndate-released: \n", "nonempty top-level date-released"), - ("version: 0.5.1\ndate-released: [2000-02-29]\n", "top-level date-released scalar"), - ("version: 0.5.1\ndate-released: 2000-02-29\ndate-released: 2000-03-01\n", "exactly one"), - ("version: 0.5.1\nversion: 0.5.2\ndate-released: 2000-02-29\n", "exactly one"), + None, + "[", + "- item\n", + "version: 0.5.1\n", + "date-released: 2000-02-29\n", + "version: 0.5.0\ndate-released: 2000-02-29\n", + "version: 0.5.1\ndate-released: 2001-02-29\n", + "version: 0.5.1\ndate-released: 20000229\n", + "version: 0.5.1\ndate-released: \n", + "version: 0.5.1\ndate-released: [2000-02-29]\n", + "version: 0.5.1\ndate-released: 2000-02-29\ndate-released: 2000-03-01\n", + "version: 0.5.1\nversion: 0.5.2\ndate-released: 2000-02-29\n", ], ) -def test_invalid_default_metadata_stops_before_generation( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - contents: str | None, - diagnostic: str, -) -> None: - citation = tmp_path / "CITATION.cff" +def test_invalid_default_metadata_never_changes_release_files(consumer: Path, contents: str | None) -> None: + citation = consumer / "CITATION.cff" if contents is not None: citation.write_text(contents, encoding="utf-8") - changelog = tmp_path / "CHANGELOG.md" + changelog = consumer / "CHANGELOG.md" changelog.write_bytes(b"retained changelog\n") - monkeypatch.chdir(tmp_path) - monkeypatch.setattr(adapter, "tooling_main", lambda _argv: pytest.fail("invalid metadata must not generate anything")) - assert adapter.main(["--tag", "v0.5.1"]) == 1 - output = capsys.readouterr() - assert output.out == "" - assert "Release changelog rejected:" in output.err - assert diagnostic in output.err - assert "just update-version" in output.err + with pytest.raises(ValueError, match=r"citation|YYYY-MM-DD"): + resolve_date(load(root=consumer), tag="v0.5.1") assert changelog.read_bytes() == b"retained changelog\n" - assert not (tmp_path / "docs").exists() if contents is not None: assert citation.read_text(encoding="utf-8") == contents -def test_optional_recipe_argument_and_alias_delegate_to_the_date_adapter() -> None: +def test_recipe_omits_absent_date_and_delegates_to_installed_cli() -> None: implicit = dry_run(ROOT, "changelog-unreleased", ["v0.5.1"]).stderr explicit = dry_run(ROOT, "changelog-release", ["v0.5.1", "2000-02-29"]).stderr - assert "tooling.release_changelog --tag 'v0.5.1' --date ''" in implicit - assert "tooling.release_changelog --tag 'v0.5.1' --date '2000-02-29'" in explicit - assert 'tooling.release_changelog --dry-run "$@"' in dry_run(ROOT, "changelog-preview", ["--tag", "v0.5.1"]).stderr + assert "changelog generate --tag 'v0.5.1'" in implicit + assert "--date" not in implicit + assert "changelog generate --tag 'v0.5.1' --date '2000-02-29'" in explicit + assert 'changelog generate --dry-run "$@"' in dry_run(ROOT, "changelog-preview", ["--tag", "v0.5.1"]).stderr diff --git a/tests/tooling/test_release_evidence.py b/tests/tooling/test_release_evidence.py index 2b2f34b..b1db199 100644 --- a/tests/tooling/test_release_evidence.py +++ b/tests/tooling/test_release_evidence.py @@ -1,17 +1,18 @@ -"""Exercise the Cargo evidence adapter against authenticated provider boundaries.""" +"""Verify consumer evidence policy through the installed release interfaces.""" -import importlib.util +import hashlib import json import subprocess -import sys +from argparse import Namespace from copy import deepcopy from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any import pytest - -if TYPE_CHECKING: - from types import ModuleType +from research_repo_tools import release_assets, release_evidence, release_interfaces, release_validation +from research_repo_tools.cli import main +from research_repo_tools.config import load +from research_repo_tools.release_interfaces import load_evidence_policy REPO_ROOT = Path(__file__).resolve().parents[2] REPOSITORY = "acgetchell/la-stack" @@ -23,8 +24,7 @@ ARTIFACT_ENDPOINT = "actions/artifacts/301" RUNS_ENDPOINT = f"actions/workflows/9/runs?event=push&branch=main&head_sha={COMMIT}&per_page=100&page=1" INVOCATION = f"https://github.com/{REPOSITORY}/actions/runs/101/attempts/2" -PIN = {"run_id": 101, "run_attempt": 2, "artifact_id": 301, "artifact_digest": DIGEST} -PIN_JSON = '{"artifact_digest":"sha256:' + "b" * 64 + '","artifact_id":301,"run_attempt":2,"run_id":101}' +POLICY = load_evidence_policy(REPO_ROOT / "tooling/release-evidence.toml") NATIVE_GATES = ( ("build (ubuntu-latest)", "Run CI checks", 201), ("build (macos-latest)", "Run CI checks", 202), @@ -42,9 +42,8 @@ class EvidenceProvider: """Keep signed inputs independent from live provider responses.""" - def __init__(self, adapter: ModuleType, root: Path) -> None: + def __init__(self, root: Path) -> None: """Create independent signed and provider identities for one CI attempt.""" - self.adapter = adapter self.root = root self.run: dict[str, Any] = { "id": 101, @@ -72,6 +71,7 @@ def __init__(self, adapter: ModuleType, root: Path) -> None: "head_sha": COMMIT, "status": "completed", "conclusion": "success", + "labels": [name.removeprefix("build (").removesuffix(")")] if name.startswith("build (") else ["ubuntu-latest"], "steps": [{"number": number, "name": step, "status": "completed", "conclusion": "success"} for number, step in enumerate(steps, 1)], } for name, steps, identifier in gates @@ -85,12 +85,32 @@ def __init__(self, adapter: ModuleType, root: Path) -> None: "workflow_run": {"id": 101, "head_sha": COMMIT, "head_branch": "main", "repository_id": 11, "head_repository_id": 11}, } self.document: dict[str, Any] = { - "schema": 1, - "repository": REPOSITORY, - "repository_id": 11, - "commit": COMMIT, - "run_id": 101, - "run_attempt": 2, + "schema": "research-repo-tools/release-evidence/v1", + "policy": { + "repository": REPOSITORY, + "workflow": ".github/workflows/ci.yml", + "workflow_name": "CI", + "ref": "refs/heads/main", + "event": "push", + "required_steps": {name: [step] for name, step, _ in NATIVE_GATES}, + "runner_labels": {name: [name.removeprefix("build (").removesuffix(")")] for name, _, _ in NATIVE_GATES[:3]}, + "artifact_prefix": "release-evidence", + "guarantee": "signed-source", + "signing_job": {"release-evidence": list(SIGNING_STEPS)}, + "path_in_vcs": "", + }, + "run": { + "id": 101, + "run_attempt": 2, + "head_sha": COMMIT, + "head_branch": "main", + "workflow_id": 9, + "path": ".github/workflows/ci.yml", + "name": "CI", + "event": "push", + "repository_id": 11, + "head_repository_id": 11, + }, "jobs": {name: identifier for name, _, identifier in NATIVE_GATES}, "package": {"name": "la-stack", "version": "0.5.42"}, } @@ -119,10 +139,10 @@ def api(self, endpoint: str, *, repository: str) -> Any: return deepcopy(self.responses[endpoint]) def git(self, args: list[str], **kwargs: Any) -> subprocess.CompletedProcess[str]: - assert args == ["--no-pager", "rev-parse", "HEAD"] + assert "rev-parse" in args or "status" in args assert kwargs["cwd"] == self.root assert kwargs["timeout"] == 30 - return subprocess.CompletedProcess(args, 0, stdout=COMMIT + "\n", stderr="") + return subprocess.CompletedProcess(args, 0, stdout=COMMIT + "\n" if "rev-parse" in args else "", stderr="") def gh(self, *args: str) -> str: assert args[:2] == ("attestation", "verify") @@ -139,33 +159,108 @@ def download(self, repository: str, artifact: dict[str, Any], directory: Path, n (directory / "validation.json").write_text(payload, encoding="utf-8") (directory / "validation.sigstore.json").write_text('{"fixture":"signed bundle"}', encoding="utf-8") + def verify(self, identity: str | None = None) -> dict[str, Any]: + """Exercise the public verifier with independently authored signed data.""" + return json.loads( + release_evidence.verify_release_evidence( + POLICY, + "0.5.42", + COMMIT, + lambda endpoint: self.api(endpoint, repository=REPOSITORY), + expected=identity.encode() if identity is not None else None, + ) + ) + + def seal(self, output: Path) -> None: + """Exercise the installed producer including its environment/output guards.""" + args = Namespace( + action="evidence", + operation="create", + configuration="tooling/release-evidence.toml", + checkout=self.root, + commit=COMMIT, + version="0.5.42", + run_id=101, + attempt=2, + output=output, + github_output=None, + tools=None, + archive=None, + ) + release_interfaces.run_evidence(args, load(root=self.root)) + + def install(self, patch: pytest.MonkeyPatch) -> None: + """Fake authenticated services and Git observations, leaving policy/CLI real.""" + patch.setattr(release_interfaces, "api", self.api) + patch.setattr(release_validation, "gh", self.gh) + patch.setattr(release_evidence, "download_artifact", self.download) + patch.setattr(release_assets, "run_git_command", self.git) + patch.setattr(release_assets, "run_git_bytes", lambda *args, **_kwargs: subprocess.CompletedProcess(args, 0, stdout=b"", stderr=b"")) + + def cli(self, *args: str) -> int: + return main( + [ + "--root", + str(self.root), + "release", + "evidence", + "verify", + "tooling/release-evidence.toml", + "--checkout", + str(self.root), + "--commit", + COMMIT, + "--version", + "0.5.42", + "--output", + str(self.root / "selection.json"), + *args, + ] + ) + + +def expected_identity() -> str: + """Independent approval record for the fixture's literal signed bytes.""" + provider = EvidenceProvider(Path()) + selected = { + **provider.document, + "schema": "research-repo-tools/release-selection/v1", + "jobs": {**provider.document["jobs"], "release-evidence": 205}, + "artifact": {key: provider.artifact[key] for key in ("id", "name", "digest", "size_in_bytes")}, + "manifest_sha256": hashlib.sha256(json.dumps(provider.document).encode()).hexdigest(), + } + return json.dumps(selected, sort_keys=True, separators=(",", ":")) + + +PIN_JSON = expected_identity() + @pytest.fixture def provider(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> EvidenceProvider: - spec = importlib.util.spec_from_file_location("la_stack_release_evidence", REPO_ROOT / "tooling/release_evidence.py") - assert spec is not None - assert spec.loader is not None - adapter = importlib.util.module_from_spec(spec) - spec.loader.exec_module(adapter) root = tmp_path / "checkout with spaces" root.mkdir() (root / "Cargo.toml").write_text('[package]\nname = "la-stack"\nversion = "0.5.42"\n', encoding="utf-8") - result = EvidenceProvider(adapter, root) - monkeypatch.setattr(adapter, "ROOT", root) - monkeypatch.setattr(adapter, "api", result.api) - monkeypatch.setattr(adapter, "gh", result.gh) - monkeypatch.setattr(adapter, "download_artifact", result.download) - monkeypatch.setattr(adapter, "run_git_command", result.git) - monkeypatch.setenv("GITHUB_SHA", COMMIT) - monkeypatch.setenv("GITHUB_RUN_ID", "101") - monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "2") + (root / "tooling").mkdir() + (root / "tooling/release-evidence.toml").write_bytes((REPO_ROOT / "tooling/release-evidence.toml").read_bytes()) + result = EvidenceProvider(root) + result.install(monkeypatch) + for name, value in { + "GITHUB_SHA": COMMIT, + "GITHUB_RUN_ID": "101", + "GITHUB_RUN_ATTEMPT": "2", + "GITHUB_REPOSITORY": REPOSITORY, + "GITHUB_EVENT_NAME": "push", + "GITHUB_REF": "refs/heads/main", + }.items(): + monkeypatch.setenv(name, value) return result def test_verify_consumes_exact_main_attempt_with_bound_attestation_flags(provider: EvidenceProvider) -> None: - assert provider.adapter.verify() == PIN + selection = provider.verify() + assert selection["run"]["id"] == 101 + assert selection["artifact"]["digest"] == DIGEST assert provider.api_calls.count(JOBS_ENDPOINT) == 2 - assert provider.api_calls[-3:] == [RUN_ENDPOINT, "actions/workflows/ci.yml", ARTIFACT_ENDPOINT] assert len(provider.downloads) == 1 repository, artifact, directory, names = provider.downloads[0] assert repository == REPOSITORY @@ -202,22 +297,21 @@ def test_seal_captures_successful_native_jobs_while_its_signing_job_is_pending(p provider.run.update(status="in_progress", conclusion=None) provider.jobs.pop() # The sealing job cannot have completed its own upload. output = provider.root / "evidence" / "validation.json" - provider.adapter.seal(output) + provider.seal(output) assert json.loads(output.read_text(encoding="utf-8")) == provider.document assert provider.api_calls == [RUN_ENDPOINT, "actions/workflows/ci.yml", JOBS_ENDPOINT, RUN_ENDPOINT, "actions/workflows/ci.yml"] assert provider.downloads == [] assert provider.gh_calls == [] original = output.read_bytes() - with pytest.raises(FileExistsError): - provider.adapter.seal(output) + provider.seal(output) # Identical immutable writes are safe retries. assert output.read_bytes() == original def test_seal_rejects_environment_checkout_mismatch_before_creating_output(provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("GITHUB_SHA", "c" * 40) output = provider.root / "evidence" / "validation.json" - with pytest.raises(ValueError, match="executing CI source differs from checkout"): - provider.adapter.seal(output) + with pytest.raises(ValueError, match="exact executing trusted push/run/attempt"): + provider.seal(output) assert not output.parent.exists() assert provider.api_calls == [] @@ -225,8 +319,8 @@ def test_seal_rejects_environment_checkout_mismatch_before_creating_output(provi def test_seal_rechecks_the_attempt_before_publishing_the_document(provider: EvidenceProvider) -> None: provider.scripted[RUN_ENDPOINT] = [deepcopy(provider.run), {**provider.run, "run_attempt": 3}] output = provider.root / "evidence" / "validation.json" - with pytest.raises(ValueError, match="selected CI attempt is stale"): - provider.adapter.seal(output) + with pytest.raises(ValueError, match="selected run or attempt changed"): + provider.seal(output) assert not output.parent.exists() assert JOBS_ENDPOINT in provider.api_calls @@ -236,25 +330,27 @@ def test_seal_never_writes_a_manifest_for_failed_required_validation(provider: E provider.jobs[job_index]["steps"][0]["conclusion"] = "failure" output = provider.root / "evidence" / "validation.json" with pytest.raises(ValueError, match="did not validate required step"): - provider.adapter.seal(output) + provider.seal(output) assert not output.parent.exists() def test_pinned_recheck_uses_only_the_approved_run_attempt_and_artifact(provider: EvidenceProvider) -> None: - identity = json.dumps(provider.adapter.verify()) + identity = json.dumps(provider.verify()) provider.api_calls.clear() del provider.responses[RUNS_ENDPOINT] - del provider.responses[ARTIFACTS_ENDPOINT] - assert provider.adapter.verify(identity) == PIN + assert provider.verify(identity) == json.loads(identity) assert provider.api_calls.count(RUN_ENDPOINT) == 2 assert provider.api_calls.count(ARTIFACT_ENDPOINT) == 2 assert len(provider.gh_calls) == 2 def test_pinned_recheck_rejects_an_attempt_replaced_by_a_rerun(provider: EvidenceProvider) -> None: - identity = json.dumps({**PIN, "run_attempt": 1}) - with pytest.raises(ValueError, match="selected CI attempt is stale"): - provider.adapter.verify(identity) + selection = provider.verify() + selection["run"]["run_attempt"] = 1 + provider.downloads.clear() + identity = json.dumps(selection) + with pytest.raises(ValueError, match="selected run or attempt changed"): + provider.verify(identity) assert provider.downloads == [] @@ -272,21 +368,21 @@ def test_pinned_recheck_rejects_an_attempt_replaced_by_a_rerun(provider: Evidenc def test_verify_rejects_another_source_event_ref_commit_or_workflow(provider: EvidenceProvider, field: str, value: Any) -> None: provider.run[field] = value with pytest.raises(ValueError, match="exact commit's trusted main-push"): - provider.adapter.verify() + provider.verify() assert provider.downloads == [] def test_verify_rejects_a_fork_run_with_matching_commit(provider: EvidenceProvider) -> None: provider.run["head_repository"] = {"id": 12, "full_name": "fork/la-stack"} with pytest.raises(ValueError, match="repository or head repository differs"): - provider.adapter.verify() + provider.verify() assert provider.downloads == [] def test_verify_rejects_a_failed_run_without_using_successful_job_fragments(provider: EvidenceProvider) -> None: provider.run["conclusion"] = "failure" with pytest.raises(ValueError, match="missing or failed main-push validation"): - provider.adapter.verify() + provider.verify() assert JOBS_ENDPOINT not in provider.api_calls assert provider.downloads == [] @@ -296,7 +392,7 @@ def test_verify_rejects_a_failed_run_without_using_successful_job_fragments(prov def test_job_identity_must_belong_to_the_exact_run_and_attempt(provider: EvidenceProvider, job_index: int, field: str, value: Any) -> None: provider.jobs[job_index][field] = value with pytest.raises(ValueError, match="belongs to another run, attempt, ref or commit"): - provider.adapter.verify() + provider.verify() assert provider.downloads == [] @@ -304,14 +400,14 @@ def test_job_identity_must_belong_to_the_exact_run_and_attempt(provider: Evidenc def test_failed_jobs_cannot_be_replaced_by_successful_step_fragments(provider: EvidenceProvider, job_index: int) -> None: provider.jobs[job_index]["conclusion"] = "failure" with pytest.raises(ValueError, match="did not succeed"): - provider.adapter.verify() + provider.verify() assert provider.downloads == [] def test_native_job_identities_cannot_alias_one_another(provider: EvidenceProvider) -> None: provider.jobs[1]["id"] = provider.jobs[0]["id"] with pytest.raises(ValueError, match="validation job IDs are not unique"): - provider.adapter.verify() + provider.verify() assert provider.downloads == [] @@ -324,7 +420,7 @@ def test_every_native_package_and_signing_job_is_unambiguous(provider: EvidenceP else: provider.jobs.pop(job_index) with pytest.raises(ValueError, match="missing or ambiguous") as failure: - provider.adapter.verify() + provider.verify() assert name in str(failure.value) assert provider.downloads == [] @@ -335,7 +431,7 @@ def test_required_native_package_and_signing_steps_must_succeed(provider: Eviden step = provider.jobs[job_index]["steps"][step_index] step["conclusion"] = conclusion with pytest.raises(ValueError, match="did not validate required step") as failure: - provider.adapter.verify() + provider.verify() assert step["name"] in str(failure.value) assert provider.downloads == [] @@ -348,7 +444,7 @@ def test_required_step_names_cannot_be_substituted_or_duplicated(provider: Evide else: steps[0]["name"] = "Package check skipped" with pytest.raises(ValueError, match="did not validate required step Verify default and exact Cargo packages"): - provider.adapter.verify() + provider.verify() @pytest.mark.parametrize( @@ -357,26 +453,26 @@ def test_required_step_names_cannot_be_substituted_or_duplicated(provider: Evide ) def test_artifact_association_must_match_the_validated_run(provider: EvidenceProvider, field: str, value: Any) -> None: provider.artifact["workflow_run"][field] = value - with pytest.raises(ValueError, match="artifact is expired or belongs to another"): - provider.adapter.verify() + with pytest.raises(ValueError, match="artifact is associated with another repository, ref, run or commit"): + provider.verify() assert provider.downloads == [] @pytest.mark.parametrize( ("field", "value", "message"), [ - ("expired", True, "artifact is expired"), + ("expired", True, "missing, expired or mismatched"), ("digest", None, "authoritative SHA-256 digest"), ("digest", "sha256:not-a-digest", "authoritative SHA-256 digest"), - ("size_in_bytes", 0, "positive integer identity"), - ("size_in_bytes", 1024 * 1024 + 1, "one-MiB limit"), - ("id", True, "positive integer identity"), + ("size_in_bytes", 0, "artifact size exceeds the release bound"), + ("size_in_bytes", 32 * 1024 * 1024 + 1, "artifact size exceeds the release bound"), + ("id", True, "artifact ID must be a positive integer"), ], ) def test_artifact_requires_unexpired_bounded_authoritative_identity(provider: EvidenceProvider, field: str, value: Any, message: str) -> None: provider.artifact[field] = value with pytest.raises(ValueError, match=message): - provider.adapter.verify() + provider.verify() assert provider.downloads == [] @@ -388,15 +484,17 @@ def test_artifact_selection_does_not_fall_back_or_choose_ambiguously(provider: E "other-attempt": [{**provider.artifact, "name": "release-evidence-101-1"}], } provider.responses[ARTIFACTS_ENDPOINT] = {"artifacts": variants[inventory]} - with pytest.raises(ValueError, match="missing or ambiguous signed CI evidence"): - provider.adapter.verify() + with pytest.raises(ValueError, match="missing or ambiguous evidence artifact"): + provider.verify() assert provider.downloads == [] def test_pinned_digest_cannot_be_replaced_by_another_valid_digest(provider: EvidenceProvider) -> None: + identity = json.dumps(provider.verify()) + provider.downloads.clear() provider.artifact["digest"] = "sha256:" + "c" * 64 - with pytest.raises(ValueError, match="approved evidence identity changed"): - provider.adapter.verify(json.dumps(PIN)) + with pytest.raises(ValueError, match="approved artifact identity changed"): + provider.verify(identity) assert provider.downloads == [] @@ -404,10 +502,10 @@ def test_pinned_digest_cannot_be_replaced_by_another_valid_digest(provider: Evid ("path", "value"), [ (("schema",), 2), - (("commit",), "c" * 40), - (("run_id",), 102), - (("run_attempt",), 1), - (("repository_id",), 12), + (("run", "head_sha"), "c" * 40), + (("run", "id"), 102), + (("run", "run_attempt"), 1), + (("run", "repository_id"), 12), (("package", "version"), "0.5.41"), (("jobs", "build (ubuntu-latest)"), 999), ], @@ -417,9 +515,9 @@ def test_signed_manifest_must_equal_current_package_and_native_identities(provid for key in path[:-1]: target = target[key] target[path[-1]] = value - with pytest.raises(ValueError, match="signed evidence differs from current exact-commit Cargo and native validation identities"): - provider.adapter.verify() - assert len(provider.gh_calls) == 1 + with pytest.raises(ValueError, match="retained evidence differs from current exact source"): + provider.verify() + assert provider.gh_calls == [] assert not provider.downloads[0][2].exists() @@ -434,8 +532,8 @@ def test_signed_manifest_must_equal_current_package_and_native_identities(provid ) def test_verified_attestation_must_name_this_push_invocation(provider: EvidenceProvider, field: str, value: str) -> None: provider.signature[0]["verificationResult"]["signature"]["certificate"][field] = value - with pytest.raises(ValueError, match="attestation run, attempt or push event differs"): - provider.adapter.verify() + with pytest.raises(ValueError, match="attestation certificate run, attempt or expected event differs"): + provider.verify() assert not provider.downloads[0][2].exists() @@ -443,13 +541,13 @@ def test_verified_attestation_must_name_this_push_invocation(provider: EvidenceP def test_duplicate_keys_cannot_replace_pinned_signed_or_verified_claims(provider: EvidenceProvider, source: str) -> None: identity = None if source == "pin": - identity = json.dumps(PIN).replace('"run_id": 101', '"run_id": 101, "run_id": 101', 1) + identity = '{"run": {"id": 101, "id": 101}}' elif source == "manifest": - provider.document_text = json.dumps(provider.document).replace('"schema": 1', '"schema": 1, "schema": 1', 1) + provider.document_text = json.dumps(provider.document).replace('"schema":', '"schema": 1, "schema":', 1) else: provider.signature_text = json.dumps(provider.signature).replace('"buildTrigger": "push"', '"buildTrigger": "push", "buildTrigger": "push"', 1) - with pytest.raises(ValueError, match="duplicate evidence key"): - provider.adapter.verify(identity) + with pytest.raises(ValueError, match="duplicate JSON key"): + provider.verify(identity) if source == "pin": assert provider.api_calls == [] else: @@ -458,18 +556,18 @@ def test_duplicate_keys_cannot_replace_pinned_signed_or_verified_claims(provider def test_successful_verification_rejects_a_rerun_started_during_consumption(provider: EvidenceProvider) -> None: provider.responses[RUN_ENDPOINT] = {**provider.run, "run_attempt": 3} - with pytest.raises(ValueError, match="selected CI attempt is stale"): - provider.adapter.verify() + with pytest.raises(ValueError, match="selected run or attempt changed"): + provider.verify() assert len(provider.gh_calls) == 1 assert not provider.downloads[0][2].exists() def test_successful_verification_rechecks_immutable_artifact_identity(provider: EvidenceProvider) -> None: provider.responses[ARTIFACT_ENDPOINT] = {**provider.artifact, "digest": "sha256:" + "c" * 64} - with pytest.raises(ValueError, match="evidence artifact changed during verification"): - provider.adapter.verify() - assert len(provider.gh_calls) == 1 - assert not provider.downloads[0][2].exists() + with pytest.raises(ValueError, match="artifact identity changed during verification"): + provider.verify() + assert provider.gh_calls == [] + assert provider.downloads == [] @pytest.mark.parametrize("binary", [False, True]) @@ -479,7 +577,7 @@ def test_cli_preserves_provider_failure_diagnostics( ) -> None: command = ["gh", "attestation", "verify", "validation.json"] output = b"provider response" if binary else "provider response" - diagnostic = b"artifact 301 expired\xff" if binary else "artifact 301 expired" + diagnostic = b"artifact 301 expired" if binary else "artifact 301 expired" error = ( subprocess.TimeoutExpired(command, 30, output=output, stderr=diagnostic) if timeout @@ -489,104 +587,49 @@ def test_cli_preserves_provider_failure_diagnostics( def fail(*args: str) -> str: raise error - monkeypatch.setattr(provider.adapter, "gh", fail) - monkeypatch.setattr(sys, "argv", ["release_evidence", "verify"]) - assert provider.adapter.main() == 1 + monkeypatch.setattr(release_validation, "gh", fail) + assert provider.cli() == 1 captured = capsys.readouterr() assert captured.out == "" assert "artifact 301 expired" in captured.err assert "provider response" in captured.err - assert "timed out after 30 seconds" in captured.err if timeout else "exit status 1" in captured.err - assert not provider.downloads[0][2].exists() - - -def test_cli_success_emits_only_the_pinned_json_identity( - provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - monkeypatch.setattr(sys, "argv", ["release_evidence", "verify"]) - assert provider.adapter.main() == 0 - captured = capsys.readouterr() - assert captured.out == PIN_JSON + "\n" - assert captured.err == "" - - -@pytest.mark.parametrize("pinned", [False, True]) -def test_cli_exports_verified_identity_with_installed_helper_and_preserves_records( - provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], pinned: bool -) -> None: - output = provider.root / "github output" - original = b"prior=value\r\n" - output.write_bytes(original) - args = ["release_evidence", "verify", "--github-output", str(output)] - if pinned: - args.extend(["--identity", PIN_JSON]) - monkeypatch.setattr(sys, "argv", args) - assert provider.adapter.main() == 0 - assert output.read_bytes() == original + f"identity={PIN_JSON}\n".encode() - assert provider.gh_calls - assert capsys.readouterr() == ("", "") + assert not (provider.root / "selection.json").exists() @pytest.mark.parametrize("failure", ["invalid-pin", "changed-pin", "failed-job", "signature", "late-artifact-change"]) -def test_cli_verification_failure_never_exports_identity( - provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], failure: str -) -> None: +def test_cli_verification_failure_never_exports_identity(provider: EvidenceProvider, capsys: pytest.CaptureFixture[str], failure: str) -> None: + selection = provider.verify() output = provider.root / "github output" original = b"prior=value\n" output.write_bytes(original) - args = ["release_evidence", "verify", "--github-output", str(output)] - if failure == "invalid-pin": - args.extend(["--identity", "{}"]) - elif failure == "changed-pin": - args.extend(["--identity", json.dumps({**PIN, "artifact_digest": "sha256:" + "c" * 64})]) + expected = provider.root / "approved.json" + args = ["--github-output", str(output)] + if failure in {"invalid-pin", "changed-pin"}: + selection["artifact"]["digest"] = "sha256:" + "c" * 64 + expected.write_text("{}" if failure == "invalid-pin" else json.dumps(selection), encoding="utf-8") + args += ["--expected", str(expected)] elif failure == "failed-job": provider.jobs[3]["steps"][0]["conclusion"] = "failure" elif failure == "signature": provider.signature[0]["verificationResult"]["signature"]["certificate"]["buildTrigger"] = "workflow_dispatch" else: provider.responses[ARTIFACT_ENDPOINT] = {**provider.artifact, "digest": "sha256:" + "c" * 64} - monkeypatch.setattr(sys, "argv", args) - assert provider.adapter.main() == 1 + assert provider.cli(*args) == 1 assert output.read_bytes() == original - captured = capsys.readouterr() - assert captured.out == "" - assert "Release evidence rejected:" in captured.err + assert not (provider.root / "selection.json").exists() + assert capsys.readouterr().out == "" -@pytest.mark.parametrize("value", [float("nan"), float("inf"), -float("inf")]) -def test_cli_rejects_nonfinite_serialization_before_export( - provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], value: float -) -> None: - output = provider.root / "github output" - output.write_bytes(b"prior=value\n") - # Guard the transport even if a future verifier returns a non-finite value. - monkeypatch.setattr(provider.adapter, "verify", lambda _identity: {**PIN, "run_id": value}) - monkeypatch.setattr(sys, "argv", ["release_evidence", "verify", "--github-output", str(output)]) - assert provider.adapter.main() == 1 - assert output.read_bytes() == b"prior=value\n" - captured = capsys.readouterr() - assert captured.out == "" - assert "Out of range float values" in captured.err +@pytest.mark.parametrize("index", range(3)) +def test_native_job_requires_its_configured_host(provider: EvidenceProvider, index: int) -> None: + provider.jobs[index]["labels"] = ["self-hosted"] + with pytest.raises(ValueError, match="required hosted platform"): + provider.verify() + assert provider.downloads == [] -def test_cli_export_permission_failure_is_not_reported_as_success( - provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] -) -> None: - output = provider.root / "github output" - output.write_bytes(b"prior=value\n") - original_open = Path.open - - def open_file(path: Path, mode: str = "r", *args: Any, **kwargs: Any) -> Any: - if path == output and mode == "ab": - raise PermissionError(13, "Permission denied", str(path)) - return original_open(path, mode, *args, **kwargs) - - # Model the OS failure portably; execute the actual installed exporter. - monkeypatch.setattr(Path, "open", open_file) - monkeypatch.setattr(sys, "argv", ["release_evidence", "verify", "--github-output", str(output)]) - assert provider.adapter.main() == 1 - assert output.read_bytes() == b"prior=value\n" - captured = capsys.readouterr() - assert captured.out == "" - assert "Permission denied" in captured.err - assert repr(str(output)) in captured.err +def test_signed_manifest_retains_one_mib_limit(provider: EvidenceProvider) -> None: + provider.document_text = " " * (1024 * 1024 + 1) + with pytest.raises(ValueError, match=r"size|bound"): + provider.verify() + assert provider.gh_calls == [] diff --git a/tests/tooling/test_release_performance.py b/tests/tooling/test_release_performance.py index 918990a..1fd9b4b 100644 --- a/tests/tooling/test_release_performance.py +++ b/tests/tooling/test_release_performance.py @@ -1,28 +1,31 @@ -"""Release preparation saves assertions; README retries never repeat measurement.""" +"""Installed preparation saves independent assertions and rejects stale inputs.""" import subprocess +from dataclasses import replace from datetime import UTC, datetime from pathlib import Path import pytest +from research_repo_tools import performance_release +from research_repo_tools.cli import main from research_repo_tools.just_inspect import dry_run from research_repo_tools.process import run_git_command from research_repo_tools.releases import PublishedRelease -from tooling import release_performance as adapter - ROOT = Path(__file__).resolve().parents[2] +INPUTS = "target/bench-reports/release-inputs.json" @pytest.fixture def consumer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - """Give preparation real local Git commits and the consumer's declared policy.""" root = tmp_path / "consumer with spaces" root.mkdir() (root / "Cargo.toml").write_text('[package]\nname = "fixture"\nversion = "0.5.2"\n', encoding="utf-8") - config = root / "tooling/measurements/full.toml" - config.parent.mkdir(parents=True) - config.write_bytes((ROOT / "tooling/measurements/full.toml").read_bytes()) + (root / ".gitignore").write_bytes(b"/target/\n") + for name in ("tooling/measurements/full.toml", "tooling/performance-release.toml"): + path = root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes((ROOT / name).read_bytes()) monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(tmp_path / "absent-gitconfig")) monkeypatch.setenv("GIT_CONFIG_NOSYSTEM", "1") run_git_command(["--no-pager", "init", "--quiet"], cwd=root) @@ -50,141 +53,120 @@ def consumer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: return root +def settings(root: Path) -> performance_release.PerformanceRelease: + return performance_release.load_performance_release(root, "tooling/performance-release.toml") + + def releases(monkeypatch: pytest.MonkeyPatch, *tags: str) -> None: - """Keep provider discovery offline while exercising the real shared selector.""" history = tuple(PublishedRelease(tag, datetime(2026, 10, index, tzinfo=UTC)) for index, tag in enumerate(tags, start=1)) - monkeypatch.setattr(adapter, "published_releases", lambda _root, **_kwargs: history) + monkeypatch.setattr(performance_release, "published_releases", lambda _root, **_kwargs: history) @pytest.mark.parametrize("published", [False, True]) -def test_preparation_resolves_prospective_head_or_published_tag( - consumer: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - published: bool, +def test_preparation_resolves_prospective_head_or_publication_order( + consumer: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], published: bool ) -> None: current = "v0.5.1" if published else "v0.5.2" (consumer / "Cargo.toml").write_text(f'[package]\nname = "fixture"\nversion = "{current[1:]}"\n', encoding="utf-8") releases(monkeypatch, *(("v0.5.0", "v0.5.1") if published else ("v0.5.1", "v0.5.0"))) - monkeypatch.setattr(adapter, "tooling_main", lambda _argv: pytest.fail("preparation must not measure or publish")) before = run_git_command(["--no-pager", "show-ref"], cwd=consumer).stdout - assert adapter.main(["capture"]) == 0 - output = capsys.readouterr().out - assert "target/bench-reports/release-inputs.json (relative to the repository root; ignored by Git)" in output - assert "No shell variables or exports are needed" in output - assert "This command does not set PREVIOUS_TAG, CURRENT_COMMIT, or PREVIOUS_COMMIT in your shell" in output - assert "saved file survives opening a new terminal" in output - assert "After measurements finish, run: just performance-readme (no arguments)" in output - assert "performance-readme reads the release tags and commits from this saved file" in output - expected = [ - current, - run_git_command(["--no-pager", "rev-parse", "v0.5.1" if published else "HEAD"], cwd=consumer).stdout.strip(), - "v0.5.0", - run_git_command(["--no-pager", "rev-parse", "v0.5.0"], cwd=consumer).stdout.strip(), - ] - assert adapter.saved_assertions(consumer) == expected + assert main(["performance", "prepare", "tooling/performance-release.toml"]) == 0 + assert f"Saved preparation: {INPUTS}" in capsys.readouterr().out + record = performance_release.load_preparation(consumer, settings(consumer)) + assert record.identities == { + "current": (current, run_git_command(["--no-pager", "rev-parse", "v0.5.1" if published else "HEAD"], cwd=consumer).stdout.strip()), + "baseline": ("v0.5.0", run_git_command(["--no-pager", "rev-parse", "v0.5.0"], cwd=consumer).stdout.strip()), + } assert run_git_command(["--no-pager", "show-ref"], cwd=consumer).stdout == before + assert not (consumer / "target/bench-reports/full").exists() -def test_explicit_pair_skips_provider_and_failed_capture_preserves_record(consumer: Path, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(adapter, "published_releases", lambda _root, **_kwargs: pytest.fail("explicit pairs need no provider")) - assert adapter.main(["capture", "v0.5.1", "v0.5.0"]) == 0 - original = (consumer / adapter.INPUTS).read_bytes() - assert adapter.main(["capture", "v0.5.2", "v0.5.0"]) == 1 # Missing current tag. - assert (consumer / adapter.INPUTS).read_bytes() == original +def test_explicit_pair_skips_provider_and_failed_preparation_preserves_record(consumer: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(performance_release, "published_releases", lambda *_args, **_kwargs: pytest.fail("explicit pairs need no provider")) + plan = replace(settings(consumer), mode="explicit", current="v0.5.1", baseline="v0.5.0") + performance_release.prepare_performance(consumer, plan) + original = (consumer / INPUTS).read_bytes() + with pytest.raises(subprocess.CalledProcessError): + performance_release.prepare_performance(consumer, replace(plan, current="v0.5.2")) + assert (consumer / INPUTS).read_bytes() == original -@pytest.mark.parametrize("failure", ["empty-history", "provider", "missing-tag", "partial-pair", "old-release"]) +@pytest.mark.parametrize("failure", ["empty-history", "provider", "missing-tag", "old-release"]) def test_failed_preparation_never_writes_partial_assertions( - consumer: Path, - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], - failure: str, + consumer: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], failure: str ) -> None: releases(monkeypatch, *(() if failure == "empty-history" else ("v0.4.6" if failure == "old-release" else "v0.5.1",))) if failure == "missing-tag": run_git_command(["--no-pager", "tag", "-d", "v0.5.1"], cwd=consumer) if failure == "provider": - def fail_provider(root: Path, repository: str) -> tuple[PublishedRelease, ...]: + def unavailable(*_args: object, **_kwargs: object) -> tuple[PublishedRelease, ...]: raise subprocess.CalledProcessError(1, ["gh", "release", "list"], stderr="provider unavailable") - monkeypatch.setattr(adapter, "published_releases", fail_provider) - args = ["capture", "v0.5.2"] if failure == "partial-pair" else ["capture"] - assert adapter.main(args) == 1 - assert not (consumer / adapter.INPUTS).exists() - output = capsys.readouterr() - assert "Release performance rejected:" in output.err - assert "performance-prepare" in output.err + monkeypatch.setattr(performance_release, "published_releases", unavailable) + assert main(["performance", "prepare", "tooling/performance-release.toml"]) == 1 + assert not (consumer / INPUTS).exists() expected = { "empty-history": "a published stable release is required", "provider": "provider unavailable", "missing-tag": "refs/tags/v0.5.1^{commit}", - "partial-pair": "current and baseline tags must be provided together", "old-release": "requires v0.5.0 or newer", } - assert expected[failure] in output.err - - -def test_readme_retry_uses_saved_assertions_and_preserves_measurements(consumer: Path, monkeypatch: pytest.MonkeyPatch) -> None: - assert adapter.main(["capture", "v0.5.1", "v0.5.0"]) == 0 - expected = adapter.saved_assertions(consumer) - payload = consumer / "target/bench-reports/full/run.json" - payload.parent.mkdir() - payload.write_bytes(b"retained measurements must not be touched") - before = payload.read_bytes() - calls: list[list[str]] = [] - - def publication(argv: list[str]) -> int: - calls.append(argv) - return 7 if len(calls) == 1 else 0 - - monkeypatch.setattr(adapter, "tooling_main", publication) - monkeypatch.setattr(adapter, "published_releases", lambda _root, **_kwargs: pytest.fail("publication must not rediscover inputs")) - for name in ("CURRENT_COMMIT", "PREVIOUS_TAG", "PREVIOUS_COMMIT"): - monkeypatch.delenv(name, raising=False) - assert adapter.main(["publish"]) == 7 - assert adapter.main(["publish"]) == 0 - assert ( - calls - == [ + assert expected[failure] in capsys.readouterr().err + + +@pytest.mark.parametrize("changed", ["head", "baseline", "declaration", "package"]) +def test_readme_rejects_stale_preparation_before_publication(consumer: Path, monkeypatch: pytest.MonkeyPatch, changed: str) -> None: + releases(monkeypatch, "v0.5.1") + performance_release.prepare_performance(consumer, settings(consumer)) + if changed == "head": + run_git_command( [ - "performance", - "publish", - "tooling/performance-readme.toml", - "--current-release", - expected[0], - "--current-revision", - expected[1], - "--baseline-release", - expected[2], - "--baseline-revision", - expected[3], - ] - ] - * 2 - ) - assert payload.read_bytes() == before - - -@pytest.mark.parametrize("record", [None, "{", '{"schema": 2, "assertions": []}', '{"schema": 1, "assertions": ["v0.5.1", "", "v0.5.0", ""]}']) -def test_missing_or_invalid_preparation_stops_before_publication(consumer: Path, monkeypatch: pytest.MonkeyPatch, record: str | None) -> None: + "--no-pager", + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + "commit", + "--allow-empty", + "--quiet", + "-m", + "later", + ], + cwd=consumer, + ) + elif changed == "baseline": + run_git_command(["--no-pager", "tag", "--force", "v0.5.1", "HEAD"], cwd=consumer) + elif changed == "declaration": + path = consumer / "tooling/measurements/full.toml" + path.write_bytes(path.read_bytes() + b"\n# changed\n") + else: + (consumer / "Cargo.toml").write_bytes(b'[package]\nname="fixture"\nversion="0.5.3"\n') + monkeypatch.setattr(performance_release, "published_releases", lambda *_args, **_kwargs: pytest.fail("README must not rediscover")) + with pytest.raises(ValueError, match="stale preparation"): + performance_release.load_readme_plan(consumer, settings(consumer)) + + +@pytest.mark.parametrize("record", [None, "{", '{"schema": 2}', '{"schema": 1, "assertions": []}']) +def test_missing_or_legacy_preparation_stops_before_readme_and_measurement(consumer: Path, record: str | None) -> None: if record is not None: - path = consumer / adapter.INPUTS + path = consumer / INPUTS path.parent.mkdir(parents=True) path.write_text(record, encoding="utf-8") - monkeypatch.setattr(adapter, "tooling_main", lambda _argv: pytest.fail("invalid preparation cannot publish")) - assert adapter.main(["publish"]) == 1 - assert adapter.main(["publish", "v0.5.1"]) == 1 + for command in ("readme", "release"): + assert main(["performance", command, "tooling/performance-release.toml"]) == 1 + assert not (consumer / "target/bench-reports/full").exists() def test_recipes_keep_preparation_measurement_and_publication_separate() -> None: preparation = dry_run(ROOT, "performance-prepare").stderr - assert preparation.index("git fetch origin --tags") < preparation.index("tooling.release_performance capture") + assert 'performance prepare tooling/performance-release.toml "$@"' in preparation + assert "git fetch" not in preparation measurement = dry_run(ROOT, "performance-release").stderr - assert measurement.count("performance measure") == measurement.count("performance promote") == 1 - assert "performance publish" not in measurement - assert "release_performance" not in measurement + assert measurement.count("performance release tooling/performance-release.toml") == 1 + assert "performance readme" not in measurement publication = dry_run(ROOT, "performance-readme").stderr - assert "tooling.release_performance publish" in publication + assert "performance readme tooling/performance-release.toml" in publication assert "performance measure" not in publication diff --git a/tests/tooling/test_release_publishing.py b/tests/tooling/test_release_publishing.py index ba05379..cc616d4 100644 --- a/tests/tooling/test_release_publishing.py +++ b/tests/tooling/test_release_publishing.py @@ -3,6 +3,7 @@ import io import json import os +import runpy import shlex import subprocess import sys @@ -13,13 +14,12 @@ import pytest import yaml -from research_repo_tools import changelog, config, release_publishing +from research_repo_tools import changelog, config, process, release_publishing from research_repo_tools.cli import main from research_repo_tools.just_inspect import dry_run from research_repo_tools.process import run_command from tests.tooling.test_release_evidence import PIN_JSON, EvidenceProvider -from tooling import release_evidence REPO_ROOT = Path(__file__).resolve().parents[2] WORKFLOW = REPO_ROOT / ".github/workflows/publish-crates.yml" @@ -202,7 +202,7 @@ def test_workflow_requires_approval_revalidation_and_absence_before_oidc() -> No gate = steps[2]["run"] assert 'just release-gate "$RELEASE_TAG"' in gate assert 'test -n "$CI_EVIDENCE"' in gate - assert 'just release-evidence --identity "$CI_EVIDENCE"' in gate + assert 'just release-evidence --expected "$RUNNER_TEMP/approved-evidence.json"' in gate assert steps[2]["env"]["CI_EVIDENCE"] == "${{ needs.verify.outputs.ci-evidence }}" assert 'release registry "$RELEASE_TAG" --expect absent' in gate assert steps[3]["id"] == "auth" @@ -221,7 +221,7 @@ def test_workflow_requires_approval_revalidation_and_absence_before_oidc() -> No def run_evidence_fixture() -> int: """Run the real verifier in a child process with simulated provider responses.""" root = Path(os.environ["FIXTURE_ROOT"]) - provider = EvidenceProvider(release_evidence, root) + provider = EvidenceProvider(root) if os.environ.get("FIXTURE_FAILURE") == "failed-job": provider.jobs[3]["steps"][0]["conclusion"] = "failure" newline = os.environ.get("FIXTURE_DIAGNOSTIC_NEWLINE", os.linesep) @@ -229,22 +229,22 @@ def run_evidence_fixture() -> int: sys.stdout.flush() print("fixture stderr diagnostic", file=sys.stderr) diagnostic = io.StringIO() + script, *arguments = sys.argv[1:] with pytest.MonkeyPatch.context() as patch: - patch.setattr(release_evidence, "ROOT", root) - patch.setattr(release_evidence, "api", provider.api) - patch.setattr(release_evidence, "gh", provider.gh) - patch.setattr(release_evidence, "download_artifact", provider.download) - patch.setattr(release_evidence, "run_git_command", provider.git) + provider.install(patch) + patch.setattr(process, "run_git_command", provider.git) with redirect_stderr(diagnostic): try: - status = release_evidence.main() + sys.argv[:] = [script, *arguments] + runpy.run_path(script, run_name="__main__") + status = 0 except SystemExit as error: status = error.code assert isinstance(status, int) print(diagnostic.getvalue(), end="", file=sys.stderr) # Just's quiet mode hides stderr. Record the actual verifier outcome so an # import/shell failure cannot masquerade as the rejection under test. - result = {"status": status, "diagnostic": diagnostic.getvalue(), "import_root": sys.path[0], "arguments": sys.argv[1:]} + result = {"status": status, "diagnostic": diagnostic.getvalue(), "import_root": sys.path[0], "arguments": arguments} (root / "verifier-result.json").write_text(json.dumps(result), encoding="utf-8") return status @@ -252,19 +252,24 @@ def run_evidence_fixture() -> int: @pytest.fixture def evidence_runner(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: """Keep real workflow shell, Just, uv, Python, verifier and output writer.""" - (tmp_path / "justfile").write_text(f"import '{(REPO_ROOT / 'justfile').as_posix()}'\n", encoding="utf-8") + source = (REPO_ROOT / "justfile").read_text(encoding="utf-8") + runtime = '[script("uvx", "--python", "3.15", "--from", "research-repo-tools==0.2.4", "python", "-I")]' + assert source.count(runtime) == 2 + # Run the exact recipe body/argument transport with the locked test runtime. + # The separate isolated-runtime smoke and pin test own uvx installation. + source = source.replace(runtime, '[script("uv", "run", "--locked", "--no-sync", "python", "fixture-bootstrap.py")]') + (tmp_path / "justfile").write_text(source, encoding="utf-8", newline="\n") (tmp_path / "Cargo.toml").write_text('[package]\nname = "la-stack"\nversion = "0.5.42"\n', encoding="utf-8") + (tmp_path / "tooling").mkdir() + (tmp_path / "tooling/release-evidence.toml").write_bytes((REPO_ROOT / "tooling/release-evidence.toml").read_bytes()) for name in ("BASH_ENV", "SHELLOPTS", "BASHOPTS", "JUST_QUIET", "JUST_DRY_RUN", "FIXTURE_FAILURE", "FIXTURE_DIAGNOSTIC_NEWLINE", "PYTHONPATH"): monkeypatch.delenv(name, raising=False) monkeypatch.setenv("FIXTURE_ROOT", str(tmp_path)) monkeypatch.setenv("FIXTURE_REPO_ROOT", str(REPO_ROOT)) + monkeypatch.setenv("RUNNER_TEMP", tmp_path.as_posix()) monkeypatch.setenv("UV_PROJECT", str(REPO_ROOT)) monkeypatch.setenv("UV_PROJECT_ENVIRONMENT", sys.prefix) - # Just executes the entry point relative to the fixture checkout. Supply - # provider responses there while retaining the actual native uv/Python launch. - bootstrap = tmp_path / "tooling/release_evidence.py" - bootstrap.parent.mkdir() - bootstrap.write_text( + (tmp_path / "fixture-bootstrap.py").write_text( 'import os, sys\nsys.path.insert(0, os.environ["FIXTURE_REPO_ROOT"])\n' "from tests.tooling.test_release_publishing import run_evidence_fixture\nsys.exit(run_evidence_fixture())\n", encoding="utf-8", @@ -311,24 +316,26 @@ def test_evidence_export_uses_real_just_and_installed_writer_without_stdout_capt else: assert report.exists(), result.stderr observed = json.loads(report.read_text(encoding="utf-8")) - assert observed["arguments"][:2] == ["verify", "--github-output"] - assert len(observed["arguments"]) == 3 + assert observed["arguments"][:1] == ["--github-output"] + assert len(observed["arguments"]) == 2 if outcome == "empty-output": - assert observed["arguments"][2] == "" + assert observed["arguments"][1] == "" else: - assert Path(observed["arguments"][2]) == output + assert Path(observed["arguments"][1]) == output assert observed["status"] == (0 if outcome == "valid" else 1) - rejection = ( - "did not validate required step Verify default and exact Cargo packages" - if outcome == "failed-job" - else "GitHub Actions output destination must be an existing regular file" - ) + rejection = { + "failed-job": "did not validate required step Verify default and exact Cargo packages", + "missing-file": repr(str(output)), + "directory": "release handoff must be a regular file without links", + "empty-output": "A publication transaction cannot contain overlapping target paths", + "valid": "", + }[outcome] assert observed["diagnostic"] == "" if outcome == "valid" else rejection in observed["diagnostic"] if outcome == "valid": - assert output.read_bytes() == original + f"identity={PIN_JSON}\n".encode() + assert output.read_bytes() == original + f"identity={PIN_JSON}\ncommit={SHA}\n".encode() if not quiet: # Diagnostics are text; the command-file bytes above must stay LF. - assert result.stdout.splitlines() == ["fixture stdout diagnostic"] + assert result.stdout.splitlines() == ["fixture stdout diagnostic", "Saved verified release evidence."] assert "fixture stderr diagnostic" in result.stderr elif outcome == "directory": assert output.is_dir() @@ -358,15 +365,15 @@ def test_evidence_bootstrap_keeps_windows_import_paths_as_environment_data( script = workflow["jobs"]["verify"]["steps"][-1]["run"] result = run_evidence_shell(script, tmp_path) assert result.returncode == 0, result.stderr - assert result.stdout == "fixture stdout diagnostic" + newline + assert result.stdout == "fixture stdout diagnostic" + newline + "Saved verified release evidence." + os.linesep observed = json.loads((tmp_path / "verifier-result.json").read_text(encoding="utf-8")) assert observed["status"] == 0 assert observed["diagnostic"] == "" assert observed["import_root"] == import_root - assert observed["arguments"][:2] == ["verify", "--github-output"] - assert len(observed["arguments"]) == 3 - assert Path(observed["arguments"][2]) == output - assert output.read_bytes() == f"identity={PIN_JSON}\n".encode() + assert observed["arguments"][:1] == ["--github-output"] + assert len(observed["arguments"]) == 2 + assert Path(observed["arguments"][1]) == output + assert output.read_bytes() == f"identity={PIN_JSON}\ncommit={SHA}\n".encode() @pytest.mark.usefixtures("evidence_runner") @@ -386,8 +393,8 @@ def test_exported_identity_is_revalidated_by_real_just_before_registry_and_token assert json.loads(report.read_text(encoding="utf-8"))["status"] == 0 report.unlink() record = output.read_text(encoding="utf-8") - assert record == f"identity={PIN_JSON}\n" - identity = record.removeprefix("identity=").removesuffix("\n") + assert record == f"identity={PIN_JSON}\ncommit={SHA}\n" + identity = record.splitlines()[0].removeprefix("identity=") variants = {"valid": identity, "empty": "", "invalid": "{", "changed": identity.replace('"run_attempt":2', '"run_attempt":1')} monkeypatch.setenv("CI_EVIDENCE", variants[pin]) monkeypatch.setenv("RELEASE_TAG", TAG) @@ -402,7 +409,9 @@ def test_exported_identity_is_revalidated_by_real_just_before_registry_and_token if pin != "empty": assert report.exists(), result.stderr observed = json.loads(report.read_text(encoding="utf-8")) - assert observed["arguments"] == ["verify", "--identity", variants[pin]] + assert observed["arguments"][:1] == ["--expected"] + assert Path(observed["arguments"][1]) == tmp_path / "approved-evidence.json" + assert (tmp_path / "approved-evidence.json").read_text(encoding="utf-8") == variants[pin] + "\n" assert result.returncode == (0 if pin == "valid" else 1), result.stderr assert registry.exists() == (pin == "valid") assert token.exists() == (pin == "valid") @@ -411,7 +420,7 @@ def test_exported_identity_is_revalidated_by_real_just_before_registry_and_token else: observed = json.loads(report.read_text(encoding="utf-8")) assert observed["status"] == (0 if pin == "valid" else 1) - rejection = "Expecting property name" if pin == "invalid" else "selected CI attempt is stale" + rejection = "Expecting property name" if pin == "invalid" else "selected run or attempt changed" assert observed["diagnostic"] == "" if pin == "valid" else rejection in observed["diagnostic"] @@ -423,9 +432,11 @@ def test_explicit_empty_identity_cannot_select_fresh_evidence(tmp_path: Path, mo original = b"prior=value\n" output.write_bytes(original) monkeypatch.setenv("GITHUB_OUTPUT", output.as_posix()) - result = run_evidence_shell('just release-evidence --identity "" --github-output "$GITHUB_OUTPUT"\n', tmp_path) + expected = tmp_path / "empty.json" + expected.write_bytes(b"") + result = run_evidence_shell('just release-evidence --expected empty.json --github-output "$GITHUB_OUTPUT"\n', tmp_path) observed = json.loads((tmp_path / "verifier-result.json").read_text(encoding="utf-8")) - assert observed["arguments"][:3] == ["verify", "--identity", ""] + assert observed["arguments"][:2] == ["--expected", "empty.json"] assert result.returncode == observed["status"] == 1 assert "Expecting value" in observed["diagnostic"] assert output.read_bytes() == original @@ -449,6 +460,7 @@ def test_actual_preupload_shell_requires_evidence_and_proven_absence(tmp_path: P **{key: value for key, value in os.environ.items() if key not in {"BASH_ENV", "SHELLOPTS", "BASHOPTS"}}, "RELEASE_TAG": TAG, "CI_EVIDENCE": "" if failure == "empty-pin" else '{"run_id": 11}', + "RUNNER_TEMP": tmp_path.as_posix(), }, check=False, ) @@ -469,7 +481,10 @@ def test_main_ci_seals_only_complete_native_and_cargo_validation() -> None: assert evidence["if"] == "github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'acgetchell/la-stack'" assert evidence["permissions"] == {"contents": "read", "actions": "read", "attestations": "write", "id-token": "write"} steps = {step["name"]: step for step in evidence["steps"] if "name" in step} - assert steps["Capture successful native validation identities"]["run"].endswith("seal --output target/release-evidence/validation.json") + capture = steps["Capture successful native validation identities"]["run"] + assert "uvx --python 3.15 --from research-repo-tools==0.2.4 python" in capture + assert '"release", "evidence", "create", "tooling/release-evidence.toml"' in capture + assert '"--output", "target/release-evidence/validation.json"' in capture assert steps["Sign original main validation evidence"]["with"]["subject-path"] == "target/release-evidence/validation.json" upload = steps["Upload signed validation evidence"]["with"] assert upload == { diff --git a/tests/tooling/test_toolchain_integration.py b/tests/tooling/test_toolchain_integration.py index 65340f0..9abde2e 100644 --- a/tests/tooling/test_toolchain_integration.py +++ b/tests/tooling/test_toolchain_integration.py @@ -68,6 +68,36 @@ def test_published_tooling_package_matches_manifest_lock_and_installation() -> N assert locked[name] == version(name) == expected +def test_isolated_release_runtime_uses_the_published_pin_outside_checkout(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Exercise the actual trusted runtime on each native CI platform.""" + expected = version("research-repo-tools") + source = (REPO_ROOT / "justfile").read_text(encoding="utf-8") + assert source.count(f'[script("uvx", "--python", "3.15", "--from", "research-repo-tools=={expected}", "python", "-I")]') == 2 + (tmp_path / "research_repo_tools.py").write_text('raise RuntimeError("untrusted checkout import")\n', encoding="utf-8") + monkeypatch.setenv("PYTHONPATH", str(tmp_path)) + result = run_command( + "uvx", + [ + "--python", + "3.15", + "--from", + f"research-repo-tools=={expected}", + "python", + "-I", + "-c", + ( + "import json, research_repo_tools; from importlib.metadata import version; " + 'print(json.dumps([version("research-repo-tools"), research_repo_tools.__file__]))' + ), + ], + timeout=180, + cwd=tmp_path, + ) + installed_version, location = json.loads(result.stdout) + assert installed_version == expected + assert not Path(location).resolve().is_relative_to(REPO_ROOT) + + @pytest.fixture def consumer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: """Use the actual recipes and declarations; record updater process boundaries.""" diff --git a/tooling/README.md b/tooling/README.md index e61df71..ce39297 100644 --- a/tooling/README.md +++ b/tooling/README.md @@ -3,9 +3,8 @@ The installed `research-repo-tools` package owns common development and performance tooling. The exact released pin and registry hashes live in `pyproject.toml` and `uv.lock`. This dependency-only environment has no local -Python distribution or console entry points. `release_evidence.py` is the -repository-specific Cargo evidence adapter around the installed validation -helpers; shared tooling is not copied into this repository. +Python distribution or console entry points. Declarative release policies select +the native Cargo gates; shared tooling is not copied into this repository. ## Setup and validation @@ -111,13 +110,10 @@ external dependencies and commit pins. The [benchmark guide](../docs/BENCHMARKING.md) owns command selection, interpretation and output paths. Measurement and publication use the installed -CLI or native Cargo commands. `release_performance.py` temporarily captures the -shared release selector's independent commit assertions during -`performance-prepare` and reuses them during the separate `performance-readme` -step. `performance-release` remains measurement and report promotion only. -The adapter will be removed when the shared package publishes equivalent commands. -That handoff is tracked in -[research-repo-tools#129](https://github.com/acgetchell/research-repo-tools/issues/129). +CLI or native Cargo commands. `performance-release.toml` composes the installed +`performance prepare`, `release` and `readme` commands. Preparation retains +independent commit assertions and declaration identity; measurement/report +promotion and README publication remain separate retry boundaries. There is no runtime configuration generator. | Declaration | Responsibility | @@ -128,6 +124,7 @@ There is no runtime configuration generator. | `criterion-inventory.toml` | Hosted la-stack-only discovery and independent required scientific IDs | | `criterion-saved.toml` | Required IDs, canonical paths, 100 samples and exact saved raw JSON bytes | | `performance-report.toml` | Independent release eligibility, current report and immutable run history | +| `performance-release.toml` | Preparation storage, publication chronology and separate phase declarations | | `performance-readme.toml` | Canonical README selection, formatter, source freshness and tagged links | | `plots/{non-exact,retained,scratch}/*.toml` | Named metric/statistic/axis selections with explicit input ownership | @@ -150,8 +147,9 @@ coverage decisions, feature configurations and runtime estimates. Working-tree measurements include tracked changes and nonignored untracked regular files. Scratch outputs belong under ignored `target/`. -Tag preparation is explicit: `just performance-prepare` fetches tags and -requires authorization to mutate Git. Measurement recipes create temporary +Tag fetching is explicit: `just performance-prepare --fetch-tags --allow-git-mutations` +requires authorization to mutate Git. Plain preparation uses local tags. +Measurement recipes create temporary worktrees. Their declared environment fixes the Rust toolchain and code-generation flags; native recipes select the host target. Change and review the declarations for a different measurement environment instead of @@ -197,13 +195,10 @@ tagging and registry publication. Declarations in `pyproject.toml` retain the concept DOI, active link policy, declared date policy, formatter and authored dependency notes. -`release_changelog.py` temporarily supplies the recorded `CITATION.cff` -`date-released` when `changelog-release` (also named `changelog-unreleased`) or -a tagged `changelog-preview` omits the date. Explicit dates remain supported. -The adapter never chooses a new date or updates citation metadata; the shared -CLI still owns generation, formatting, archives and publication. -The shared replacement is tracked in -[research-repo-tools#130](https://github.com/acgetchell/research-repo-tools/issues/130). +The shared changelog command uses the recorded `CITATION.cff` date when +`changelog-release` (also named `changelog-unreleased`) or a tagged preview +omits the date. `changelog.date-source = "citation"` owns that policy; +explicit dates remain supported. Generation never advances citation metadata. The publishing declaration selects only `la-stack`, requires native platform and scientific checks on the exact commit, and requires the durable Criterion @@ -212,10 +207,11 @@ use the shared release contract; `release-package` owns native default/exact Cargo dry-runs. Main CI signs those packaging and native validation identities; `release-evidence` verifies the original main-push attempt and immutable signed artifact before publication and rechecks its pinned identity after approval. -The adapter uses the installed `ci.export_environment` helper for explicit -`--github-output` export; the [release guide](../docs/RELEASING.md#reusable-ci-evidence) -owns the transport and approval contract. Cargo evidence policy stays local. -`tests/tooling/test_release_evidence.py` checks the adapter's trust boundaries, while +`release-evidence.toml` owns the package, workflow, native job and signer policy; +`release-assets.toml` owns the draft-target policy. Both recipes use the pinned +installed runtime outside the observed checkout. The [release guide](../docs/RELEASING.md#reusable-ci-evidence) +owns the transactional output and frozen approval contract. +`tests/tooling/test_release_evidence.py` checks installed policy boundaries, while `tests/tooling/test_release_publishing.py` checks consumer policy and upload barriers without registry credentials. The release guide owns retention, complete-rerun recovery and the distinction between reused checks and Cargo repackaging. diff --git a/tooling/performance-release.toml b/tooling/performance-release.toml new file mode 100644 index 0000000..7c8782c --- /dev/null +++ b/tooling/performance-release.toml @@ -0,0 +1,11 @@ +# Independently retriable preparation, measurement/promotion and README publication. +schema = 1 +repository = "acgetchell/la-stack" +measurement = "tooling/measurements/full.toml" +report = "tooling/performance-report.toml" +publication = "tooling/performance-readme.toml" +preparation = "target/bench-reports/release-inputs.json" +payload = "target/bench-reports/full/run.json" +manifest = "target/bench-reports/full/evidence.json" +mode = "infer-release" +order = "published" diff --git a/tooling/release-assets.toml b/tooling/release-assets.toml new file mode 100644 index 0000000..1aff5c7 --- /dev/null +++ b/tooling/release-assets.toml @@ -0,0 +1,3 @@ +schema = 1 +repository = "acgetchell/la-stack" +expected-title = "tag" diff --git a/tooling/release-evidence.toml b/tooling/release-evidence.toml new file mode 100644 index 0000000..09b7b38 --- /dev/null +++ b/tooling/release-evidence.toml @@ -0,0 +1,31 @@ +schema = 1 +repository = "acgetchell/la-stack" +package = "la-stack" +workflow = ".github/workflows/ci.yml" +workflow-name = "CI" +branch = "main" +artifact-prefix = "release-evidence" +guarantee = "signed-source" + +[[jobs]] +name = "build (ubuntu-latest)" +steps = [ "Run CI checks" ] +runner-labels = [ "ubuntu-latest" ] + +[[jobs]] +name = "build (macos-latest)" +steps = [ "Run CI checks" ] +runner-labels = [ "macos-latest" ] + +[[jobs]] +name = "build (windows-latest)" +steps = [ "Run CI checks" ] +runner-labels = [ "windows-latest" ] + +[[jobs]] +name = "release-package" +steps = [ "Verify default and exact Cargo packages" ] + +[signing-job] +name = "release-evidence" +steps = [ "Capture successful native validation identities", "Sign original main validation evidence", "Upload signed validation evidence" ] diff --git a/tooling/release_changelog.py b/tooling/release_changelog.py deleted file mode 100644 index 8c1fd59..0000000 --- a/tooling/release_changelog.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Default prospective changelog dates to the release metadata already prepared. - -This temporary adapter supplies an explicit date to the installed shared CLI; -generation, retained history, formatting and publication remain upstream. -""" - -import argparse -import subprocess -import sys -from datetime import date -from pathlib import Path - -import yaml -from research_repo_tools.cli import main as tooling_main -from research_repo_tools.process import ExecutableNotFoundError, format_exception_diagnostics -from yaml.nodes import MappingNode, ScalarNode - - -def citation_date(root: Path, tag: str) -> str: - """Read one valid date for the requested citation version without using a clock.""" - document = yaml.compose((root / "CITATION.cff").read_text(encoding="utf-8"), Loader=yaml.SafeLoader) - if not isinstance(document, MappingNode): - msg = "CITATION.cff must contain a top-level mapping" - raise TypeError(msg) - - def scalar(field: str) -> str: - values = [value for key, value in document.value if isinstance(key, ScalarNode) and key.value == field] - message = f"CITATION.cff must contain exactly one nonempty top-level {field} scalar" - if len(values) != 1 or not isinstance(values[0], ScalarNode): - raise ValueError(message) - value: object = values[0].value - if not isinstance(value, str) or not value: - raise ValueError(message) - return value - - version = scalar("version") - if tag.removeprefix("v") != version: - raise ValueError(f"CITATION.cff version {version!r} does not match {tag}; run just update-version first") - released = scalar("date-released") - if date.fromisoformat(released).isoformat() != released: - msg = "CITATION.cff date-released must use YYYY-MM-DD form" - raise ValueError(msg) - return released - - -def main(argv: list[str] | None = None) -> int: - """Resolve an omitted date and delegate generation or preview to the shared CLI.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--tag") - parser.add_argument("--date") - parser.add_argument("--dry-run", action="store_true") - args = parser.parse_args(argv) - try: - released = args.date or (citation_date(Path.cwd(), args.tag) if args.tag else None) - command = ["changelog", "generate"] - if args.tag is not None: - command.extend(["--tag", args.tag]) - if released is not None: - command.extend(["--date", released]) - if args.dry_run: - command.append("--dry-run") - return tooling_main(command) - except (ExecutableNotFoundError, OSError, TypeError, ValueError, subprocess.SubprocessError, yaml.YAMLError) as error: - print(f"Release changelog rejected: {format_exception_diagnostics(error, single_line=True)}", file=sys.stderr) - print("Run just update-version first to record the matching citation version and release date, or provide an explicit date.", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tooling/release_evidence.py b/tooling/release_evidence.py deleted file mode 100644 index 85fb3c1..0000000 --- a/tooling/release_evidence.py +++ /dev/null @@ -1,261 +0,0 @@ -"""Bind Cargo publication to signed, successful main-push CI evidence. - -The installed research-repo-tools owns run/job validation and safe artifact -retrieval. This adapter supplies la-stack's Cargo policy and evidence document; -it never uploads a crate, reruns checks, or accepts substitute evidence. -""" - -import argparse -import json -import os -import re -import subprocess -import sys -import tempfile -import tomllib -from pathlib import Path -from typing import Any - -from research_repo_tools.ci import export_environment -from research_repo_tools.process import format_exception_diagnostics, run_git_command -from research_repo_tools.release_validation import ValidationPolicy, api, download_artifact, gh, select_main_run, validate_jobs, validate_run - -ROOT = Path(__file__).resolve().parents[1] -REPOSITORY = "acgetchell/la-stack" -POLICY = ValidationPolicy( - REPOSITORY, - ".github/workflows/ci.yml", - "CI", - ( - *((f"build ({platform})", ("Run CI checks",)) for platform in ("ubuntu-latest", "macos-latest", "windows-latest")), - ("release-package", ("Verify default and exact Cargo packages",)), - ), -) -EVIDENCE_STEPS = ("Capture successful native validation identities", "Sign original main validation evidence", "Upload signed validation evidence") - - -def read(endpoint: str) -> Any: - """Read authenticated JSON in this repository only.""" - return api(endpoint, repository=REPOSITORY) - - -def object_value(value: Any) -> dict[str, Any]: - """Require a JSON object at a provider boundary.""" - if not isinstance(value, dict): - msg = "expected an evidence object" - raise TypeError(msg) - return value - - -def positive_integer(value: Any) -> int: - """Reject missing, boolean, or nonpositive provider identifiers.""" - if type(value) is not int or value <= 0: - msg = "expected a positive integer identity" - raise ValueError(msg) - return value - - -def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - """Reject duplicate keys instead of silently replacing evidence fields.""" - result: dict[str, Any] = {} - for key, value in pairs: - if key in result: - raise ValueError(f"duplicate evidence key: {key}") - result[key] = value - return result - - -def pages(endpoint: str, field: str) -> list[dict[str, Any]]: - """Read a bounded, fully paginated provider collection.""" - result: list[dict[str, Any]] = [] - for page in range(1, 101): - data = object_value(read(f"{endpoint}?per_page=100&page={page}")) - entries = data.get(field) - if not isinstance(entries, list): - raise TypeError(f"invalid {field} collection") - result.extend(object_value(entry) for entry in entries) - if len(entries) < 100: - return result - raise ValueError(f"{field} exceeds the bounded page limit") - - -def checkout_commit() -> str: - """Resolve the checkout independently of event or artifact claims.""" - return run_git_command(["--no-pager", "rev-parse", "HEAD"], cwd=ROOT, timeout=30).stdout.strip() - - -def current_run(commit: str, identifier: int, attempt: int, *, completed: bool = True) -> dict[str, Any]: - """Reject reruns which replaced the selected current attempt.""" - run = object_value(read(f"actions/runs/{positive_integer(identifier)}")) - validate_run(POLICY, run, commit, object_value(read("actions/workflows/ci.yml")), completed=completed) - if run["run_attempt"] != positive_integer(attempt): - msg = "selected CI attempt is stale; inspect the complete main-push run" - raise ValueError(msg) - return run - - -def manifest(run: dict[str, Any]) -> dict[str, Any]: - """Capture native check and default/exact package job identities.""" - jobs = pages(f"actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs", "jobs") - package = tomllib.loads((ROOT / "Cargo.toml").read_text(encoding="utf-8"))["package"] - return { - "schema": 1, - "repository": REPOSITORY, - "repository_id": run["repository"]["id"], - "commit": run["head_sha"], - "run_id": run["id"], - "run_attempt": run["run_attempt"], - "jobs": validate_jobs(POLICY, run, jobs), - "package": {"name": package["name"], "version": package["version"]}, - } - - -def seal(output: Path) -> None: - """Produce the document signed by this main-push CI attempt.""" - commit = checkout_commit() - if os.environ.get("GITHUB_SHA") != commit: - msg = "executing CI source differs from checkout" - raise ValueError(msg) - run = current_run(commit, int(os.environ["GITHUB_RUN_ID"]), int(os.environ["GITHUB_RUN_ATTEMPT"]), completed=False) - document = manifest(run) - current_run(commit, run["id"], run["run_attempt"], completed=False) - output.parent.mkdir(parents=True, exist_ok=True) - with output.open("x", encoding="utf-8", newline="\n") as stream: - stream.write(json.dumps(document, sort_keys=True) + "\n") - - -def artifact_identity(data: dict[str, Any], run: dict[str, Any]) -> dict[str, Any]: - """Validate immutable artifact association before shared ZIP retrieval.""" - name = f"release-evidence-{run['id']}-{run['run_attempt']}" - association = object_value(data.get("workflow_run")) - expected = { - "id": run["id"], - "head_sha": run["head_sha"], - "head_branch": "main", - "repository_id": run["repository"]["id"], - "head_repository_id": run["head_repository"]["id"], - } - if data.get("name") != name or data.get("expired") is not False or any(association.get(key) != value for key, value in expected.items()): - msg = "evidence artifact is expired or belongs to another run, attempt, repository, ref or commit" - raise ValueError(msg) - digest = data.get("digest") - if not isinstance(digest, str) or re.fullmatch(r"sha256:[0-9a-f]{64}", digest) is None: - msg = "evidence artifact lacks an authoritative SHA-256 digest" - raise ValueError(msg) - size = positive_integer(data.get("size_in_bytes")) - if size > 1024 * 1024: - msg = "evidence artifact exceeds the one-MiB limit" - raise ValueError(msg) - return {"id": positive_integer(data.get("id")), "name": name, "digest": digest, "size_in_bytes": size} - - -def find_artifact(run: dict[str, Any]) -> dict[str, Any]: - """Require one evidence artifact for the selected attempt.""" - name = f"release-evidence-{run['id']}-{run['run_attempt']}" - matches = [entry for entry in pages(f"actions/runs/{run['id']}/artifacts", "artifacts") if entry.get("name") == name] - if len(matches) != 1: - msg = "missing or ambiguous signed CI evidence; inspect retention and rerun all CI jobs if needed" - raise ValueError(msg) - return artifact_identity(matches[0], run) - - -def verify_signature(directory: Path, run: dict[str, Any]) -> None: - """Verify Sigstore provenance, including the exact signing attempt.""" - result = json.loads( - gh( - "attestation", - "verify", - str(directory / "validation.json"), - "--bundle", - str(directory / "validation.sigstore.json"), - "--repo", - REPOSITORY, - "--signer-workflow", - f"{REPOSITORY}/{POLICY.workflow}", - "--signer-digest", - run["head_sha"], - "--source-digest", - run["head_sha"], - "--source-ref", - "refs/heads/main", - "--deny-self-hosted-runners", - "--predicate-type", - "https://slsa.dev/provenance/v1", - "--format", - "json", - ), - object_pairs_hook=unique_object, - ) - if not isinstance(result, list) or len(result) != 1: - msg = "expected exactly one verified CI attestation" - raise ValueError(msg) - verification = object_value(object_value(result[0]).get("verificationResult")) - certificate = object_value(object_value(verification.get("signature")).get("certificate")) - invocation = f"https://github.com/{REPOSITORY}/actions/runs/{run['id']}/attempts/{run['run_attempt']}" - if certificate.get("runInvocationURI") != invocation or certificate.get("buildTrigger") != "push": - msg = "attestation run, attempt or push event differs" - raise ValueError(msg) - - -def verify(identity: str | None = None) -> dict[str, Any]: - """Reuse one exact CI attempt, or recheck the same pinned identity after approval.""" - commit = checkout_commit() - pin = object_value(json.loads(identity, object_pairs_hook=unique_object)) if identity is not None else None - if pin is not None and set(pin) != {"run_id", "run_attempt", "artifact_id", "artifact_digest"}: - msg = "invalid pinned evidence identity" - raise ValueError(msg) - run = current_run(commit, pin["run_id"], pin["run_attempt"]) if pin is not None else select_main_run(POLICY, commit, read) - expected = manifest(run) - signing_policy = ValidationPolicy(REPOSITORY, POLICY.workflow, POLICY.workflow_name, (("release-evidence", EVIDENCE_STEPS),)) - validate_jobs(signing_policy, run, pages(f"actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs", "jobs")) - artifact = ( - artifact_identity(object_value(read(f"actions/artifacts/{positive_integer(pin['artifact_id'])}")), run) if pin is not None else find_artifact(run) - ) - result = {"run_id": run["id"], "run_attempt": run["run_attempt"], "artifact_id": artifact["id"], "artifact_digest": artifact["digest"]} - if pin is not None and pin != result: - msg = "approved evidence identity changed" - raise ValueError(msg) - with tempfile.TemporaryDirectory() as temporary: - directory = Path(temporary) / "evidence" - download_artifact(REPOSITORY, artifact, directory, ("validation.json", "validation.sigstore.json")) - verify_signature(directory, run) - document = json.loads((directory / "validation.json").read_text(encoding="utf-8"), object_pairs_hook=unique_object) - if document != expected: - msg = "signed evidence differs from current exact-commit Cargo and native validation identities" - raise ValueError(msg) - current_run(commit, run["id"], run["run_attempt"]) - if artifact_identity(object_value(read(f"actions/artifacts/{artifact['id']}")), run) != artifact: - msg = "evidence artifact changed during verification" - raise ValueError(msg) - return result - - -def main() -> int: - """Expose signing and read-only consumption without publication credentials.""" - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - commands.add_parser("seal").add_argument("--output", type=Path, required=True) - verification = commands.add_parser("verify") - verification.add_argument("--identity") - verification.add_argument("--github-output", type=Path, help="append identity to an existing GitHub Actions output file instead of stdout") - args = parser.parse_args() - try: - if args.command == "seal": - seal(args.output) - else: - identity = json.dumps(verify(args.identity), sort_keys=True, separators=(",", ":"), allow_nan=False) - if args.github_output is not None: - if not args.github_output.is_file(): - msg = "GitHub Actions output destination must be an existing regular file" - raise ValueError(msg) - export_environment(args.github_output, ("identity",), environment={"identity": identity}) - else: - print(identity) - except (KeyError, OSError, TypeError, ValueError, subprocess.SubprocessError) as error: - print(f"Release evidence rejected: {format_exception_diagnostics(error, single_line=True)}", file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tooling/release_performance.py b/tooling/release_performance.py deleted file mode 100644 index fb94901..0000000 --- a/tooling/release_performance.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Capture release inputs during preparation and reuse them for README publication. - -This temporary adapter composes public research-repo-tools APIs. Release -selection and measurement/publication implementations remain upstream. -""" - -import argparse -import json -import re -import subprocess -import sys -from pathlib import Path -from typing import cast - -from research_repo_tools.cli import main as tooling_main -from research_repo_tools.common_measurement import CompleteMeasurementPlan -from research_repo_tools.files import replace_many -from research_repo_tools.measurement import load_measurement -from research_repo_tools.process import ExecutableNotFoundError, format_exception_diagnostics, run_git_command -from research_repo_tools.release_metadata import read_package_info -from research_repo_tools.release_pairs import ReleasePair, resolve_pair -from research_repo_tools.releases import published_releases - -INPUTS = Path("target/bench-reports/release-inputs.json") - - -def capture(root: Path, current: str, baseline: str) -> tuple[ReleasePair, str, str]: - """Resolve independent source assertions with the same policy as measurement.""" - if bool(current) != bool(baseline): - msg = "current and baseline tags must be provided together" - raise ValueError(msg) - package_tag = "v" + read_package_info(root).version - releases = () if current else published_releases(root, repository="acgetchell/la-stack") - pair = resolve_pair( - "explicit" if current else "infer-release", - package_tag=package_tag, - releases=releases, - order="published", - current=current or None, - baseline=baseline or None, - ) - configuration = load_measurement(root, "tooling/measurements/full.toml") - policy = configuration.measurement if isinstance(configuration, CompleteMeasurementPlan) else configuration - policy.validate_release(pair.current) - policy.validate_release(pair.baseline) - tagged = bool(current) or any(item.tag == pair.current for item in releases) - - def commit(reference: str) -> str: - return run_git_command(["--no-pager", "rev-parse", "--verify", f"{reference}^{{commit}}"], cwd=root, timeout=30).stdout.strip() - - return pair, commit(f"refs/tags/{pair.current}" if tagged else "HEAD"), commit(f"refs/tags/{pair.baseline}") - - -def prepare(root: Path, current: str, baseline: str) -> None: - """Save assertions atomically after all release and commit lookups succeed.""" - pair, current_revision, baseline_revision = capture(root, current, baseline) - assertions = [pair.current, current_revision, pair.baseline, baseline_revision] - replace_many({root / INPUTS: (json.dumps({"schema": 1, "assertions": assertions}, sort_keys=True) + "\n").encode("utf-8")}) - print(f"Prepared {pair.current} ({current_revision}) vs {pair.baseline} ({baseline_revision})") - print(f"Saved release tags and commits to {INPUTS.as_posix()} (relative to the repository root; ignored by Git).") - print("No shell variables or exports are needed. This command does not set PREVIOUS_TAG, CURRENT_COMMIT, or PREVIOUS_COMMIT in your shell.") - print("The saved file survives opening a new terminal. After measurements finish, run: just performance-readme (no arguments).") - print("performance-readme reads the release tags and commits from this saved file.") - - -def saved_assertions(root: Path) -> list[str]: - """Require a complete preparation record, independent of measured evidence.""" - document = json.loads((root / INPUTS).read_text(encoding="utf-8")) - if not isinstance(document, dict) or set(document) != {"schema", "assertions"} or type(document["schema"]) is not int or document["schema"] != 1: - msg = "invalid release preparation record" - raise ValueError(msg) - assertions = document["assertions"] - if not isinstance(assertions, list) or len(assertions) != 4 or any(not isinstance(value, str) or not value for value in assertions): - msg = "release preparation requires four nonempty tag/commit assertions" - raise ValueError(msg) - pair = ReleasePair(assertions[0], assertions[2]) - if pair.current == pair.baseline or any(re.fullmatch(r"(?:[0-9a-f]{40}|[0-9a-f]{64})", assertions[index]) is None for index in (1, 3)): - msg = "release preparation requires distinct tags and full commit identities" - raise ValueError(msg) - return cast("list[str]", assertions) - - -def publish(root: Path, assertions: list[str]) -> int: - """Publish retained evidence only; never acquire or promote measurements.""" - if any(assertions) and not all(assertions): - msg = "provide all four tag/commit assertions, or omit all four to use just performance-prepare" - raise ValueError(msg) - current, current_revision, baseline, baseline_revision = assertions if all(assertions) else saved_assertions(root) - return tooling_main( - [ - "performance", - "publish", - "tooling/performance-readme.toml", - "--current-release", - current, - "--current-revision", - current_revision, - "--baseline-release", - baseline, - "--baseline-revision", - baseline_revision, - ] - ) - - -def main(argv: list[str] | None = None) -> int: - """Keep preparation and retryable README publication separate from measurements.""" - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="command", required=True) - preparation = commands.add_parser("capture") - preparation.add_argument("current", nargs="?", default="") - preparation.add_argument("baseline", nargs="?", default="") - publication = commands.add_parser("publish") - for name in ("current", "current_revision", "baseline", "baseline_revision"): - publication.add_argument(name, nargs="?", default="") - args = parser.parse_args(argv) - try: - if args.command == "capture": - prepare(Path.cwd(), args.current, args.baseline) - return 0 - return publish(Path.cwd(), [args.current, args.current_revision, args.baseline, args.baseline_revision]) - except (ExecutableNotFoundError, KeyError, OSError, TypeError, ValueError, subprocess.SubprocessError) as error: - print(f"Release performance rejected: {format_exception_diagnostics(error, single_line=True)}", file=sys.stderr) - print("Run just performance-prepare to capture inputs, or supply all four assertions to just performance-readme.", file=sys.stderr) - return 1 - - -if __name__ == "__main__": - sys.exit(main())