diff --git a/.github/workflows/publish-crates.yml b/.github/workflows/publish-crates.yml index 05f686e..c28fffc 100644 --- a/.github/workflows/publish-crates.yml +++ b/.github/workflows/publish-crates.yml @@ -54,9 +54,7 @@ jobs: GH_TOKEN: ${{ github.token }} run: | set -euo pipefail - identity=$(just release-evidence) - test -n "$identity" - printf 'identity=%s\n' "$identity" >> "$GITHUB_OUTPUT" + just release-evidence --github-output "$GITHUB_OUTPUT" publish: needs: verify @@ -89,7 +87,7 @@ jobs: set -euo pipefail just release-gate "$RELEASE_TAG" test -n "$CI_EVIDENCE" - just release-evidence "$CI_EVIDENCE" + just release-evidence --identity "$CI_EVIDENCE" uv run --locked --no-sync research-repo-tools release registry "$RELEASE_TAG" --expect absent - name: Obtain a short-lived crates.io token diff --git a/docs/RELEASING.md b/docs/RELEASING.md index b4f98d0..3a38715 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -54,9 +54,24 @@ package identities. Pending matching runs receive a bounded wait; missing, failed, ambiguous or expired evidence stops publication. PR, manual dispatch, fork, local and partial-rerun results cannot replace it. -The publishing workflow reuses these completed checks, captures the run/attempt -and artifact ID/digest, and revalidates that same identity after deployment -approval. It also retains the independent release metadata, exact-commit SARIF, +Local verification prints the validated identity as deterministic single-line +JSON. In Actions, `just release-evidence --github-output "$GITHUB_OUTPUT"` +uses the installed `research_repo_tools.ci.export_environment` helper to append +the `identity` output only after verification succeeds. The destination must +already exist; export errors fail the command. Existing output records survive, +and normal or quiet Just execution transports the same JSON without capturing +stdout. Non-finite JSON values are rejected. The shared helper owns command-file +transport; this adapter retains the Cargo-specific evidence policy. +The Just recipe launches uv and Python directly so JSON arguments never pass +through the native Windows-to-MSYS shell boundary. Omitting `--identity` selects +fresh evidence; supplying an empty identity is an error. + +The publishing workflow reuses these completed checks and passes the run/attempt +and artifact ID/digest through `steps.evidence.outputs.identity` and +`needs.verify.outputs.ci-evidence` to `CI_EVIDENCE`. After deployment approval, +it requires a nonempty value and revalidates that exact identity with +`just release-evidence --identity "$CI_EVIDENCE"` before token acquisition. +It also retains the independent release metadata, exact-commit SARIF, durable Criterion asset, immutable GitHub release and registry-absence gates. Release benchmarks remain a separate scientific measurement workflow; CI evidence does not replace their full sampling or durable archive. diff --git a/justfile b/justfile index 02bbcc0..11d84c4 100644 --- a/justfile +++ b/justfile @@ -401,9 +401,14 @@ python-sync: _ensure-uv python-typecheck: python-sync uv run --locked research-repo-tools python typecheck -# Verify signed default/exact packaging and native CI for HEAD; optionally pin the evidence identity. -release-evidence identity="": - uv run --locked --no-sync python -m tooling.release_evidence verify --identity {{ quote(identity) }} +# Verify signed Cargo/native CI evidence; accept --identity JSON and --github-output PATH. +[positional-arguments] +[script("uv", "run", "--locked", "--no-sync", "python")] +release-evidence *args: + import runpy + import sys + sys.argv[1:1] = ["verify"] + runpy.run_path("tooling/release_evidence.py", run_name="__main__") # Require the release event, reviewed commit, native checks, assets, and notes. release-gate tag: diff --git a/tests/tooling/test_release_evidence.py b/tests/tooling/test_release_evidence.py index 0201906..2b2f34b 100644 --- a/tests/tooling/test_release_evidence.py +++ b/tests/tooling/test_release_evidence.py @@ -24,6 +24,7 @@ RUNS_ENDPOINT = f"actions/workflows/9/runs?event=push&branch=main&head_sha={COMMIT}&per_page=100&page=1" INVOCATION = f"https://github.com/{REPOSITORY}/actions/runs/101/attempts/2" PIN = {"run_id": 101, "run_attempt": 2, "artifact_id": 301, "artifact_digest": DIGEST} +PIN_JSON = '{"artifact_digest":"sha256:' + "b" * 64 + '","artifact_id":301,"run_attempt":2,"run_id":101}' NATIVE_GATES = ( ("build (ubuntu-latest)", "Run CI checks", 201), ("build (macos-latest)", "Run CI checks", 202), @@ -440,7 +441,7 @@ def test_verified_attestation_must_name_this_push_invocation(provider: EvidenceP @pytest.mark.parametrize("source", ["pin", "manifest", "attestation"]) def test_duplicate_keys_cannot_replace_pinned_signed_or_verified_claims(provider: EvidenceProvider, source: str) -> None: - identity = "" + identity = None if source == "pin": identity = json.dumps(PIN).replace('"run_id": 101', '"run_id": 101, "run_id": 101', 1) elif source == "manifest": @@ -505,5 +506,87 @@ def test_cli_success_emits_only_the_pinned_json_identity( monkeypatch.setattr(sys, "argv", ["release_evidence", "verify"]) assert provider.adapter.main() == 0 captured = capsys.readouterr() - assert json.loads(captured.out) == PIN + assert captured.out == PIN_JSON + "\n" assert captured.err == "" + + +@pytest.mark.parametrize("pinned", [False, True]) +def test_cli_exports_verified_identity_with_installed_helper_and_preserves_records( + provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], pinned: bool +) -> None: + output = provider.root / "github output" + original = b"prior=value\r\n" + output.write_bytes(original) + args = ["release_evidence", "verify", "--github-output", str(output)] + if pinned: + args.extend(["--identity", PIN_JSON]) + monkeypatch.setattr(sys, "argv", args) + assert provider.adapter.main() == 0 + assert output.read_bytes() == original + f"identity={PIN_JSON}\n".encode() + assert provider.gh_calls + assert capsys.readouterr() == ("", "") + + +@pytest.mark.parametrize("failure", ["invalid-pin", "changed-pin", "failed-job", "signature", "late-artifact-change"]) +def test_cli_verification_failure_never_exports_identity( + provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], failure: str +) -> None: + output = provider.root / "github output" + original = b"prior=value\n" + output.write_bytes(original) + args = ["release_evidence", "verify", "--github-output", str(output)] + if failure == "invalid-pin": + args.extend(["--identity", "{}"]) + elif failure == "changed-pin": + args.extend(["--identity", json.dumps({**PIN, "artifact_digest": "sha256:" + "c" * 64})]) + elif failure == "failed-job": + provider.jobs[3]["steps"][0]["conclusion"] = "failure" + elif failure == "signature": + provider.signature[0]["verificationResult"]["signature"]["certificate"]["buildTrigger"] = "workflow_dispatch" + else: + provider.responses[ARTIFACT_ENDPOINT] = {**provider.artifact, "digest": "sha256:" + "c" * 64} + monkeypatch.setattr(sys, "argv", args) + assert provider.adapter.main() == 1 + assert output.read_bytes() == original + captured = capsys.readouterr() + assert captured.out == "" + assert "Release evidence rejected:" in captured.err + + +@pytest.mark.parametrize("value", [float("nan"), float("inf"), -float("inf")]) +def test_cli_rejects_nonfinite_serialization_before_export( + provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], value: float +) -> None: + output = provider.root / "github output" + output.write_bytes(b"prior=value\n") + # Guard the transport even if a future verifier returns a non-finite value. + monkeypatch.setattr(provider.adapter, "verify", lambda _identity: {**PIN, "run_id": value}) + monkeypatch.setattr(sys, "argv", ["release_evidence", "verify", "--github-output", str(output)]) + assert provider.adapter.main() == 1 + assert output.read_bytes() == b"prior=value\n" + captured = capsys.readouterr() + assert captured.out == "" + assert "Out of range float values" in captured.err + + +def test_cli_export_permission_failure_is_not_reported_as_success( + provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + output = provider.root / "github output" + output.write_bytes(b"prior=value\n") + original_open = Path.open + + def open_file(path: Path, mode: str = "r", *args: Any, **kwargs: Any) -> Any: + if path == output and mode == "ab": + raise PermissionError(13, "Permission denied", str(path)) + return original_open(path, mode, *args, **kwargs) + + # Model the OS failure portably; execute the actual installed exporter. + monkeypatch.setattr(Path, "open", open_file) + monkeypatch.setattr(sys, "argv", ["release_evidence", "verify", "--github-output", str(output)]) + assert provider.adapter.main() == 1 + assert output.read_bytes() == b"prior=value\n" + captured = capsys.readouterr() + assert captured.out == "" + assert "Permission denied" in captured.err + assert repr(str(output)) in captured.err diff --git a/tests/tooling/test_release_publishing.py b/tests/tooling/test_release_publishing.py index daa2447..7d657c2 100644 --- a/tests/tooling/test_release_publishing.py +++ b/tests/tooling/test_release_publishing.py @@ -1,11 +1,13 @@ """Exercise la-stack's published release contract without credentials or uploads.""" +import io import json import os import shlex import subprocess import sys import tomllib +from contextlib import redirect_stderr from pathlib import Path from typing import Any @@ -16,6 +18,9 @@ from research_repo_tools.just_inspect import dry_run from research_repo_tools.process import run_command +from tests.tooling.test_release_evidence import PIN_JSON, EvidenceProvider +from tooling import release_evidence + REPO_ROOT = Path(__file__).resolve().parents[2] WORKFLOW = REPO_ROOT / ".github/workflows/publish-crates.yml" SHA = "a" * 40 @@ -194,7 +199,7 @@ def test_workflow_requires_approval_revalidation_and_absence_before_oidc() -> No gate = steps[2]["run"] assert 'just release-gate "$RELEASE_TAG"' in gate assert 'test -n "$CI_EVIDENCE"' in gate - assert 'just release-evidence "$CI_EVIDENCE"' in gate + assert 'just release-evidence --identity "$CI_EVIDENCE"' in gate assert steps[2]["env"]["CI_EVIDENCE"] == "${{ needs.verify.outputs.ci-evidence }}" assert 'release registry "$RELEASE_TAG" --expect absent' in gate assert steps[3]["id"] == "auth" @@ -210,45 +215,217 @@ def test_workflow_requires_approval_revalidation_and_absence_before_oidc() -> No assert "--no-verify" not in commands -@pytest.mark.parametrize(("outcome", "expected_exit"), [("valid", 0), ("empty", 1), ("failed", 17)]) -def test_evidence_capture_exports_json_through_real_just_or_fails_without_output(tmp_path: Path, outcome: str, expected_exit: int) -> None: +def run_evidence_fixture() -> int: + """Run the real verifier in a child process with simulated provider responses.""" + root = Path(os.environ["FIXTURE_ROOT"]) + provider = EvidenceProvider(release_evidence, root) + if os.environ.get("FIXTURE_FAILURE") == "failed-job": + provider.jobs[3]["steps"][0]["conclusion"] = "failure" + newline = os.environ.get("FIXTURE_DIAGNOSTIC_NEWLINE", os.linesep) + sys.stdout.buffer.write(("fixture stdout diagnostic" + newline).encode("utf-8")) + sys.stdout.flush() + print("fixture stderr diagnostic", file=sys.stderr) + diagnostic = io.StringIO() + with pytest.MonkeyPatch.context() as patch: + patch.setattr(release_evidence, "ROOT", root) + patch.setattr(release_evidence, "api", provider.api) + patch.setattr(release_evidence, "gh", provider.gh) + patch.setattr(release_evidence, "download_artifact", provider.download) + patch.setattr(release_evidence, "run_git_command", provider.git) + with redirect_stderr(diagnostic): + try: + status = release_evidence.main() + except SystemExit as error: + status = error.code + assert isinstance(status, int) + print(diagnostic.getvalue(), end="", file=sys.stderr) + # Just's quiet mode hides stderr. Record the actual verifier outcome so an + # import/shell failure cannot masquerade as the rejection under test. + result = {"status": status, "diagnostic": diagnostic.getvalue(), "import_root": sys.path[0], "arguments": sys.argv[1:]} + (root / "verifier-result.json").write_text(json.dumps(result), encoding="utf-8") + return status + + +@pytest.fixture +def evidence_runner(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Keep real workflow shell, Just, uv, Python, verifier and output writer.""" + (tmp_path / "justfile").write_text(f"import '{(REPO_ROOT / 'justfile').as_posix()}'\n", encoding="utf-8") + (tmp_path / "Cargo.toml").write_text('[package]\nname = "la-stack"\nversion = "0.5.42"\n', encoding="utf-8") + for name in ("BASH_ENV", "SHELLOPTS", "BASHOPTS", "JUST_QUIET", "JUST_DRY_RUN", "FIXTURE_FAILURE", "FIXTURE_DIAGNOSTIC_NEWLINE", "PYTHONPATH"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("FIXTURE_ROOT", str(tmp_path)) + monkeypatch.setenv("FIXTURE_REPO_ROOT", str(REPO_ROOT)) + monkeypatch.setenv("UV_PROJECT", str(REPO_ROOT)) + monkeypatch.setenv("UV_PROJECT_ENVIRONMENT", sys.prefix) + # Just executes the entry point relative to the fixture checkout. Supply + # provider responses there while retaining the actual native uv/Python launch. + bootstrap = tmp_path / "tooling/release_evidence.py" + bootstrap.parent.mkdir() + bootstrap.write_text( + 'import os, sys\nsys.path.insert(0, os.environ["FIXTURE_REPO_ROOT"])\n' + "from tests.tooling.test_release_publishing import run_evidence_fixture\nsys.exit(run_evidence_fixture())\n", + encoding="utf-8", + newline="\n", + ) + + +def run_evidence_shell(script: str, root: Path) -> subprocess.CompletedProcess[str]: + """Execute workflow source from a file, as Actions does, with explicit env.""" + source = root / "workflow.sh" + source.write_text(script, encoding="utf-8", newline="\n") + # Windows putenv removes empty values from the inherited native environment. + # Build the child environment from the mapping to keep empty distinct from unset. + return run_command("bash", ["--noprofile", "--norc", "-eu", source.as_posix()], cwd=root, env=dict(os.environ), check=False, timeout=60) + + +@pytest.mark.usefixtures("evidence_runner") +@pytest.mark.parametrize("quiet", [False, True], ids=["normal", "quiet"]) +@pytest.mark.parametrize("outcome", ["valid", "failed-job", "missing-file", "directory", "unset-output", "empty-output"]) +def test_evidence_export_uses_real_just_and_installed_writer_without_stdout_capture( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, quiet: bool, outcome: str +) -> None: workflow = yaml.safe_load(WORKFLOW.read_bytes()) script = workflow["jobs"]["verify"]["steps"][-1]["run"] - root = tmp_path / "consumer with spaces" - root.mkdir() - (root / "justfile").write_text(f"import '{(REPO_ROOT / 'justfile').as_posix()}'\n", encoding="utf-8") - output = root / "github output" - identity = {"run_id": 101, "run_attempt": 2, "artifact_id": 301, "artifact_digest": "sha256:" + "a" * 64} - # Substitute only the evidence producer. The actual workflow shell, Just, - # and imported release-evidence recipe own stdout capture and propagation. - producer = ( - "uv() {\n" - ' [[ "$*" == "run --locked --no-sync python -m tooling.release_evidence verify --identity " ]] || return 90\n' - " printf '%s\\n' 'fixture evidence diagnostic' >&2\n" - " printf '%s\\n' \"$FIXTURE_EVIDENCE\"\n" - ' return "$FIXTURE_EXIT"\n' - "}\n" - "export -f uv\n" - ) - result = run_command( - "bash", - ["--noprofile", "--norc", "-euc", producer + script], - cwd=root, - env={ - **{key: value for key, value in os.environ.items() if key not in {"BASH_ENV", "SHELLOPTS", "BASHOPTS", "JUST_QUIET", "JUST_DRY_RUN"}}, - "GITHUB_OUTPUT": output.as_posix(), - "FIXTURE_EVIDENCE": "" if outcome == "empty" else json.dumps(identity, sort_keys=True), - "FIXTURE_EXIT": "17" if outcome == "failed" else "0", - }, - check=False, - ) - assert result.returncode == expected_exit, result.stderr - assert result.stdout == "" - if outcome == "valid": - assert output.read_text(encoding="utf-8").splitlines() == ["identity=" + json.dumps(identity, sort_keys=True)] - assert "fixture evidence diagnostic" in result.stderr + output = tmp_path / "github output" + original = b"prior=value\r\n" + if outcome == "directory": + output.mkdir() + elif outcome != "missing-file": + output.write_bytes(original) + monkeypatch.setenv("GITHUB_OUTPUT", "" if outcome == "empty-output" else output.as_posix()) + if outcome == "unset-output": + monkeypatch.delenv("GITHUB_OUTPUT") + monkeypatch.setenv("JUST_QUIET", str(quiet).lower()) + monkeypatch.setenv("FIXTURE_FAILURE", outcome) + marker = tmp_path / "verification-passed" + result = run_evidence_shell(script + f"\ntouch {shlex.quote(marker.as_posix())}", tmp_path) + assert result.returncode == (0 if outcome == "valid" else 1), result.stderr + assert marker.exists() == (outcome == "valid") + report = tmp_path / "verifier-result.json" + if outcome == "unset-output": + assert not report.exists() + assert "GITHUB_OUTPUT: unbound variable" in result.stderr else: + assert report.exists(), result.stderr + observed = json.loads(report.read_text(encoding="utf-8")) + assert observed["arguments"][:2] == ["verify", "--github-output"] + assert len(observed["arguments"]) == 3 + if outcome == "empty-output": + assert observed["arguments"][2] == "" + else: + assert Path(observed["arguments"][2]) == output + assert observed["status"] == (0 if outcome == "valid" else 1) + rejection = ( + "did not validate required step Verify default and exact Cargo packages" + if outcome == "failed-job" + else "GitHub Actions output destination must be an existing regular file" + ) + assert observed["diagnostic"] == "" if outcome == "valid" else rejection in observed["diagnostic"] + if outcome == "valid": + assert output.read_bytes() == original + f"identity={PIN_JSON}\n".encode() + if not quiet: + # Diagnostics are text; the command-file bytes above must stay LF. + assert result.stdout.splitlines() == ["fixture stdout diagnostic"] + assert "fixture stderr diagnostic" in result.stderr + elif outcome == "directory": + assert output.is_dir() + assert list(output.iterdir()) == [] + elif outcome == "missing-file": assert not output.exists() + else: + assert output.read_bytes() == original + + +@pytest.mark.usefixtures("evidence_runner") +@pytest.mark.parametrize( + ("import_root", "newline"), [(r"C:\agent\test dir\native", "\n"), (r"D:\build's root\new\repo", "\r\n")], ids=["backslashes-lf", "apostrophe-crlf"] +) +def test_evidence_bootstrap_keeps_windows_import_paths_as_environment_data( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, import_root: str, newline: str +) -> None: + # Exercise literal Windows path data on every host. PYTHONPATH supplies the + # real module after the intentionally nonexistent path inserted by bootstrap. + monkeypatch.setenv("FIXTURE_REPO_ROOT", import_root) + monkeypatch.setenv("PYTHONPATH", str(REPO_ROOT)) + monkeypatch.setenv("FIXTURE_DIAGNOSTIC_NEWLINE", newline) + output = tmp_path / "github output" + output.touch() + monkeypatch.setenv("GITHUB_OUTPUT", output.as_posix()) + workflow = yaml.safe_load(WORKFLOW.read_bytes()) + script = workflow["jobs"]["verify"]["steps"][-1]["run"] + result = run_evidence_shell(script, tmp_path) + assert result.returncode == 0, result.stderr + assert result.stdout == "fixture stdout diagnostic" + newline + observed = json.loads((tmp_path / "verifier-result.json").read_text(encoding="utf-8")) + assert observed["status"] == 0 + assert observed["diagnostic"] == "" + assert observed["import_root"] == import_root + assert observed["arguments"][:2] == ["verify", "--github-output"] + assert len(observed["arguments"]) == 3 + assert Path(observed["arguments"][2]) == output + assert output.read_bytes() == f"identity={PIN_JSON}\n".encode() + + +@pytest.mark.usefixtures("evidence_runner") +@pytest.mark.parametrize("quiet", [False, True], ids=["normal", "quiet"]) +@pytest.mark.parametrize("pin", ["valid", "empty", "invalid", "changed"]) +def test_exported_identity_is_revalidated_by_real_just_before_registry_and_token( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, quiet: bool, pin: str +) -> None: + workflow = yaml.safe_load(WORKFLOW.read_bytes()) + output = tmp_path / "github output" + output.touch() + monkeypatch.setenv("GITHUB_OUTPUT", output.as_posix()) + monkeypatch.setenv("JUST_QUIET", str(quiet).lower()) + exported = run_evidence_shell(workflow["jobs"]["verify"]["steps"][-1]["run"], tmp_path) + assert exported.returncode == 0, exported.stderr + report = tmp_path / "verifier-result.json" + assert json.loads(report.read_text(encoding="utf-8"))["status"] == 0 + report.unlink() + record = output.read_text(encoding="utf-8") + assert record == f"identity={PIN_JSON}\n" + identity = record.removeprefix("identity=").removesuffix("\n") + variants = {"valid": identity, "empty": "", "invalid": "{", "changed": identity.replace('"run_attempt":2', '"run_attempt":1')} + monkeypatch.setenv("CI_EVIDENCE", variants[pin]) + monkeypatch.setenv("RELEASE_TAG", TAG) + registry = tmp_path / "registry-reached" + token = tmp_path / "token-reached" + # The other release and registry gates are independently covered below. + # Keep evidence verification real and stop before any credentials or upload. + runner = f'uv() {{\n [[ "$4 $5 $6" == "research-repo-tools release registry" ]] || return 90\n touch {shlex.quote(registry.as_posix())}\n}}\n' + runner += 'just() { if [[ "$1" == release-gate ]]; then return 0; fi; command just "$@"; }\n' + script = workflow["jobs"]["publish"]["steps"][2]["run"] + result = run_evidence_shell(runner + script + f"\ntouch {shlex.quote(token.as_posix())}", tmp_path) + if pin != "empty": + assert report.exists(), result.stderr + observed = json.loads(report.read_text(encoding="utf-8")) + assert observed["arguments"] == ["verify", "--identity", variants[pin]] + assert result.returncode == (0 if pin == "valid" else 1), result.stderr + assert registry.exists() == (pin == "valid") + assert token.exists() == (pin == "valid") + if pin == "empty": + assert not report.exists() + else: + observed = json.loads(report.read_text(encoding="utf-8")) + assert observed["status"] == (0 if pin == "valid" else 1) + rejection = "Expecting property name" if pin == "invalid" else "selected CI attempt is stale" + assert observed["diagnostic"] == "" if pin == "valid" else rejection in observed["diagnostic"] + + +@pytest.mark.usefixtures("evidence_runner") +@pytest.mark.parametrize("quiet", [False, True], ids=["normal", "quiet"]) +def test_explicit_empty_identity_cannot_select_fresh_evidence(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, quiet: bool) -> None: + monkeypatch.setenv("JUST_QUIET", str(quiet).lower()) + output = tmp_path / "github output" + original = b"prior=value\n" + output.write_bytes(original) + monkeypatch.setenv("GITHUB_OUTPUT", output.as_posix()) + result = run_evidence_shell('just release-evidence --identity "" --github-output "$GITHUB_OUTPUT"\n', tmp_path) + observed = json.loads((tmp_path / "verifier-result.json").read_text(encoding="utf-8")) + assert observed["arguments"][:3] == ["verify", "--identity", ""] + assert result.returncode == observed["status"] == 1 + assert "Expecting value" in observed["diagnostic"] + assert output.read_bytes() == original @pytest.mark.parametrize(("failure", "expected_exit"), [("none", 0), ("evidence", 1), ("ci-evidence", 1), ("empty-pin", 1), ("present", 1), ("unavailable", 2)]) diff --git a/tooling/README.md b/tooling/README.md index 956b252..2a9463f 100644 --- a/tooling/README.md +++ b/tooling/README.md @@ -22,6 +22,25 @@ runs the installed-package integration tests. `just ci` is the final repository validation. Native Linux, macOS and Windows CI is required for adoption; local macOS results do not establish the other platforms. +For subprocess fixtures, run Bash and Python source from files, and pass paths +and payloads as arguments or environment data. Nested `-c` commands add Windows +argument parsing boundaries that POSIX shell quoting does not protect. Supply an +explicit child environment when testing empty values: Windows `putenv` removes +them from the inherited native environment. Exercise Windows path data and +quoting characters as well as native Windows execution. Negative tests must +prove the received arguments, intended rejection and unchanged output; a nonzero +exit alone also accepts a broken import or shell. When quiet mode hides +diagnostics, retain the verifier's arguments, exit status and diagnostic in a +fixture-owned result file. + +The `release-evidence` recipe uses Just's native uv/Python script interpreter. +MSYS applies shell-like argument expansion when started by a native Windows +program, which can corrupt JSON passed as Bash positional arguments; see +[MSYS2 #522](https://github.com/msys2/MSYS2-packages/issues/522). The integration +fixture keeps the real Just, uv and Python launch and substitutes only the +entry point's provider responses. It checks the exact received identity and +rejects an explicitly empty pin rather than selecting fresh evidence. + Semgrep shares the root Python 3.15 environment and lockfile. Preserve its native-wheel requirement and compatibility overrides in `pyproject.toml`. Local scans, fixture checks and SARIF retain the Linux stack-limit workaround @@ -159,6 +178,9 @@ use the shared release contract; `release-package` owns native default/exact Cargo dry-runs. Main CI signs those packaging and native validation identities; `release-evidence` verifies the original main-push attempt and immutable signed artifact before publication and rechecks its pinned identity after approval. +The adapter uses the installed `ci.export_environment` helper for explicit +`--github-output` export; the [release guide](../docs/RELEASING.md#reusable-ci-evidence) +owns the transport and approval contract. Cargo evidence policy stays local. `tests/tooling/test_release_evidence.py` checks the adapter's trust boundaries, while `tests/tooling/test_release_publishing.py` checks consumer policy and upload barriers without registry credentials. The release guide owns retention, complete-rerun diff --git a/tooling/release_evidence.py b/tooling/release_evidence.py index c9c23ba..85fb3c1 100644 --- a/tooling/release_evidence.py +++ b/tooling/release_evidence.py @@ -16,6 +16,7 @@ from pathlib import Path from typing import Any +from research_repo_tools.ci import export_environment from research_repo_tools.process import format_exception_diagnostics, run_git_command from research_repo_tools.release_validation import ValidationPolicy, api, download_artifact, gh, select_main_run, validate_jobs, validate_run @@ -196,10 +197,10 @@ def verify_signature(directory: Path, run: dict[str, Any]) -> None: raise ValueError(msg) -def verify(identity: str = "") -> dict[str, Any]: +def verify(identity: str | None = None) -> dict[str, Any]: """Reuse one exact CI attempt, or recheck the same pinned identity after approval.""" commit = checkout_commit() - pin = object_value(json.loads(identity, object_pairs_hook=unique_object)) if identity else None + pin = object_value(json.loads(identity, object_pairs_hook=unique_object)) if identity is not None else None if pin is not None and set(pin) != {"run_id", "run_attempt", "artifact_id", "artifact_digest"}: msg = "invalid pinned evidence identity" raise ValueError(msg) @@ -234,13 +235,22 @@ def main() -> int: parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest="command", required=True) commands.add_parser("seal").add_argument("--output", type=Path, required=True) - commands.add_parser("verify").add_argument("--identity", default="") + verification = commands.add_parser("verify") + verification.add_argument("--identity") + verification.add_argument("--github-output", type=Path, help="append identity to an existing GitHub Actions output file instead of stdout") args = parser.parse_args() try: if args.command == "seal": seal(args.output) else: - print(json.dumps(verify(args.identity), sort_keys=True)) + identity = json.dumps(verify(args.identity), sort_keys=True, separators=(",", ":"), allow_nan=False) + if args.github_output is not None: + if not args.github_output.is_file(): + msg = "GitHub Actions output destination must be an existing regular file" + raise ValueError(msg) + export_environment(args.github_output, ("identity",), environment={"identity": identity}) + else: + print(identity) except (KeyError, OSError, TypeError, ValueError, subprocess.SubprocessError) as error: print(f"Release evidence rejected: {format_exception_diagnostics(error, single_line=True)}", file=sys.stderr) return 1