Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 2 additions & 4 deletions .github/workflows/publish-crates.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,7 @@ jobs:
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
identity=$(just release-evidence)
test -n "$identity"
printf 'identity=%s\n' "$identity" >> "$GITHUB_OUTPUT"
just release-evidence --github-output "$GITHUB_OUTPUT"

publish:
needs: verify
Expand Down Expand Up @@ -89,7 +87,7 @@ jobs:
set -euo pipefail
just release-gate "$RELEASE_TAG"
test -n "$CI_EVIDENCE"
just release-evidence "$CI_EVIDENCE"
just release-evidence --identity "$CI_EVIDENCE"
uv run --locked --no-sync research-repo-tools release registry "$RELEASE_TAG" --expect absent

- name: Obtain a short-lived crates.io token
Expand Down
21 changes: 18 additions & 3 deletions docs/RELEASING.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,24 @@ package identities. Pending matching runs receive a bounded wait; missing,
failed, ambiguous or expired evidence stops publication. PR, manual dispatch,
fork, local and partial-rerun results cannot replace it.

The publishing workflow reuses these completed checks, captures the run/attempt
and artifact ID/digest, and revalidates that same identity after deployment
approval. It also retains the independent release metadata, exact-commit SARIF,
Local verification prints the validated identity as deterministic single-line
JSON. In Actions, `just release-evidence --github-output "$GITHUB_OUTPUT"`
uses the installed `research_repo_tools.ci.export_environment` helper to append
the `identity` output only after verification succeeds. The destination must
already exist; export errors fail the command. Existing output records survive,
and normal or quiet Just execution transports the same JSON without capturing
stdout. Non-finite JSON values are rejected. The shared helper owns command-file
transport; this adapter retains the Cargo-specific evidence policy.
The Just recipe launches uv and Python directly so JSON arguments never pass
through the native Windows-to-MSYS shell boundary. Omitting `--identity` selects
fresh evidence; supplying an empty identity is an error.

The publishing workflow reuses these completed checks and passes the run/attempt
and artifact ID/digest through `steps.evidence.outputs.identity` and
`needs.verify.outputs.ci-evidence` to `CI_EVIDENCE`. After deployment approval,
it requires a nonempty value and revalidates that exact identity with
`just release-evidence --identity "$CI_EVIDENCE"` before token acquisition.
It also retains the independent release metadata, exact-commit SARIF,
durable Criterion asset, immutable GitHub release and registry-absence gates.
Release benchmarks remain a separate scientific measurement workflow; CI
evidence does not replace their full sampling or durable archive.
Expand Down
11 changes: 8 additions & 3 deletions justfile
Original file line number Diff line number Diff line change
Expand Up @@ -401,9 +401,14 @@ python-sync: _ensure-uv
python-typecheck: python-sync
uv run --locked research-repo-tools python typecheck

# Verify signed default/exact packaging and native CI for HEAD; optionally pin the evidence identity.
release-evidence identity="":
uv run --locked --no-sync python -m tooling.release_evidence verify --identity {{ quote(identity) }}
# Verify signed Cargo/native CI evidence; accept --identity JSON and --github-output PATH.
[positional-arguments]
[script("uv", "run", "--locked", "--no-sync", "python")]
release-evidence *args:
import runpy
import sys
sys.argv[1:1] = ["verify"]
runpy.run_path("tooling/release_evidence.py", run_name="__main__")

# Require the release event, reviewed commit, native checks, assets, and notes.
release-gate tag:
Expand Down
87 changes: 85 additions & 2 deletions tests/tooling/test_release_evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
RUNS_ENDPOINT = f"actions/workflows/9/runs?event=push&branch=main&head_sha={COMMIT}&per_page=100&page=1"
INVOCATION = f"https://github.com/{REPOSITORY}/actions/runs/101/attempts/2"
PIN = {"run_id": 101, "run_attempt": 2, "artifact_id": 301, "artifact_digest": DIGEST}
PIN_JSON = '{"artifact_digest":"sha256:' + "b" * 64 + '","artifact_id":301,"run_attempt":2,"run_id":101}'
NATIVE_GATES = (
("build (ubuntu-latest)", "Run CI checks", 201),
("build (macos-latest)", "Run CI checks", 202),
Expand Down Expand Up @@ -440,7 +441,7 @@ def test_verified_attestation_must_name_this_push_invocation(provider: EvidenceP

@pytest.mark.parametrize("source", ["pin", "manifest", "attestation"])
def test_duplicate_keys_cannot_replace_pinned_signed_or_verified_claims(provider: EvidenceProvider, source: str) -> None:
identity = ""
identity = None
if source == "pin":
identity = json.dumps(PIN).replace('"run_id": 101', '"run_id": 101, "run_id": 101', 1)
elif source == "manifest":
Expand Down Expand Up @@ -505,5 +506,87 @@ def test_cli_success_emits_only_the_pinned_json_identity(
monkeypatch.setattr(sys, "argv", ["release_evidence", "verify"])
assert provider.adapter.main() == 0
captured = capsys.readouterr()
assert json.loads(captured.out) == PIN
assert captured.out == PIN_JSON + "\n"
assert captured.err == ""


@pytest.mark.parametrize("pinned", [False, True])
def test_cli_exports_verified_identity_with_installed_helper_and_preserves_records(
provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], pinned: bool
) -> None:
output = provider.root / "github output"
original = b"prior=value\r\n"
output.write_bytes(original)
args = ["release_evidence", "verify", "--github-output", str(output)]
if pinned:
args.extend(["--identity", PIN_JSON])
monkeypatch.setattr(sys, "argv", args)
assert provider.adapter.main() == 0
assert output.read_bytes() == original + f"identity={PIN_JSON}\n".encode()
assert provider.gh_calls
assert capsys.readouterr() == ("", "")


@pytest.mark.parametrize("failure", ["invalid-pin", "changed-pin", "failed-job", "signature", "late-artifact-change"])
def test_cli_verification_failure_never_exports_identity(
provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], failure: str
) -> None:
output = provider.root / "github output"
original = b"prior=value\n"
output.write_bytes(original)
args = ["release_evidence", "verify", "--github-output", str(output)]
if failure == "invalid-pin":
args.extend(["--identity", "{}"])
elif failure == "changed-pin":
args.extend(["--identity", json.dumps({**PIN, "artifact_digest": "sha256:" + "c" * 64})])
elif failure == "failed-job":
provider.jobs[3]["steps"][0]["conclusion"] = "failure"
elif failure == "signature":
provider.signature[0]["verificationResult"]["signature"]["certificate"]["buildTrigger"] = "workflow_dispatch"
else:
provider.responses[ARTIFACT_ENDPOINT] = {**provider.artifact, "digest": "sha256:" + "c" * 64}
monkeypatch.setattr(sys, "argv", args)
assert provider.adapter.main() == 1
assert output.read_bytes() == original
captured = capsys.readouterr()
assert captured.out == ""
assert "Release evidence rejected:" in captured.err


@pytest.mark.parametrize("value", [float("nan"), float("inf"), -float("inf")])
def test_cli_rejects_nonfinite_serialization_before_export(
provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], value: float
) -> None:
output = provider.root / "github output"
output.write_bytes(b"prior=value\n")
# Guard the transport even if a future verifier returns a non-finite value.
monkeypatch.setattr(provider.adapter, "verify", lambda _identity: {**PIN, "run_id": value})
monkeypatch.setattr(sys, "argv", ["release_evidence", "verify", "--github-output", str(output)])
assert provider.adapter.main() == 1
assert output.read_bytes() == b"prior=value\n"
captured = capsys.readouterr()
assert captured.out == ""
assert "Out of range float values" in captured.err


def test_cli_export_permission_failure_is_not_reported_as_success(
provider: EvidenceProvider, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
) -> None:
output = provider.root / "github output"
output.write_bytes(b"prior=value\n")
original_open = Path.open

def open_file(path: Path, mode: str = "r", *args: Any, **kwargs: Any) -> Any:
if path == output and mode == "ab":
raise PermissionError(13, "Permission denied", str(path))
return original_open(path, mode, *args, **kwargs)

# Model the OS failure portably; execute the actual installed exporter.
monkeypatch.setattr(Path, "open", open_file)
monkeypatch.setattr(sys, "argv", ["release_evidence", "verify", "--github-output", str(output)])
assert provider.adapter.main() == 1
assert output.read_bytes() == b"prior=value\n"
captured = capsys.readouterr()
assert captured.out == ""
assert "Permission denied" in captured.err
assert repr(str(output)) in captured.err
Loading
Loading