diff --git a/.github/workflows/native_release.yml b/.github/workflows/native_release.yml index ac10725..1ab80ed 100644 --- a/.github/workflows/native_release.yml +++ b/.github/workflows/native_release.yml @@ -9,7 +9,19 @@ on: publish: description: Publish the prepared artifacts as a GitHub release type: boolean - default: true + default: false + prepared_run_id: + description: Replay a successful preparation run without rebuilding (empty to build) + required: false + type: string + expected_source_sha: + description: Full owner source SHA from the prepared run (required for replay) + required: false + type: string + manifest_sha256: + description: Qualified manifest.json SHA256 (required for replay) + required: false + type: string prerelease: description: Mark the GitHub release as a prerelease type: boolean @@ -18,8 +30,13 @@ on: permissions: contents: read +concurrency: + group: native-release-${{ inputs.tag }} + cancel-in-progress: false + jobs: build: + if: ${{ !inputs.prepared_run_id }} name: build (${{ matrix.target }}) runs-on: ${{ matrix.runs_on }} strategy: @@ -70,6 +87,7 @@ jobs: if-no-files-found: error package: + if: ${{ !inputs.prepared_run_id }} needs: build # The SwiftPM XCFramework needs xcodebuild. runs-on: macos-latest @@ -91,15 +109,21 @@ jobs: cp -R "$dir"/. "bin/${dir#artifacts/bin-}/" done - name: Package - run: python3 tools/package_release.py --tag "${{ inputs.tag }}" --apple-xcframework + env: + RELEASE_TAG: ${{ inputs.tag }} + run: python3 tools/package_release.py --tag "$RELEASE_TAG" --apple-xcframework - name: Validate the XCFramework privacy manifests + env: + RELEASE_TAG: ${{ inputs.tag }} run: >- python3 tools/apple_privacy_manifest.py --audit-imports - "dist/stable-diffusion-native-apple-xcframework-${{ inputs.tag }}.zip" + "dist/stable-diffusion-native-apple-xcframework-$RELEASE_TAG.zip" - name: Link the XCFramework into a SwiftPM consumer + env: + RELEASE_TAG: ${{ inputs.tag }} run: >- python3 tools/apple_xcframework.py consumer - "dist/stable-diffusion-native-apple-xcframework-${{ inputs.tag }}.zip" + "dist/stable-diffusion-native-apple-xcframework-$RELEASE_TAG.zip" - name: Upload the prepared release artifacts uses: actions/upload-artifact@v4 with: @@ -108,24 +132,81 @@ jobs: if-no-files-found: error retention-days: 14 - release: + verify: needs: package - if: ${{ inputs.publish }} + if: ${{ always() && !cancelled() && (needs.package.result == 'success' || inputs.prepared_run_id != '') }} runs-on: ubuntu-latest permissions: - contents: write + contents: read + actions: read + outputs: + source: ${{ steps.verify.outputs.source }} + digest: ${{ steps.verify.outputs.digest }} steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false - uses: actions/download-artifact@v4 + if: ${{ !inputs.prepared_run_id }} with: name: stable-diffusion-native-${{ inputs.tag }} path: dist - - uses: softprops/action-gh-release@v2 + - name: Verify release inputs and prepared bytes + id: verify + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + PREPARED_RUN_ID: ${{ inputs.prepared_run_id }} + EXPECTED_SOURCE_SHA: ${{ inputs.expected_source_sha }} + MANIFEST_SHA256: ${{ inputs.manifest_sha256 }} + run: | + if [ -n "$PREPARED_RUN_ID" ]; then + python3 tools/prepared_release.py replay --run-id "$PREPARED_RUN_ID" \ + --tag "$RELEASE_TAG" --source "$EXPECTED_SOURCE_SHA" \ + --manifest-sha256 "$MANIFEST_SHA256" + source="$EXPECTED_SOURCE_SHA" + digest="$MANIFEST_SHA256" + else + source="$GITHUB_SHA" + digest="$(sha256sum dist/manifest.json | cut -d ' ' -f 1)" + upstream="$(git ls-tree HEAD third_party/stable-diffusion.cpp | awk '{print $3}')" + python3 tools/prepared_release.py verify --tag "$RELEASE_TAG" \ + --source "$source" --manifest-sha256 "$digest" --upstream "$upstream" + fi + echo "source=$source" >> "$GITHUB_OUTPUT" + echo "digest=$digest" >> "$GITHUB_OUTPUT" + echo "Manifest SHA256: $digest" >> "$GITHUB_STEP_SUMMARY" + echo "Prepared source: $source" >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 + with: + name: verified-release-${{ github.run_id }} + path: dist/ + if-no-files-found: error + retention-days: 14 + + release: + needs: verify + if: ${{ always() && !cancelled() && needs.verify.result == 'success' && inputs.publish && github.ref == 'refs/heads/main' && github.repository == 'leehack/stable-diffusion-native' }} + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/download-artifact@v4 with: - tag_name: ${{ inputs.tag }} - target_commitish: ${{ github.sha }} - prerelease: ${{ inputs.prerelease }} - files: | - dist/*.tar.gz - dist/*.zip - dist/manifest.json - dist/SHA256SUMS + name: verified-release-${{ github.run_id }} + path: dist + - name: Verify again and create the release without overwriting + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + EXPECTED_SOURCE_SHA: ${{ needs.verify.outputs.source }} + MANIFEST_SHA256: ${{ needs.verify.outputs.digest }} + PRERELEASE: ${{ inputs.prerelease }} + run: | + flags=() + if [ "$PRERELEASE" = true ]; then flags+=(--prerelease); fi + python3 tools/prepared_release.py publish --tag "$RELEASE_TAG" \ + --source "$EXPECTED_SOURCE_SHA" --manifest-sha256 "$MANIFEST_SHA256" "${flags[@]}" diff --git a/README.md b/README.md index e22537d..d39c5bc 100644 --- a/README.md +++ b/README.md @@ -658,7 +658,37 @@ XCFramework without creating a tag or GitHub release. The workflow uploads `stable-diffusion-native-` as a run artifact retained for 14 days. Build and packaging jobs have read-only repository permissions; only the separate publication job has write permission, and it is skipped in preparation mode. -Normal release runs keep `publish=true` by default. +Preparation is the default (`publish=false`). To publish a previously qualified +artifact, dispatch the same workflow **on `main`**, setting `prepared_run_id`, +`expected_source_sha` (the full owner commit), `manifest_sha256` (the SHA256 of +its exact `manifest.json`), the same `tag`, and explicitly `publish=true`. +Replay skips every build and packaging job. It requires a successful +`workflow_dispatch` of this repository's release workflow on `main`, verifies +its source against current main history and the upstream gitlink, and checks +the unexpired named artifact, complete target inventory and every archive's +size/checksum. The publisher verifies the bytes again after the job transfer. +The prepared run's source, rather than the later publisher commit, becomes +the release tag. Future manifests also record that owner source in `source`; +older manifests bind it through the trusted run and upstream gitlink. + +A replay with `publish=false` performs the same read-only verification. For +example, after qualification (replace the digest with its recorded value): + +```bash +gh workflow run native_release.yml -R leehack/stable-diffusion-native --ref main \ + -f tag=v0.3.0-2 -f prepared_run_id=41540643141 \ + -f expected_source_sha=1c1f03bbb4578c3fbf80a96ad90718ba78a09407 \ + -f manifest_sha256="$QUALIFIED_MANIFEST_SHA256" -f publish=false -f prerelease=true +``` + +The artifact retention window is 14 days; an expired or missing artifact fails +closed and needs a new preparation and qualification. Existing tags or releases +also fail closed. Same-tag runs are serialized, and publication atomically +creates a new tag before creating the release and uploading assets, with no +overwrite option. A failure after tag creation may leave a partial tag/release; +inspect it before an explicit recovery. A rerun never silently replaces it. +An explicit `publish=true` without a prepared run retains the original +build-and-publish mode. Archive names: diff --git a/tests/test_apple_privacy_manifest.py b/tests/test_apple_privacy_manifest.py index 6a48602..eb0efa4 100644 --- a/tests/test_apple_privacy_manifest.py +++ b/tests/test_apple_privacy_manifest.py @@ -334,7 +334,7 @@ def test_release_audits_the_packaged_xcframework_before_upload(self) -> None: -1 < workflow.find("python3 tools/package_release.py") < workflow.find(self.AUDIT) - < workflow.find("softprops/action-gh-release") + < workflow.find("tools/prepared_release.py publish") ) def test_pull_requests_run_the_mach_o_tests_and_the_audit_on_macos(self) -> None: diff --git a/tests/test_prepared_release.py b/tests/test_prepared_release.py new file mode 100644 index 0000000..c138fe8 --- /dev/null +++ b/tests/test_prepared_release.py @@ -0,0 +1,249 @@ +"""Prepared archives stay bound to the qualified run, source, tag and bytes.""" + +import argparse +import copy +import io +import json +import os +import stat +import subprocess +import sys +import tarfile +import tempfile +import unittest +import zipfile +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "tools")) +import prepared_release as release + +TAG = "v0.3.0-2" +SOURCE = "1" * 40 +UPSTREAM = "2" * 40 + + +class DistributionTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.dist = Path(self.tmp.name) / "dist" + self.dist.mkdir() + self.manifest = {"schemaVersion": 1, "package": release.PACKAGE, "tag": TAG, + "upstream": {"repository": "leejet/stable-diffusion.cpp", + "commit": UPSTREAM, "patches": []}, "artifacts": []} + for target in release.TARGETS: + for kind in ("runtime", "symbols"): + name = f"{release.PACKAGE}-{kind}-{target}-{TAG}.tar.gz" + path = self.dist / name + with tarfile.open(path, "w:gz") as archive: + info = json.dumps({"target": target, "upstream": self.manifest["upstream"]}).encode() + entry = tarfile.TarInfo("build-info.json") + entry.size = len(info) + archive.addfile(entry, io.BytesIO(info)) + self.add_artifact(target, kind, path) + path = self.dist / f"{release.PACKAGE}-apple-xcframework-{TAG}.zip" + with zipfile.ZipFile(path, "w") as archive: + archive.writestr("example", b"xcframework") + self.add_artifact("apple", "xcframework", path) + self.write_manifest() + + def add_artifact(self, target, kind, path): + self.manifest["artifacts"].append({"target": target, "kind": kind, + "file": path.name, "sha256": release.sha256(path), "size": path.stat().st_size}) + + def write_manifest(self): + (self.dist / "manifest.json").write_text(json.dumps(self.manifest)) + (self.dist / "SHA256SUMS").write_text("".join( + f"{a['sha256']} {a['file']}\n" for a in self.manifest["artifacts"])) + self.digest = release.sha256(self.dist / "manifest.json") + + def verify(self): + return release.verify_dist(self.dist, TAG, SOURCE, self.digest, UPSTREAM) + + def test_legacy_and_owner_source_manifests(self): + self.verify() + self.manifest["source"] = {"repository": release.REPOSITORY, "commit": SOURCE} + self.write_manifest() + self.verify() + self.manifest["source"]["commit"] = "3" * 40 + self.write_manifest() + with self.assertRaisesRegex(ValueError, "owner source"): + self.verify() + + def test_mutations_fail_closed(self): + for mutation in ("tag", "upstream", "duplicate", "missing", "extra", "hash", "sums", "digest", "symlink"): + with self.subTest(mutation=mutation): + backup = copy.deepcopy(self.manifest) + if mutation == "tag": self.manifest["tag"] = "v0.3.0-1" + if mutation == "upstream": self.manifest["upstream"]["commit"] = "3" * 40 + if mutation == "duplicate": self.manifest["artifacts"].append(self.manifest["artifacts"][0]) + if mutation == "missing": self.manifest["artifacts"].pop() + if mutation == "hash": self.manifest["artifacts"][0]["sha256"] = "0" * 64 + self.write_manifest() + extra = self.dist / "extra" + if mutation == "extra": extra.write_text("extra") + if mutation == "symlink": extra.symlink_to(self.dist / self.manifest["artifacts"][0]["file"]) + if mutation == "sums": (self.dist / "SHA256SUMS").write_text("") + if mutation == "digest": self.digest = "0" * 64 + with self.assertRaises(ValueError): self.verify() + if extra.exists(): extra.unlink() + self.manifest = backup + self.write_manifest() + + def test_archive_build_source_is_checked_even_with_updated_hashes(self): + artifact = self.manifest["artifacts"][0] + path = self.dist / artifact["file"] + with tarfile.open(path, "w:gz") as archive: + data = json.dumps({"target": artifact["target"], "upstream": {"commit": "3" * 40}}).encode() + info = tarfile.TarInfo("build-info.json") + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + artifact.update(sha256=release.sha256(path), size=path.stat().st_size) + self.write_manifest() + with self.assertRaisesRegex(ValueError, "build source"): + self.verify() + + def test_download_envelope_rejects_unsafe_members(self): + for name, mode in (("../escape", 0), ("nested/file", 0), ("file", stat.S_IFLNK | 0o777), + ("file", stat.S_IFIFO | 0o600), ("file\\escape", 0)): + with self.subTest(name=name, mode=mode): + bundle = Path(self.tmp.name) / "bundle.zip" + with zipfile.ZipFile(bundle, "w") as archive: + info = zipfile.ZipInfo(name) + info.external_attr = mode << 16 + archive.writestr(info, "payload") + with self.assertRaises(ValueError): + release.unpack(bundle, Path(self.tmp.name) / "unpacked") + + def test_publish_verifies_again_and_uses_atomic_ref_creation(self): + args = argparse.Namespace(tag=TAG, source=SOURCE, manifest_sha256=self.digest, + dist=self.dist, prerelease=True) + env = {"GITHUB_REPOSITORY": release.REPOSITORY, "GITHUB_WORKFLOW": "Native Build & Release", + "GITHUB_ACTIONS": "true", "GITHUB_REF": "refs/heads/main"} + calls = [] + def fake_api(path, **kwargs): + calls.append((path, kwargs)) + if path.startswith("contents/"): return {"sha": UPSTREAM} + return None + with patch.dict(os.environ, env), patch.object(release, "api", side_effect=fake_api), \ + patch.object(release.subprocess, "run") as run: + release.publish(args) + self.assertEqual(calls[-1], ("git/refs", {"method": "POST", "data": { + "ref": f"refs/tags/{TAG}", "sha": SOURCE}})) + command = run.call_args.args[0] + self.assertIn("--verify-tag", command) + self.assertNotIn("--clobber", command) + self.assertEqual(len([x for x in command if str(self.dist) in x]), 27) + (self.dist / "SHA256SUMS").write_text("corrupt") + with patch.dict(os.environ, env), patch.object(release, "api", side_effect=fake_api), \ + patch.object(release.subprocess, "run") as run: + with self.assertRaises(ValueError): release.publish(args) + run.assert_not_called() + + def test_replay_preserves_every_prepared_byte(self): + bundle = Path(self.tmp.name) / "prepared.zip" + with zipfile.ZipFile(bundle, "w") as archive: + for path in self.dist.iterdir(): + archive.write(path, path.name) + dest = Path(self.tmp.name) / "replayed" + args = argparse.Namespace(tag=TAG, source=SOURCE, manifest_sha256=self.digest, + run_id="123", dist=dest) + artifact = {"id": 789, "digest": "sha256:" + release.sha256(bundle)} + def download(command, **kwargs): + self.assertEqual(command[-1], f"repos/{release.REPOSITORY}/actions/artifacts/789/zip") + kwargs["stdout"].write(bundle.read_bytes()) + with patch.object(release, "provenance", return_value=(artifact, UPSTREAM)), \ + patch.object(release.subprocess, "run", side_effect=download), \ + patch.object(release, "check_absent"): + release.replay(args) + self.assertEqual({p.name: p.read_bytes() for p in self.dist.iterdir()}, + {p.name: p.read_bytes() for p in dest.iterdir()}) + + def test_partial_publication_reports_the_created_tag(self): + args = argparse.Namespace(tag=TAG, source=SOURCE, manifest_sha256=self.digest, + dist=self.dist, prerelease=True) + env = {"GITHUB_REPOSITORY": release.REPOSITORY, "GITHUB_WORKFLOW": "Native Build & Release", + "GITHUB_ACTIONS": "true", "GITHUB_REF": "refs/heads/main"} + with patch.dict(os.environ, env), patch.object(release, "api", return_value={"sha": UPSTREAM}), \ + patch.object(release, "check_absent"), \ + patch.object(release.subprocess, "run", side_effect=subprocess.CalledProcessError(1, "gh")): + with self.assertRaisesRegex(ValueError, "tag .* was created.*partial tag/release"): + release.publish(args) + + def test_tag_race_never_creates_or_uploads_release(self): + args = argparse.Namespace(tag=TAG, source=SOURCE, manifest_sha256=self.digest, + dist=self.dist, prerelease=True) + env = {"GITHUB_REPOSITORY": release.REPOSITORY, "GITHUB_WORKFLOW": "Native Build & Release", + "GITHUB_ACTIONS": "true", "GITHUB_REF": "refs/heads/main"} + def fake_api(path, **kwargs): + if path.startswith("contents/"): return {"sha": UPSTREAM} + if path == "git/refs": raise ValueError("tag won by other publisher") + return None + with patch.dict(os.environ, env), patch.object(release, "api", side_effect=fake_api), \ + patch.object(release.subprocess, "run") as run: + with self.assertRaises(ValueError): release.publish(args) + run.assert_not_called() + + +class ProvenanceTest(unittest.TestCase): + def setUp(self): + self.run = {"id": 123, "workflow_id": 456, "path": release.WORKFLOW, + "event": "workflow_dispatch", "status": "completed", "conclusion": "success", + "repository": {"full_name": release.REPOSITORY, "id": 1}, + "head_repository": {"full_name": release.REPOSITORY}, + "head_branch": "main", "head_sha": SOURCE} + self.artifact = {"id": 789, "name": f"{release.PACKAGE}-{TAG}", "expired": False, + "expires_at": "2999-01-01T00:00:00Z", "workflow_run": { + "id": 123, "head_sha": SOURCE, "repository_id": 1, "head_repository_id": 1}} + self.artifacts = [self.artifact] + self.status = "ahead" + + def api(self, path, **kwargs): + if path == "actions/runs/123": return self.run + if path == "actions/workflows/native_release.yml": return {"id": 456} + if path.startswith("compare/"): return {"status": self.status} + if path.startswith("contents/"): return {"sha": UPSTREAM, "submodule_git_url": + "https://github.com/leejet/stable-diffusion.cpp.git"} + if "/artifacts?" in path: return {"artifacts": self.artifacts} + raise AssertionError(path) + + def test_exact_successful_owner_run(self): + with patch.object(release, "api", side_effect=self.api): + self.assertEqual(release.provenance("123", SOURCE, TAG), (self.artifact, UPSTREAM)) + + def test_untrusted_runs_fail_closed(self): + for key, value in (("workflow_id", 999), ("path", "other.yml"), ("event", "pull_request"), + ("status", "in_progress"), ("conclusion", "failure"), ("head_sha", "3" * 40), + ("head_branch", "feature"), ("head_repository", {"full_name": "fork/repo"})): + with self.subTest(key=key): + original = self.run[key] + self.run[key] = value + with patch.object(release, "api", side_effect=self.api), self.assertRaises(ValueError): + release.provenance("123", SOURCE, TAG) + self.run[key] = original + + def test_expired_missing_ambiguous_and_foreign_artifacts(self): + for mutation in ("expired", "expiry", "missing", "duplicate", "foreign", "not-main"): + with self.subTest(mutation=mutation): + saved = copy.deepcopy(self.artifact) + if mutation == "expired": self.artifact["expired"] = True + if mutation == "expiry": self.artifact["expires_at"] = "2000-01-01T00:00:00Z" + if mutation == "missing": self.artifacts = [] + if mutation == "duplicate": self.artifacts = [self.artifact, self.artifact] + if mutation == "foreign": self.artifact["workflow_run"]["head_repository_id"] = 2 + if mutation == "not-main": self.status = "diverged" + with patch.object(release, "api", side_effect=self.api), self.assertRaises(ValueError): + release.provenance("123", SOURCE, TAG) + self.artifact = saved + self.artifacts = [self.artifact] + self.status = "ahead" + + def test_existing_tag_or_release_blocks_publication(self): + for results in (({},), (None, {})): + with patch.object(release, "api", side_effect=results), self.assertRaises(ValueError): + release.check_absent(TAG) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_release_workflow.py b/tests/test_release_workflow.py index 668a71b..b5955ed 100644 --- a/tests/test_release_workflow.py +++ b/tests/test_release_workflow.py @@ -1,5 +1,6 @@ """Keep preparation runs unable to enter the release-writing job.""" +import re import unittest from pathlib import Path @@ -18,10 +19,43 @@ def test_preparation_has_no_release_write_path(self): self.assertIn("name: stable-diffusion-native-${{ inputs.tag }}", preparation) self.assertIn("retention-days: 14", preparation) job_header = release.split(" steps:", 1)[0] - self.assertIn(" needs: package\n", job_header) - self.assertIn(" if: ${{ inputs.publish }}\n", job_header) + self.assertIn(" needs: verify\n", job_header) + self.assertIn("inputs.publish && github.ref == 'refs/heads/main'", job_header) self.assertIn(" contents: write\n", job_header) - self.assertEqual(workflow.count("action-gh-release@"), 1) + self.assertNotIn("action-gh-release@", workflow) + self.assertIn("tools/prepared_release.py publish", release) + self.assertIn("tools/prepared_release.py replay", preparation) + self.assertIn("if: ${{ !inputs.prepared_run_id }}", preparation) + self.assertIn("group: native-release-${{ inputs.tag }}", preparation) + self.assertIn("cancel-in-progress: false", preparation) + + def test_replay_publication_ignores_skipped_builds_but_requires_verification(self): + workflow = (ROOT / ".github/workflows/native_release.yml").read_text() + release = workflow.split("\n release:\n", 1)[1] + expression = release.split("if: ${{", 1)[1].split("}}", 1)[0].strip() + # Actions implicitly adds success() only when there is no status check. + # Replay intentionally skips build/package, so that implicit gate is false. + if not re.search(r"\b(always|cancelled|success|failure)\(", expression): + expression = f"success() && ({expression})" + for verified, cancelled, expected in ( + ("success", False, True), + ("failure", False, False), + ("cancelled", False, False), + ("skipped", False, False), + ("success", True, False), + ): + with self.subTest(verified=verified, cancelled=cancelled): + condition = expression.replace("needs.verify.result", repr(verified)) + condition = condition.replace("inputs.publish", "True") + condition = condition.replace("github.ref", repr("refs/heads/main")) + condition = condition.replace("github.repository", repr("leehack/stable-diffusion-native")) + condition = condition.replace("&&", " and ").replace("!", "not ") + actual = eval(condition, {"__builtins__": {}}, { + "always": lambda: True, + "cancelled": lambda: cancelled, + "success": lambda: False, + }) + self.assertEqual(actual, expected) def test_linux_runtime_builds_keep_the_qualified_ubuntu_baseline(self): # The floating label moved to Ubuntu 26.04 and a different software @@ -35,13 +69,13 @@ def test_linux_runtime_builds_keep_the_qualified_ubuntu_baseline(self): rf"target: {target}, runs_on: ubuntu-24\.04,", ) - def test_normal_release_mode_remains_explicit_and_enabled(self): + def test_publication_requires_explicit_opt_in(self): workflow = (ROOT / ".github/workflows/native_release.yml").read_text() self.assertIn( " publish:\n" " description: Publish the prepared artifacts as a GitHub release\n" " type: boolean\n" - " default: true\n", + " default: false\n", workflow, ) diff --git a/tools/package_release.py b/tools/package_release.py index 88f1436..b261a0c 100644 --- a/tools/package_release.py +++ b/tools/package_release.py @@ -15,6 +15,7 @@ import json import re import shutil +import subprocess import sys import tarfile from pathlib import Path @@ -120,6 +121,12 @@ def main() -> None: "schemaVersion": 1, "package": PACKAGE, "tag": args.tag, + "source": { + "repository": "leehack/stable-diffusion-native", + "commit": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=REPO_ROOT, text=True + ).strip(), + }, "upstream": upstream_info(), "artifacts": artifacts, } diff --git a/tools/prepared_release.py b/tools/prepared_release.py new file mode 100644 index 0000000..0a5a897 --- /dev/null +++ b/tools/prepared_release.py @@ -0,0 +1,241 @@ +#!/usr/bin/env python3 +"""Verify a prepared release, or publish its verified bytes from the release job.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import shutil +import stat +import subprocess +import tarfile +import tempfile +import zipfile +from datetime import datetime, timezone +from pathlib import Path + +from build import TARGETS +from package_release import PACKAGE, TAG_PATTERN, sha256 + +REPOSITORY = "leehack/stable-diffusion-native" +WORKFLOW = ".github/workflows/native_release.yml" +SHA_PATTERN = re.compile(r"[0-9a-f]{40}") +DIGEST_PATTERN = re.compile(r"[0-9a-f]{64}") + + +def require(condition: bool, message: str) -> None: + if not condition: + raise ValueError(message) + + +def api(path: str, *, method: str = "GET", data: dict | None = None, + missing_ok: bool = False) -> dict | None: + command = ["gh", "api", f"repos/{REPOSITORY}/{path}", "--method", method] + if data is not None: + command += ["--input", "-"] + result = subprocess.run(command, input=json.dumps(data) if data is not None else None, + text=True, capture_output=True) + if result.returncode: + if missing_ok and "(HTTP 404)" in result.stderr: + return None + raise ValueError(f"GitHub API failed: {method} {path.split('?')[0]}") + return json.loads(result.stdout) + + +def validate_inputs(tag: str, source: str, digest: str) -> None: + require(TAG_PATTERN.fullmatch(tag) is not None, "invalid release tag") + require(SHA_PATTERN.fullmatch(source) is not None, "expected source must be a full SHA") + require(DIGEST_PATTERN.fullmatch(digest) is not None, "expected manifest SHA256 required") + + +def provenance(run_id: str, source: str, tag: str) -> tuple[dict, str]: + require(re.fullmatch(r"[1-9][0-9]*", run_id) is not None, "invalid prepared run ID") + run = api(f"actions/runs/{run_id}") + workflow = api("actions/workflows/native_release.yml") + require(run["id"] == int(run_id) and run["workflow_id"] == workflow["id"] + and run["path"] == WORKFLOW, "prepared run is not the release workflow") + require(run["event"] == "workflow_dispatch" and run["status"] == "completed" + and run["conclusion"] == "success", "prepared run did not succeed") + require(run["repository"]["full_name"] == REPOSITORY + and run["head_repository"]["full_name"] == REPOSITORY + and run["head_branch"] == "main" and run["head_sha"] == source, + "prepared source repository, branch or SHA mismatch") + comparison = api(f"compare/{source}...main") + require(comparison["status"] in ("ahead", "identical"), + "prepared source is not on main") + upstream = api(f"contents/third_party/stable-diffusion.cpp?ref={source}") + require(upstream["submodule_git_url"] == + "https://github.com/leejet/stable-diffusion.cpp.git", + "unexpected upstream repository") + artifacts = [] + page = 1 + while True: + batch = api(f"actions/runs/{run_id}/artifacts?per_page=100&page={page}")["artifacts"] + artifacts.extend(batch) + if len(batch) < 100: + break + page += 1 + matches = [a for a in artifacts if a["name"] == f"{PACKAGE}-{tag}"] + require(len(matches) == 1, "prepared artifact missing or ambiguous") + artifact = matches[0] + require(not artifact["expired"] and datetime.fromisoformat( + artifact["expires_at"].replace("Z", "+00:00")) > datetime.now(timezone.utc), + "prepared artifact expired") + origin = artifact["workflow_run"] + require(origin["id"] == int(run_id) and origin["head_sha"] == source + and origin["repository_id"] == run["repository"]["id"] + and origin["head_repository_id"] == run["repository"]["id"], + "artifact provenance mismatch") + return artifact, upstream["sha"] + + +def unpack(archive: Path, dest: Path) -> None: + require(not dest.exists(), "destination must not exist") + with zipfile.ZipFile(archive) as bundle: + names = set() + for entry in bundle.infolist(): + name = entry.filename + mode = entry.external_attr >> 16 + require(name not in names and name not in ("", ".", "..") + and "/" not in name and "\\" not in name + and not entry.is_dir() and not stat.S_ISLNK(mode) + and stat.S_IFMT(mode) in (0, stat.S_IFREG), + "artifact contains duplicate, nested or non-regular files") + names.add(name) + dest.mkdir(parents=True) + for entry in bundle.infolist(): + with bundle.open(entry) as src, (dest / entry.filename).open("xb") as out: + shutil.copyfileobj(src, out) + + +def verify_dist(dist: Path, tag: str, source: str, digest: str, upstream: str) -> dict: + validate_inputs(tag, source, digest) + require(dist.is_dir() and not dist.is_symlink(), "invalid distribution directory") + files = list(dist.iterdir()) + require(all(p.is_file() and not p.is_symlink() for p in files), + "distribution contains non-regular files") + require(sha256(dist / "manifest.json") == digest, "manifest SHA256 mismatch") + manifest = json.loads((dist / "manifest.json").read_text()) + require(manifest["schemaVersion"] == 1 and manifest["package"] == PACKAGE + and manifest["tag"] == tag, "manifest package, schema or tag mismatch") + require(manifest["upstream"]["repository"] == "leejet/stable-diffusion.cpp" + and manifest["upstream"]["commit"] == upstream, "manifest upstream mismatch") + # Legacy prepared runs bind the owner SHA through run provenance; new packages + # record it too. The manifest digest is always an explicit replay input. + if "source" in manifest: + require(manifest["source"] == {"repository": REPOSITORY, "commit": source}, + "manifest owner source mismatch") + expected = {(name, kind): f"{PACKAGE}-{kind}-{name}-{tag}.tar.gz" + for name in TARGETS for kind in ("runtime", "symbols")} + expected[("apple", "xcframework")] = f"{PACKAGE}-apple-xcframework-{tag}.zip" + seen = set() + sums = {} + for artifact in manifest["artifacts"]: + key = (artifact["target"], artifact["kind"]) + require(key in expected and key not in seen and artifact["file"] == expected[key], + "manifest archive inventory mismatch") + seen.add(key) + path = dist / expected[key] + require(type(artifact["size"]) is int and path.stat().st_size == artifact["size"] + and DIGEST_PATTERN.fullmatch(artifact["sha256"]) is not None + and sha256(path) == artifact["sha256"], "archive size or SHA256 mismatch") + sums[path.name] = artifact["sha256"] + if key[1] == "runtime": + with tarfile.open(path, "r:gz") as archive: + entries = [e for e in archive.getmembers() if e.name == "build-info.json"] + require(len(entries) == 1 and entries[0].isfile(), "missing build information") + info = json.load(archive.extractfile(entries[0])) + require(info["target"] == key[0] and info["upstream"] == manifest["upstream"], + "archive build source mismatch") + require(seen == set(expected), "manifest is missing release targets") + require({p.name for p in files} == set(sums) | {"manifest.json", "SHA256SUMS"}, + "distribution has extra or missing files") + lines = (dist / "SHA256SUMS").read_text().splitlines() + require(len(lines) == len(sums) and set(lines) == {f"{v} {k}" for k, v in sums.items()}, + "SHA256SUMS does not match manifest") + return manifest + + +def check_absent(tag: str) -> None: + require(api(f"releases/tags/{tag}", missing_ok=True) is None, + "release already exists; refusing to overwrite") + require(api(f"git/ref/tags/{tag}", missing_ok=True) is None, + "tag already exists; refusing to reuse") + + +def replay(args: argparse.Namespace) -> None: + validate_inputs(args.tag, args.source, args.manifest_sha256) + artifact, upstream = provenance(args.run_id, args.source, args.tag) + with tempfile.TemporaryDirectory() as tmp: + archive = Path(tmp) / "artifact.zip" + with archive.open("wb") as output: + subprocess.run(["gh", "api", f"repos/{REPOSITORY}/actions/artifacts/" + f"{artifact['id']}/zip"], stdout=output, check=True) + require(artifact.get("digest") == "sha256:" + sha256(archive), + "downloaded artifact digest mismatch") + unpack(archive, args.dist) + verify_dist(args.dist, args.tag, args.source, args.manifest_sha256, upstream) + check_absent(args.tag) + print(f"Verified {args.tag}: source {args.source}, manifest {args.manifest_sha256}") + + +def publish(args: argparse.Namespace) -> None: + require(os.environ.get("GITHUB_REPOSITORY") == REPOSITORY + and os.environ.get("GITHUB_WORKFLOW") == "Native Build & Release" + and os.environ.get("GITHUB_ACTIONS") == "true" + and os.environ.get("GITHUB_REF") == "refs/heads/main", + "publish only through release workflow on main") + validate_inputs(args.tag, args.source, args.manifest_sha256) + upstream = api(f"contents/third_party/stable-diffusion.cpp?ref={args.source}")["sha"] + verify_dist(args.dist, args.tag, args.source, args.manifest_sha256, upstream) + check_absent(args.tag) + # Creating the ref is atomic: another publisher winning the race fails this + # request, before release creation or any asset upload. Existing tags are held. + api("git/refs", method="POST", data={"ref": f"refs/tags/{args.tag}", "sha": args.source}) + command = ["gh", "release", "create", args.tag, "--repo", REPOSITORY, + "--verify-tag", "--title", args.tag, "--notes", + f"Prepared source: {args.source}\nManifest SHA256: {args.manifest_sha256}"] + if args.prerelease: + command.append("--prerelease") + command.extend(str(p) for p in sorted(args.dist.iterdir())) + try: + subprocess.run(command, check=True) + except subprocess.CalledProcessError as error: + raise ValueError( + f"tag {args.tag} was created, but publication did not finish; " + "inspect the partial tag/release state before recovery; no overwrite attempted" + ) from error + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + for name in ("replay", "verify", "publish"): + command = sub.add_parser(name) + command.add_argument("--tag", required=True) + command.add_argument("--source", required=True) + command.add_argument("--manifest-sha256", required=True) + command.add_argument("--dist", type=Path, default=Path("dist")) + if name == "replay": + command.add_argument("--run-id", required=True) + if name == "verify": + command.add_argument("--upstream", required=True) + if name == "publish": + command.add_argument("--prerelease", action="store_true") + args = parser.parse_args() + try: + if args.command == "replay": + replay(args) + elif args.command == "publish": + publish(args) + else: + verify_dist(args.dist, args.tag, args.source, args.manifest_sha256, args.upstream) + check_absent(args.tag) + except (ValueError, KeyError, OSError, tarfile.TarError, zipfile.BadZipFile) as error: + parser.exit(1, f"error: {error}\n") + + +if __name__ == "__main__": + main()