fix: sign and publish releases the way gitops-reverser does

.devcontainer/Dockerfile

@@ -56,6 +56,9 @@ RUN apt-get update \
 # HELM_VERSION / HELM_SHA256 -> https://github.com/helm/helm/releases; the checksum is
 #   get.helm.sh/helm-<version>-linux-amd64.tar.gz.sha256sum. The e2e fixture installs
 #   Traefik with it.
+# COSIGN_VERSION        -> https://github.com/sigstore/cosign/releases; release.yml's image
+#   job installs the same version with cosign-installer, so keep the two in step.
+# ORAS_VERSION          -> https://github.com/oras-project/oras/releases
 ENV PATH="/go/bin:/usr/local/go/bin:${PATH}" \
     GOLANGCI_LINT_VERSION=v2.13.2 \
     TASK_VERSION=v3.53.1 \
@@ -64,7 +67,9 @@ ENV PATH="/go/bin:/usr/local/go/bin:${PATH}" \
     ACTIONLINT_VERSION=1.7.12 \
     HADOLINT_VERSION=2.15.1 \
     HELM_VERSION=v4.3.0 \
-    HELM_SHA256=86584a54def73570558f66f5111cc53dfed56689637ae32c1201205d494f54fb
+    HELM_SHA256=86584a54def73570558f66f5111cc53dfed56689637ae32c1201205d494f54fb \
+    COSIGN_VERSION=v3.1.3 \
+    ORAS_VERSION=1.3.4
 
 RUN test "$(dpkg --print-architecture)" = "amd64" \
     || (echo "This devcontainer currently supports amd64 only." && exit 1)
@@ -92,6 +97,32 @@ RUN curl -fsSL -o /tmp/helm.tar.gz "https://get.helm.sh/helm-${HELM_VERSION}-lin
     && rm /tmp/helm.tar.gz \
     && helm version --short
 
+# cosign signs the published Helm chart (release.yml), and verifies what a release
+# published from here.
+RUN asset="cosign-linux-amd64" \
+    && base="https://github.com/sigstore/cosign/releases/download/${COSIGN_VERSION}" \
+    && curl -fsSL -o /tmp/cosign "${base}/${asset}" \
+    && curl -fsSL -o /tmp/cosign-checksums.txt "${base}/cosign_checksums.txt" \
+    && awk -v a="${asset}" '$2 == a { print $1 "  /tmp/cosign" }' /tmp/cosign-checksums.txt \
+       | sha256sum -c - \
+    && install -m 0755 /tmp/cosign /usr/local/bin/cosign \
+    && rm /tmp/cosign /tmp/cosign-checksums.txt \
+    && cosign version
+
+# oras pushes charts/artifacthub-repo.yml beside the chart under the `artifacthub.io` tag
+# (release.yml), a plain OCI artifact that `helm push` cannot produce. The checksum file
+# comes from the same release as the archive.
+RUN asset="oras_${ORAS_VERSION}_linux_amd64.tar.gz" \
+    && base="https://github.com/oras-project/oras/releases/download/v${ORAS_VERSION}" \
+    && curl -fsSL -o /tmp/oras.tar.gz "${base}/${asset}" \
+    && curl -fsSL -o /tmp/oras-checksums.txt "${base}/oras_${ORAS_VERSION}_checksums.txt" \
+    && awk -v a="${asset}" '$2 == a { print $1 "  /tmp/oras.tar.gz" }' /tmp/oras-checksums.txt \
+       | sha256sum -c - \
+    && tar -xzO -f /tmp/oras.tar.gz oras > /usr/local/bin/oras \
+    && chmod +x /usr/local/bin/oras \
+    && rm /tmp/oras.tar.gz /tmp/oras-checksums.txt \
+    && oras version
+
 RUN curl -fsSL -o /usr/local/bin/hadolint "https://github.com/hadolint/hadolint/releases/download/v${HADOLINT_VERSION}/hadolint-Linux-x86_64" \
     && chmod +x /usr/local/bin/hadolint \
     && hadolint --version

.github/workflows/ci.yml

@@ -75,6 +75,8 @@ jobs:
             kubectl version --client
             k3d version
             helm version --short
+            cosign version
+            oras version
             docker --version
             openssl version
           '

.github/workflows/release.yml

@@ -1,26 +1,36 @@
-# Versions, and the published image and Helm chart.
+# Versions, and the published image and Helm chart. The shape follows gitops-reverser's
+# release.yml, where it was tuned first; the differences are noted where they occur.
 #
-# On every push to main, release-please keeps a release pull request up to date from the
-# conventional-commit messages: `fix:` is a patch, and `feat:` and `feat!:` are both a minor
+# On every push to main, the full CI workflow runs first, and only on a green commit does
+# release-please do anything: `fix:` is a patch, and `feat:` and `feat!:` are both a minor
 # while the version is below 1.0 (`bump-minor-pre-major`). The first release is 0.1.0
 # (`initial-version`; without it release-please starts at 1.0.0). Nothing moves to 1.0 on
 # its own: that is a deliberate `release-as` once other people use krm-foyer happily.
-# Merging that pull request creates the tag and GitHub release. The image is then published only after the full CI workflow has passed
-# on that exact commit.
+#
+# The repository has immutable releases switched on: a published release accepts no more
+# assets. So release-please creates the release as a draft (`draft` in
+# release-please-config.json), the publish jobs sign, attest and attach to that draft, and
+# publish-release publishes it last. A failed publish job leaves an unpublished draft, never
+# a half-populated immutable release.
 #
 # Repository setting required: Settings -> Actions -> General -> "Allow GitHub Actions to
 # create and approve pull requests", or release-please cannot open its pull request.
 #
 # A pull request opened with GITHUB_TOKEN does not trigger other workflows, so CI does not
 # run on the release pull request itself. That is acceptable because it only changes the
-# changelog and the manifest, and the ci job below validates the merged commit before anything
-# is published.
+# changelog, the manifest and the chart's versions, and the ci job below validates the
+# merged commit before anything is published.
 name: release
 
 on:
   push:
     branches: [main]
 
+env:
+  REGISTRY: ghcr.io
+  IMAGE: ghcr.io/configbutler/krm-foyer
+  CHART_REGISTRY: ghcr.io/configbutler/charts
+
 permissions:
   contents: read
 
@@ -30,10 +40,19 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  ci:
+    name: ci
+    uses: ./.github/workflows/ci.yml
+    with:
+      ref: ${{ github.sha }}
+    permissions:
+      contents: read
+
   release-please:
     name: release-please
     runs-on: ubuntu-latest
     timeout-minutes: 10
+    needs: ci
     permissions:
       contents: write
       pull-requests: write
@@ -48,37 +67,37 @@ jobs:
           token: ${{ secrets.GITHUB_TOKEN }}
           config-file: release-please-config.json
           manifest-file: .release-please-manifest.json
+          # Cut the release here, but do not open the next release pull request yet. A
+          # draft release is invisible to release-please's own "latest release" lookup, so
+          # a pull request opened now would be computed against the whole history and
+          # propose a bogus version. release-please-pr opens it once the release is
+          # published.
+          skip-github-pull-request: true
 
-  ci:
-    name: ci
-    uses: ./.github/workflows/ci.yml
-    with:
-      ref: ${{ github.sha }}
-    permissions:
-      contents: read
-
+  # Unlike gitops-reverser, which retags the multi-arch digests its CI run already pushed,
+  # the image is built here: krm-foyer is amd64 only, and its CI builds without pushing.
   image:
     name: publish the image
     runs-on: ubuntu-latest
     timeout-minutes: 20
-    needs: [release-please, ci]
+    needs: [ci, release-please]
     if: needs.release-please.outputs.release_created == 'true'
     permissions:
-      contents: read
+      contents: write # attach the SBOM and the release notes to the draft release
       packages: write
-      id-token: write # keyless signing of the build provenance
+      id-token: write # cosign keyless signing + attestation OIDC
       attestations: write
     env:
-      IMAGE: ghcr.io/configbutler/krm-foyer
       VERSION: ${{ needs.release-please.outputs.version }}
+      TAG_NAME: ${{ needs.release-please.outputs.tag_name }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
           ref: ${{ github.sha }}
       - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
       - name: log in to ghcr.io
-        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login "$REGISTRY" -u "${{ github.actor }}" --password-stdin
       - id: build
         uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
         with:
@@ -88,33 +107,110 @@ jobs:
           tags: ${{ env.IMAGE }}:${{ env.VERSION }}
           provenance: mode=max
           sbom: true
+      - uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
+        with:
+          # Keep in step with COSIGN_VERSION in .devcontainer/Dockerfile, where the chart
+          # job gets its cosign. Without it the action installs its own, older, release.
+          cosign-release: v3.1.3
+      - name: sign the image (cosign keyless)
+        # One signature on the digest covers every tag that points at it.
+        run: cosign sign --yes "$IMAGE@${{ steps.build.outputs.digest }}"
       - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
         with:
           subject-name: ${{ env.IMAGE }}
           subject-digest: ${{ steps.build.outputs.digest }}
           push-to-registry: true
+      - name: generate the SBOM (SPDX)
+        uses: anchore/sbom-action@3ad7283483fc7af8ff2b4ea19663c2d5ca935e26 # v0.24.2
+        with:
+          image: ${{ env.IMAGE }}@${{ steps.build.outputs.digest }}
+          format: spdx-json
+          output-file: sbom.spdx.json
+          upload-artifact: false
+          upload-release-assets: false
+      - uses: actions/attest-sbom@c604332985a26aa8cf1bdc465b92731239ec6b9e # v4.1.0
+        with:
+          subject-name: ${{ env.IMAGE }}
+          subject-digest: ${{ steps.build.outputs.digest }}
+          sbom-path: sbom.spdx.json
+          push-to-registry: true
+      # The image is signed and attested in the registry above, but OpenSSF Scorecard's
+      # Signed-Releases check only looks at the GitHub release's assets. So the SBOM is
+      # attached with a signature (*.sigstore.json) and SLSA provenance (*.intoto.jsonl)
+      # beside it.
+      - name: sign the SBOM asset (cosign keyless)
+        run: cosign sign-blob --bundle sbom.spdx.json.sigstore.json --yes sbom.spdx.json
+      - id: attest-sbom-asset
+        uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
+        with:
+          subject-path: sbom.spdx.json
+      - name: rename the provenance bundle to the suffix Scorecard recognises
+        run: cp "${{ steps.attest-sbom-asset.outputs.bundle-path }}" sbom.spdx.json.intoto.jsonl
+      - name: attach the SBOM and the release notes to the draft release
+        uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3
+        with:
+          tag_name: ${{ env.TAG_NAME }}
+          # Still a draft: an immutable release rejects assets once published, and
+          # publish-release publishes it after every job has attached its own.
+          draft: true
+          append_body: true
+          files: |
+            sbom.spdx.json
+            sbom.spdx.json.sigstore.json
+            sbom.spdx.json.intoto.jsonl
+          body: |
+            ## Install
+
+            ```bash
+            helm install krm-foyer oci://${{ env.CHART_REGISTRY }}/krm-foyer --version ${{ env.VERSION }}
+            ```
+
+            The image is `${{ env.IMAGE }}:${{ env.VERSION }}` (linux/amd64), digest
+            `${{ steps.build.outputs.digest }}`.
+
+            ## Verify this release
+
+            The image and the chart are signed with cosign (keyless). The image also carries
+            SLSA build provenance and an SPDX SBOM as registry attestations.
+
+            ```bash
+            cosign verify \
+              --certificate-identity-regexp '^https://github.com/ConfigButler/krm-foyer/\.github/workflows/release\.yml@refs/heads/main$' \
+              --certificate-oidc-issuer https://token.actions.githubusercontent.com \
+              ${{ env.IMAGE }}:${{ env.VERSION }}
 
-  # The Helm chart, as an OCI artifact beside the image, as gitops-reverser publishes its
-  # own: helm install krm-foyer oci://ghcr.io/configbutler/charts/krm-foyer --version X.
-  # After the image, because the chart's default image tag is its appVersion: a chart is
-  # never published before the image it points at. Helm comes from the CI image, like
-  # every other tool; it is the same artifact the ci job built for this commit.
+            cosign verify \
+              --certificate-identity-regexp '^https://github.com/ConfigButler/krm-foyer/\.github/workflows/release\.yml@refs/heads/main$' \
+              --certificate-oidc-issuer https://token.actions.githubusercontent.com \
+              ${{ env.CHART_REGISTRY }}/krm-foyer:${{ env.VERSION }}
+
+            gh attestation verify oci://${{ env.IMAGE }}:${{ env.VERSION }} --repo ConfigButler/krm-foyer
+            ```
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+  # The Helm chart, as an OCI artifact beside the image: helm install krm-foyer
+  # oci://ghcr.io/configbutler/charts/krm-foyer --version X. After the image, because the
+  # chart's default image tag is its appVersion: a chart is never published before the
+  # image it points at. helm, cosign and oras come from the CI image, like every other
+  # tool. gitops-reverser runs this job inside that image with `container:`; krm-foyer's CI
+  # image is a workflow artifact rather than a registry image, so each step docker-runs it.
   chart:
     name: publish the Helm chart
     runs-on: ubuntu-latest
     timeout-minutes: 15
-    needs: [release-please, ci, image]
+    needs: [ci, release-please, image]
     if: needs.release-please.outputs.release_created == 'true'
     permissions:
       contents: read
       packages: write
-      id-token: write # keyless signing of the build provenance
-      attestations: write
+      id-token: write # cosign keyless signing
     env:
       CHART: ghcr.io/configbutler/charts/krm-foyer
       VERSION: ${{ needs.release-please.outputs.version }}
       CI_IMAGE: krm-foyer-ci:local
       CI_WORKDIR: /workspaces/krm-foyer
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -131,8 +227,6 @@ jobs:
       # (release-please-config.json); a chart that disagrees with the tag is not pushed.
       - id: push
         name: package and push the chart
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           docker run --rm -v "$GITHUB_WORKSPACE:$CI_WORKDIR" -w "$CI_WORKDIR" \
             -e GITHUB_TOKEN -e GITHUB_ACTOR -e VERSION -e CHART "$CI_IMAGE" bash -lc '
@@ -149,8 +243,74 @@ jobs:
           digest="$(awk '/^Digest:/ { print $2 }' "$RUNNER_TEMP/push.log")"
           [ -n "$digest" ] || { echo "no digest in helm push's output" >&2; exit 1; }
           echo "digest=$digest" >> "$GITHUB_OUTPUT"
-      - uses: actions/attest-build-provenance@4d101475d8b20a2381f78447822ac1eab6504dd8 # v4.2.2
+      # Each tool logs in for itself: `helm registry login` writes helm's own registry
+      # config, and cosign reads ~/.docker/config.json, so without its own login the
+      # signature upload fails with UNAUTHORIZED. The ACTIONS_ID_TOKEN_REQUEST_* variables
+      # carry the job's OIDC token into the container for keyless signing.
+      - name: sign the chart (cosign keyless)
+        env:
+          DIGEST: ${{ steps.push.outputs.digest }}
+        run: |
+          docker run --rm -e GITHUB_TOKEN -e GITHUB_ACTOR -e CHART -e DIGEST \
+            -e ACTIONS_ID_TOKEN_REQUEST_URL -e ACTIONS_ID_TOKEN_REQUEST_TOKEN "$CI_IMAGE" bash -lc '
+              set -euo pipefail
+              cosign login ghcr.io -u "$GITHUB_ACTOR" -p "$GITHUB_TOKEN"
+              cosign sign --yes "$CHART@$DIGEST"
+            '
+      # Artifact Hub reads the repository metadata (ownership claim, verified publisher)
+      # from a plain OCI artifact under the fixed `artifacthub.io` tag of the chart
+      # repository, recognised by its media types. `helm push` cannot produce that shape,
+      # so oras does: https://artifacthub.io/docs/topics/repositories/helm-charts/. It is
+      # pushed again on every release, so the file in Git stays the source of truth.
+      - name: push the Artifact Hub repository metadata
+        run: |
+          docker run --rm -v "$GITHUB_WORKSPACE:$CI_WORKDIR" -w "$CI_WORKDIR/charts" \
+            -e GITHUB_TOKEN -e GITHUB_ACTOR -e CHART "$CI_IMAGE" bash -lc '
+              set -euo pipefail
+              echo "$GITHUB_TOKEN" | oras login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
+              oras push "$CHART:artifacthub.io" \
+                --config /dev/null:application/vnd.cncf.artifacthub.config.v1+yaml \
+                artifacthub-repo.yml:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml
+            '
+
+  # Publishes the draft release once every asset is attached; see the top of this file.
+  publish-release:
+    name: publish the GitHub release
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs: [release-please, image, chart]
+    if: needs.release-please.outputs.release_created == 'true'
+    permissions:
+      contents: write # publish the draft release, which creates the tag
+    steps:
+      - name: publish the draft release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          TAG_NAME: ${{ needs.release-please.outputs.tag_name }}
+        run: gh release edit "$TAG_NAME" --draft=false --repo "${{ github.repository }}"
+
+  # The pull request half of release-please, run separately so it sees a published
+  # release instead of the draft. It runs after publish-release when a release was cut,
+  # and on every other green push so the pending release pull request keeps up; after a
+  # failed publish (the release still a draft) it opens nothing.
+  release-please-pr:
+    name: release-please (pull request)
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    needs: [ci, release-please, publish-release]
+    if: >-
+      ${{ !cancelled()
+          && needs.ci.result == 'success'
+          && needs.release-please.result == 'success'
+          && (needs.publish-release.result == 'success'
+              || needs.release-please.outputs.release_created != 'true') }}
+    permissions:
+      contents: write
+      pull-requests: write
+    steps:
+      - uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0
         with:
-          subject-name: ${{ env.CHART }}
-          subject-digest: ${{ steps.push.outputs.digest }}
-          push-to-registry: true
+          token: ${{ secrets.GITHUB_TOKEN }}
+          config-file: release-please-config.json
+          manifest-file: .release-please-manifest.json
+          skip-github-release: true