fix(arc): read the App's id from an existing App Secret

plugins/filter/arc.py

@@ -417,13 +417,14 @@ def _expand_profile(org_name: str, app_secret: str, index: int, profile: Any, er
     return expanded
 
 
-def arc_profiles(orgs: Sequence[Any]) -> dict[str, Any]:
+def arc_profiles(orgs: Sequence[Any], require_app_id: bool = True) -> dict[str, Any]:
     """Expand github_runner_arc_orgs entries into one record per scale-set profile, and check them.
 
     Pass every org configured anywhere in the inventory, not one host's, so that the cross-host checks (an org configured twice, two profiles sharing a namespace, more than one autoscaled profile) see the whole fleet.
 
     Args:
-        orgs: github_runner_arc_orgs entries, each with name, app_id, image and a non-empty scale_set_profiles list, at most one of private_key and private_key_op_reference, and optionally app_secret_name. A profile may override its namespace and release_name, and set scale_set_labels in place of runs_on_label.
+        require_app_id: whether each org must set app_id. The role needs it only when it writes the App Secret; otherwise the App's id is in the existing App Secret, and an app_id that is set is only checked against it.
+        orgs: github_runner_arc_orgs entries, each with name, app_id (see require_app_id), image and a non-empty scale_set_profiles list, at most one of private_key and private_key_op_reference, and optionally app_secret_name. A profile may override its namespace and release_name, and set scale_set_labels in place of runs_on_label.
 
     Returns:
         A dict with ``errors`` (messages, empty when valid), ``profiles`` (one dict per profile with org_name, suffix, namespace, release, app_secret, scale_set_labels, values_file, node_selector, autoscale, sizing, max_runners (the static maxRunners the role sets, or None), label and settings, the profile's own keys) and ``autoscaled`` (the one profile flagged autoscale, or None).
@@ -442,9 +443,10 @@ def arc_profiles(orgs: Sequence[Any]) -> dict[str, Any]:
         if not org_name:
             errors.append(f"{where} has no name")
             continue
-        for key in ("app_id", "image"):
-            if not _text(org, key):
-                errors.append(f"{org_name}: {key} is required")
+        if require_app_id and not _text(org, "app_id"):
+            errors.append(f"{org_name}: app_id is required when the role writes the App Secret (github_runner_arc_manage_secrets); with an existing App Secret it is read from that Secret's github_app_id")
+        if not _text(org, "image"):
+            errors.append(f"{org_name}: image is required")
         # Presence only: reading the value would decrypt an ansible-vault string for no reason.
         if org.get("private_key") is not None and org.get("private_key_op_reference") is not None:
             errors.append(f"{org_name}: set private_key or private_key_op_reference, not both (the 1Password adapter fills private_key from the reference)")

roles/github_runner_arc/README.md

@@ -12,7 +12,7 @@ Before touching the cluster the role then checks the secrets it is about to writ
 
 ## Variables
 
-- `github_runner_arc_orgs`: the orgs this host installs. Each entry has `name`, `app_id`, `image`, `private_key` (the App's PEM private key, from any source Ansible reads) or `private_key_op_reference` (an `op://` reference that the `github_runner_secrets_onepassword` adapter resolves into `private_key`), optionally `installation_id` (resolved from the App when empty), optionally `app_secret_name` (the App Secret's name in each of the org's namespaces, default `<org>-github-app`), and `scale_set_profiles`, a list of profiles:
+- `github_runner_arc_orgs`: the orgs this host installs. Each entry has `name`, `app_id` (needed only when the role writes the App Secret, see `github_runner_arc_manage_secrets`), `image`, `private_key` (the App's PEM private key, from any source Ansible reads) or `private_key_op_reference` (an `op://` reference that the `github_runner_secrets_onepassword` adapter resolves into `private_key`), optionally `installation_id` (resolved from the App when empty), optionally `app_secret_name` (the App Secret's name in each of the org's namespaces, default `<org>-github-app`), and `scale_set_profiles`, a list of profiles:
   - `suffix`: appended to the namespace (`arc-runners-<org><suffix>`) and release (`<org>-runners<suffix>`) names. Default empty.
   - `namespace`, `release_name`: the profile's namespace and Helm release name, in place of the names derived from the suffix. The release name is also the scale set's name on GitHub, so two profiles of one org cannot share it.
   - `values_file`: Helm values for the release, a Jinja template read from `github_runner_arc_values_dir` on the control node (or an absolute path). Without it the role's `templates/runner-scale-set-values.yaml.j2` is used, driven by the profile's own `max_runners`, `min_runners`, `container_mode` (`dind` for the chart's Docker-in-Docker mode) and `resources` (the runner container's requests and limits).
@@ -31,7 +31,7 @@ Before touching the cluster the role then checks the secrets it is about to writ
 - `github_runner_arc_listener_probes_enabled`: readiness and liveness probes on each listener's metrics endpoint, so a listener that starts and then fails its GitHub authentication is not counted ready during a rollout. Needs metrics on.
 - `github_runner_arc_node_label_key`, `github_runner_arc_node_label_value`: when the key is set, the controller, listeners and runner pods are restricted to nodes carrying the label. The role labels the nodes named in `github_runner_arc_labelled_nodes`.
 - `github_runner_arc_ghcr_username`, `github_runner_arc_ghcr_token`, `github_runner_arc_heartbeat_gh_token`: secrets, as plain variables. The pull credential is needed only for a static pull Secret.
-- `github_runner_arc_manage_secrets`: `false` stops the role writing the App Secrets and `heartbeat-gh-token`, and instead checks before any change that they already exist. The role then needs no `private_key` or `installation_id`.
+- `github_runner_arc_manage_secrets`: `false` stops the role writing the App Secrets and `heartbeat-gh-token`, and instead checks before any change that they already exist, and that each App Secret holds `github_app_id`, `github_app_installation_id` and `github_app_private_key`. The role then needs no `private_key` or `installation_id`, and no `app_id` either: it reads the App's id from the Secret where it needs it. An `app_id` that is set must match the Secret's `github_app_id`.
 - `github_runner_arc_manage_image_pull_secret`: `false` stops the role writing the pull Secret, so an existing one named `github_runner_arc_image_pull_secret_name` is used as it is, kept current by something else. No pull credential is needed and the pull check is skipped; the role checks the Secret exists. Follows `github_runner_arc_manage_secrets` unless set.
 - `github_runner_arc_image_pull_secret_source`: where a pull Secret the role writes gets its credential: `static` (the default) from `github_runner_arc_ghcr_username` and `github_runner_arc_ghcr_token`, or `app` from each org's GitHub App, renewed in the cluster (see [Image pull Secret from the GitHub App](#image-pull-secret-from-the-github-app)). `github_runner_arc_image_pull_secret_renewal_schedule` (default every 15 minutes) and `github_runner_arc_image_pull_secret_renewer_image` configure the renewal.
 - `github_runner_arc_image_pull_registry`, `github_runner_arc_image_pull_secret_name`, `github_runner_arc_verify_image_pull`: the pull Secret's registry and name, and whether to prove the credential before writing it.
@@ -162,8 +162,8 @@ github_runner_arc_manage_secrets: false
 github_runner_arc_manage_image_pull_secret: false
 github_runner_arc_image_pull_secret_name: "existing-pull-secret"
 github_runner_arc_orgs:
+  # No app_id: the role reads it from the existing App Secret.
   - name: ExampleOrg
-    app_id: 123456
     app_secret_name: "existing-app-secret"
     image: "ghcr.io/example/github-runner:2026.01.01"
     scale_set_profiles:

roles/github_runner_arc/defaults/main.yml

@@ -1,7 +1,7 @@
 ---
 # ARC (Actions Runner Controller) in an existing Kubernetes cluster: the controller, each org's runner scale sets, and the fleet-health platform (heartbeat and autoscaler). Each part is gated on this host's own github_runner_arc_orgs or github_runner_arc_heartbeat_gist_id, so any host that can reach the cluster's API can be the one Ansible installs from: a k3s server host brought up by github_runner_cluster, or localhost with a kubeconfig (playbooks/arc.yml). On a host github_runner_cluster made an agent, this role installs nothing.
 
-# Orgs whose runner scale sets this host installs. Each entry: name, app_id, image, private_key (the App's PEM private key, from any source Ansible reads) or private_key_op_reference (an op:// reference the 1Password adapter resolves into private_key), optionally installation_id (resolved from the App when empty), optionally app_secret_name (the App Secret's name in each of the org's namespaces, default <org>-github-app), and scale_set_profiles, a list of {suffix, namespace?, release_name?, values_file?, max_runners?, node_selector?, runs_on_label?, scale_set_labels?, autoscale?, sizing?}. sizing takes uniform node figures, or measured: true to read each eligible node's free capacity from the cluster at install time. Each profile becomes one namespace (namespace, default arc-runners-<org><suffix>), two Secrets in it, and one Helm release (release_name, default <org>-runners<suffix>); setting both overrides lets the role take over scale sets an existing install created under its own names. See the README for every profile key.
+# Orgs whose runner scale sets this host installs. Each entry: name, app_id (needed only when the role writes the App Secret; otherwise read from the existing Secret, and checked against it when set), image, private_key (the App's PEM private key, from any source Ansible reads) or private_key_op_reference (an op:// reference the 1Password adapter resolves into private_key), optionally installation_id (resolved from the App when empty), optionally app_secret_name (the App Secret's name in each of the org's namespaces, default <org>-github-app), and scale_set_profiles, a list of {suffix, namespace?, release_name?, values_file?, max_runners?, node_selector?, runs_on_label?, scale_set_labels?, autoscale?, sizing?}. sizing takes uniform node figures, or measured: true to read each eligible node's free capacity from the cluster at install time. Each profile becomes one namespace (namespace, default arc-runners-<org><suffix>), two Secrets in it, and one Helm release (release_name, default <org>-runners<suffix>); setting both overrides lets the role take over scale sets an existing install created under its own names. See the README for every profile key.
 github_runner_arc_orgs: []
 
 # Directory on the control node that a relative scale_set_profiles[].values_file is read from. Values files are Jinja templates rendered on the control node, so the target host needs no checkout. A profile with no values_file uses the role's own templates/runner-scale-set-values.yaml.j2.

roles/github_runner_arc/meta/argument_specs.yml

@@ -9,7 +9,7 @@ argument_specs:
         elements: dict
         default: []
         description:
-          - Orgs whose runner scale sets this host installs. Each entry has name, app_id, image, private_key or private_key_op_reference, and optionally installation_id and app_secret_name (the App Secret's name, default <org>-github-app).
+          - Orgs whose runner scale sets this host installs. Each entry has name, app_id (needed only when the role writes the App Secret; otherwise the existing Secret supplies it), image, private_key or private_key_op_reference, and optionally installation_id and app_secret_name (the App Secret's name, default <org>-github-app).
           - Each entry's scale_set_profiles is a non-empty list of profiles, each with suffix and optionally namespace (default arc-runners-<org><suffix>), release_name (default <org>-runners<suffix>), values_file, max_runners, min_runners, container_mode, resources, node_selector, runs_on_label, scale_set_labels, autoscale and sizing.
           - max_runners, when set, is the release's maxRunners and takes precedence over its values file and its sizing.
           - scale_set_labels replaces runs_on_label with the whole list of scaleSetLabels; an empty list sets none, so jobs target the release name.

roles/github_runner_arc/tasks/check_existing_secret_contents.yml

@@ -0,0 +1,44 @@
+---
+# Included from check_existing_secrets.yml with github_runner_arc_existing_secrets registered: fails on a referenced Secret that does not exist, or an App Secret the controller could not use.
+
+# Reduced to names, key names and the App's id, as JSON, which is all the checks below need, so that no task loops over the Secrets' data: a loop prints its items when it fails, and under -v when it skips.
+- name: Reduce the Secrets to what the checks need
+  ansible.builtin.set_fact:
+    github_runner_arc_existing_secret_checks: >-
+      {%- set checks = namespace(entries=[]) -%}
+      {%- for result in github_runner_arc_existing_secrets.results -%}
+      {%- set data = (result.resources | first | default({})).data | default({}, true) -%}
+      {%- set checks.entries = checks.entries + [{'namespace': result.item.0, 'name': result.item.1, 'exists': result.resources | length > 0,
+                                              'key_names': data.keys() | list, 'app_id': data.github_app_id | default('') | b64decode | trim}] -%}
+      {%- endfor -%}
+      {{ checks.entries | to_json }}
+  no_log: true
+
+- name: Fail when a referenced Secret does not exist
+  ansible.builtin.fail:
+    msg: >-
+      Secret {{ item.name }} does not exist in namespace {{ item.namespace }}.
+      The role is not managing it (github_runner_arc_manage_secrets or github_runner_arc_manage_image_pull_secret is false), so create it first: an App Secret holds github_app_id, github_app_installation_id and github_app_private_key;
+      a pull Secret has type kubernetes.io/dockerconfigjson; heartbeat-gh-token holds a gist-scoped GitHub token under the key token.
+  loop: "{{ github_runner_arc_existing_secret_checks | from_json }}"
+  loop_control:
+    label: "{{ item.namespace }}/{{ item.name }}"
+  when: not item.exists
+
+# The App Secrets this host does not write come first, one per profile. The role reads the App's id from them rather than requiring the org's app_id, so a Secret missing a key, or disagreeing with an app_id that is set, fails now rather than as a listener that cannot authenticate.
+- name: Check each existing App Secret holds what the controller reads
+  ansible.builtin.fail:
+    msg: >-
+      App Secret {{ item.0.namespace }}/{{ item.0.name }} of {{ item.1.org_name }}
+      {{ ('has no ' ~ (github_runner_arc_missing_keys | join(', ')) ~ '; it needs github_app_id, github_app_installation_id and github_app_private_key.')
+         if github_runner_arc_missing_keys | length > 0 else
+         ('holds github_app_id ' ~ item.0.app_id ~ ', but the org sets app_id ' ~ github_runner_arc_org_app_id ~ ". Correct or remove the org's app_id.") }}
+  loop: "{{ github_runner_arc_existing_secret_checks | from_json | zip(github_runner_arc_local.profiles) | list if not (github_runner_arc_manage_secrets | bool) else [] }}"
+  loop_control:
+    label: "{{ item.0.namespace }}/{{ item.0.name }}"
+  vars:
+    github_runner_arc_missing_keys: "{{ ['github_app_id', 'github_app_installation_id', 'github_app_private_key'] | reject('in', item.0.key_names) | list }}"
+    github_runner_arc_org_app_id: "{{ (github_runner_arc_orgs | selectattr('name', 'equalto', item.1.org_name) | first).app_id | default('', true) | string }}"
+  when:
+    - item.0.exists
+    - github_runner_arc_missing_keys | length > 0 or (github_runner_arc_org_app_id | length > 0 and github_runner_arc_org_app_id != item.0.app_id)

roles/github_runner_arc/tasks/check_existing_secrets.yml

@@ -20,14 +20,8 @@
   loop_control:
     label: "{{ item.0 }}/{{ item.1 }}"
   register: github_runner_arc_existing_secrets
+  # The results hold the Secrets' data.
+  no_log: true
 
-- name: Fail when a referenced Secret does not exist
-  ansible.builtin.fail:
-    msg: >-
-      Secret {{ item.item.1 }} does not exist in namespace {{ item.item.0 }}.
-      The role is not managing it (github_runner_arc_manage_secrets or github_runner_arc_manage_image_pull_secret is false), so create it first: an App Secret holds github_app_id, github_app_installation_id and github_app_private_key;
-      a pull Secret has type kubernetes.io/dockerconfigjson; heartbeat-gh-token holds a gist-scoped GitHub token under the key token.
-  loop: "{{ github_runner_arc_existing_secrets.results }}"
-  loop_control:
-    label: "{{ item.item.0 }}/{{ item.item.1 }}"
-  when: item.resources | length == 0
+- name: Check the Secrets found
+  ansible.builtin.include_tasks: check_existing_secret_contents.yml

roles/github_runner_arc/tasks/validate.yml

@@ -4,13 +4,14 @@
 # This host's orgs come from github_runner_arc_orgs as this play resolves it, which includes play and role variables and -e; another host's can only be read from hostvars, which holds its inventory variables and facts. Taking this host's from hostvars too would miss orgs passed as a play variable, and every check and lookup built on the fleet (the scale sets' namespaces that measured sizing leaves out, the autoscaled profile) would then not see them. An implicit localhost is not in groups['all'] at all.
 - name: Expand every org configured anywhere in the inventory into its scale-set profiles
   ansible.builtin.set_fact:
+    # Whether an org needs app_id depends on whether its host writes the App Secret, which only that host's own check knows, so the fleet-wide expansion (for the cross-host checks) does not ask for it.
     github_runner_arc_fleet: >-
       {{
         ((groups['all'] | reject('equalto', inventory_hostname) | map('extract', hostvars) | map(attribute='github_runner_arc_orgs', default=[]) | flatten)
          + github_runner_arc_orgs)
-        | exadev.github_runner.arc_profiles
+        | exadev.github_runner.arc_profiles(require_app_id=false)
       }}
-    github_runner_arc_local: "{{ github_runner_arc_orgs | exadev.github_runner.arc_profiles }}"
+    github_runner_arc_local: "{{ github_runner_arc_orgs | exadev.github_runner.arc_profiles(require_app_id=github_runner_arc_manage_secrets | bool) }}"
 
 - name: Fail on invalid orgs or scale-set profiles
   ansible.builtin.fail:

tests/unit/test_arc_filters.py

@@ -106,11 +106,17 @@ def test_invalid_autoscale_value_is_an_error(self) -> None:
 
     def test_missing_required_org_fields_are_errors(self) -> None:
         result = arc.arc_profiles([{"name": "X", "scale_set_profiles": [{}]}, {"app_id": 1}, {"name": "Y", "app_id": 1, "image": "i", "scale_set_profiles": []}])
-        self.assertIn("X: app_id is required", result["errors"])
+        self.assertIn("X: app_id is required when the role writes the App Secret (github_runner_arc_manage_secrets); with an existing App Secret it is read from that Secret's github_app_id", result["errors"])
         self.assertIn("X: image is required", result["errors"])
         self.assertIn("github_runner_arc_orgs[1] has no name", result["errors"])
         self.assertIn("Y: scale_set_profiles must be a non-empty list", result["errors"])
 
+    def test_app_id_is_optional_when_the_role_does_not_write_the_app_secret(self) -> None:
+        without = {"name": "Example", "image": "ghcr.io/example/runner:1", "scale_set_profiles": [{"max_runners": 1}]}
+        self.assertEqual(arc.arc_profiles([without], require_app_id=False)["errors"], [])
+        self.assertEqual(arc.arc_profiles([without])["errors"], ["Example: app_id is required when the role writes the App Secret (github_runner_arc_manage_secrets); with an existing App Secret it is read from that Secret's github_app_id"])
+        self.assertIn("Example: image is required", arc.arc_profiles([{**without, "image": ""}], require_app_id=False)["errors"])
+
     def test_a_private_key_and_a_1password_reference_together_are_an_error(self) -> None:
         both = dict(org(), private_key="pem", private_key_op_reference="op://v/i/f")
         self.assertIn("Example: set private_key or private_key_op_reference, not both (the 1Password adapter fills private_key from the reference)", arc.arc_profiles([both])["errors"])