Connect the Simulation Entrypoint to the Public API's database with its own role

.github/scripts/cloud-run-simulation-entry-smoke.sh

@@ -25,6 +25,13 @@ previous_revision="${3:-}"
 #   and model.
 # This is the identity the gateway reports from the routing registry; the
 # numerical release sentinel checks what the worker computes.
+# REQUIRE_SIMULATION_DATABASE_ACCESS=true (candidate checks only): /ready must
+# report the simulation service's database reached, with every privilege its
+# role needs held, which shows that the role's URL reached the revision and the
+# role holds its grants. /ready only says yes or no; the revision's Cloud Run
+# logs say why. Stable and rollback checks leave it unset, because an older
+# image may not use the database.
+require_database_access="${REQUIRE_SIMULATION_DATABASE_ACCESS:-}"
 expected_identity="${EXPECTED_IDENTITY:-none}"
 expected_policyengine_version="${EXPECTED_POLICYENGINE_VERSION:-}"
 expected_app_name="${EXPECTED_APP_NAME:-}"
@@ -39,6 +46,14 @@ case "${expected_identity}" in
     exit 2
     ;;
 esac
+case "${require_database_access}" in
+  "" | true) ;;
+  *)
+    printf 'REQUIRE_SIMULATION_DATABASE_ACCESS must be true or empty, not %s\n' \
+      "${require_database_access}" >&2
+    exit 2
+    ;;
+esac
 case "${expected_latest}" in
   "" | true) ;;
   *)
@@ -63,6 +78,10 @@ if [ "${expected_identity}" = "required" ] ||
 fi
 routing_max_attempts="${SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS:-24}"
 routing_retry_delay_seconds="${SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS:-5}"
+# A cold candidate may not reach its database within /ready's three-second
+# wait; a failed database check is asked about again, up to this many
+# requests in all.
+database_ready_max_attempts="${SIMULATION_DATABASE_READY_MAX_ATTEMPTS:-6}"
 
 if ! [[ "${routing_max_attempts}" =~ ^[1-9][0-9]*$ ]]; then
   printf 'SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS must be a positive integer\n' >&2
@@ -72,6 +91,10 @@ if ! [[ "${routing_retry_delay_seconds}" =~ ^[0-9]+$ ]]; then
   printf 'SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS must be a non-negative integer\n' >&2
   exit 2
 fi
+if ! [[ "${database_ready_max_attempts}" =~ ^[1-9][0-9]*$ ]]; then
+  printf 'SIMULATION_DATABASE_READY_MAX_ATTEMPTS must be a positive integer\n' >&2
+  exit 2
+fi
 
 health_headers="$(mktemp)"
 health_body="$(mktemp)"
@@ -131,8 +154,31 @@ while true; do
   attempt=$((attempt + 1))
 done
 
-curl --fail --silent --show-error "${base_url}/ready" |
-  jq -e '.status == "ready"' >/dev/null
+ready_body="$(curl --fail --silent --show-error "${base_url}/ready")"
+jq -e '.status == "ready"' <<<"${ready_body}" >/dev/null
+if [ "${require_database_access}" = "true" ]; then
+  database_attempt=1
+  while [ "${database_attempt}" -lt "${database_ready_max_attempts}" ] &&
+    jq -e '.simulation_database.reachable == false' <<<"${ready_body}" >/dev/null; do
+    printf '%s: the simulation database check failed; asking again (%s/%s)\n' \
+      "${base_url}" "${database_attempt}" "${database_ready_max_attempts}" >&2
+    sleep "${routing_retry_delay_seconds}"
+    ready_body="$(curl --fail --silent --show-error "${base_url}/ready")"
+    database_attempt=$((database_attempt + 1))
+  done
+  if ! jq -e '
+    .simulation_database.reachable == true
+    and .simulation_database.has_required_privileges == true
+  ' <<<"${ready_body}" >/dev/null; then
+    printf '%s: the simulation service cannot use the simulation database: %s\n' \
+      "${base_url}" "$(jq -c '.simulation_database' <<<"${ready_body}")" >&2
+    printf '%s %s %s\n' \
+      "The revision's Cloud Run logs say why: look for" \
+      "simulation_database_unreachable (the error's type and SQLSTATE) or" \
+      "simulation_database_missing_privileges (with each one missing)." >&2
+    exit 1
+  fi
+fi
 curl --fail --silent --show-error "${base_url}/versions" --output "${versions_body}"
 jq -e 'type == "object"' "${versions_body}" >/dev/null
 

.github/scripts/deploy-cloud-run-simulation-entry.sh

@@ -58,6 +58,28 @@ resolve_secret_reference() {
   printf '%s:%s' "${reference}" "${highest_enabled_version}"
 }
 
+runtime_secrets=()
+
+# Resolve one secret per assignment: an array assignment takes its exit status
+# from its last command substitution only, so resolving several secrets in
+# one would let an earlier failure pass unnoticed.
+add_runtime_secret() {
+  local variable_name="$1"
+  local resolved_reference
+  resolved_reference="$(resolve_secret_reference "$2")"
+  runtime_secrets+=("${variable_name}=${resolved_reference}")
+}
+
+# The Stage 12 comparison path's own copy of add_runtime_secret, so the
+# Stage 12 block resolves its secrets through a wrapper of its own and the
+# wrapper is removed with it. Both still call resolve_secret_reference.
+add_stage12_runtime_secret() {
+  local variable_name="$1"
+  local resolved_reference
+  resolved_reference="$(resolve_secret_reference "$2")"
+  runtime_secrets+=("${variable_name}=${resolved_reference}")
+}
+
 required_variables=(
   PROJECT_ID
   REGION
@@ -77,6 +99,7 @@ required_variables=(
   OLD_GATEWAY_AUTH_AUDIENCE_VALUE
   OLD_GATEWAY_AUTH_CLIENT_ID_VALUE
   OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME
+  SIMULATION_SERVICE_DATABASE_URL_SECRET_NAME
   OBSERVABILITY_SERVICE_NAMESPACE
   OBSERVABILITY_TRACE_PROJECT_ID
   OTEL_EXPORTER_OTLP_ENDPOINT
@@ -115,11 +138,12 @@ export SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE="${SIMULATION_ENTRYPOINT_
 [[ "${SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE}" == "0" || "${SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE}" == "1" ]] \
   || fail "SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE must be 0 or 1"
 
-runtime_secrets=(
-  "OLD_GATEWAY_AUTH_CLIENT_SECRET=$(
-    resolve_secret_reference "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}"
-  )"
-)
+add_runtime_secret OLD_GATEWAY_AUTH_CLIENT_SECRET \
+  "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}"
+# The simulation service role's URL for the database that holds the Public
+# API's tables, on every deploy whatever the Stage 12 setting.
+add_runtime_secret SIMULATION_SERVICE_DATABASE_URL \
+  "${SIMULATION_SERVICE_DATABASE_URL_SECRET_NAME}"
 
 if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then
   stage12_variables=(
@@ -138,11 +162,9 @@ if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then
     || fail "STAGE12_ARTIFACT_BUCKET_VALUE is invalid"
   validate_secret_reference "${STAGE12_DATABASE_URL_SECRET_NAME}"
 
-  runtime_secrets+=(
-    "MODAL_TOKEN_ID=$(resolve_secret_reference "${MODAL_TOKEN_ID_SECRET_NAME}")"
-    "MODAL_TOKEN_SECRET=$(resolve_secret_reference "${MODAL_TOKEN_SECRET_SECRET_NAME}")"
-    "STAGE12_DATABASE_URL=$(resolve_secret_reference "${STAGE12_DATABASE_URL_SECRET_NAME}")"
-  )
+  add_stage12_runtime_secret MODAL_TOKEN_ID "${MODAL_TOKEN_ID_SECRET_NAME}"
+  add_stage12_runtime_secret MODAL_TOKEN_SECRET "${MODAL_TOKEN_SECRET_SECRET_NAME}"
+  add_stage12_runtime_secret STAGE12_DATABASE_URL "${STAGE12_DATABASE_URL_SECRET_NAME}"
 fi
 
 runtime_environment_file="${RUNNER_TEMP}/simulation-entry-runtime-environment.yaml"

.github/workflows/pr.yml

@@ -18,7 +18,7 @@ jobs:
     # workflows on every pull request. See docs/engineering/skills/github-prs.md.
     name: CI complete
     if: ${{ always() }}
-    needs: [test, lint, pyright, quality-guards, docker-build, test-integration]
+    needs: [test, simulation-database, lint, pyright, quality-guards, docker-build, test-integration]
     runs-on: ubuntu-latest
     permissions:
       contents: read
@@ -53,6 +53,7 @@ jobs:
           - projects/policyengine-simulation-gateway
           - libs/policyengine-simulation-contract
           - libs/policyengine-simulation-observability
+          - libs/policyengine-simulation-persistence
           - libs/policyengine-stage12-persistence
           - libs/policyengine-fastapi
 
@@ -78,6 +79,56 @@ jobs:
       working-directory: ${{ matrix.project }}
       run: uv run --frozen pytest tests/ -v
 
+  simulation-database:
+    name: Simulation database
+    # Runs the persistence tests that need PostgreSQL, which the Test job
+    # skips, on a fresh server. Full CI runs only on the lowest unmerged pull
+    # request in a stack.
+    if: ${{ !github.event.pull_request.stack || github.event.pull_request.stack.base.ref == github.event.pull_request.base.ref }}
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    services:
+      postgres:
+        image: postgres:17
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: simulation_ci
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd="pg_isready -U postgres"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=20
+    env:
+      SIMULATION_TEST_DATABASE_URL: postgresql://postgres:simulation_ci@127.0.0.1:5432/postgres
+      # A missing server fails the tests instead of skipping them.
+      SIMULATION_REQUIRE_DATABASE_TESTS: "1"
+
+    steps:
+    - uses: actions/checkout@v6
+
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: '3.13'
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v8.1.0
+      with:
+        version: "0.13.6"
+        enable-cache: true
+
+    - name: Install dependencies
+      working-directory: libs/policyengine-simulation-persistence
+      run: uv sync --extra test --frozen
+
+    - name: Run the persistence tests on PostgreSQL
+      working-directory: libs/policyengine-simulation-persistence
+      run: uv run --frozen pytest tests/ -v -m postgres
+
   lint:
     name: Lint
     # Full CI runs only on the lowest unmerged pull request in a stack.
@@ -164,6 +215,7 @@ jobs:
         package:
           - libs/policyengine-simulation-contract
           - libs/policyengine-simulation-observability
+          - libs/policyengine-simulation-persistence
           - libs/policyengine-stage12-persistence
           - libs/policyengine-fastapi
           - projects/policyengine-simulation-entry

.github/workflows/simulation-deploy.reusable.yml

@@ -241,6 +241,7 @@ jobs:
           OLD_GATEWAY_AUTH_AUDIENCE_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_AUDIENCE }}
           OLD_GATEWAY_AUTH_CLIENT_ID_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_CLIENT_ID }}
           OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME: ${{ vars.OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME }}
+          SIMULATION_SERVICE_DATABASE_URL_SECRET_NAME: ${{ vars.SIMULATION_SERVICE_DATABASE_URL_SECRET_NAME }}
           STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }}
           STAGE12_ARTIFACT_BUCKET_VALUE: ${{ vars.STAGE12_ARTIFACT_BUCKET }}
           STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }}
@@ -734,6 +735,8 @@ jobs:
           EXPECTED_UK_DATASET_SHA256: ${{ inputs.deploy_existing_stack && needs.prepare.outputs.uk_dataset_sha256 || '' }}
           # Latest must route to this release only when it should advance it.
           EXPECTED_LATEST: ${{ inputs.deploy_existing_stack && needs.update_routing.outputs.advances_latest == 'true' && 'true' || '' }}
+          # The candidate reaches its database, and its role holds its grants.
+          REQUIRE_SIMULATION_DATABASE_ACCESS: "true"
         run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${{ needs.deploy_entrypoint.outputs.candidate_url }}" "${{ needs.deploy_entrypoint.outputs.revision }}"
 
       - name: Download the numerical release sentinel

AGENTS.md

@@ -91,13 +91,14 @@ same-repository draft PR.
 
 - The simulation service lives in `projects/policyengine-simulation-executor`.
 - API integration tests live in `projects/policyengine-apis-integ`.
-- PR CI runs simulation unit tests, Ruff format and lint checks, Pyright for
-  every package, quality guards (including the rules registry), Docker build,
-  and local integration tests. Two jobs run separately, each from its own
-  workflow, on every PR, stacked ones included: `PR description` checks that
-  every open PR's description starts with `Fixes #ISSUE_NUMBER` for an open
-  issue in this repository, and `Changelog fragment` checks that the PR adds a
-  changelog fragment relative to its own base branch.
+- PR CI runs simulation unit tests, the persistence tests that need
+  PostgreSQL, Ruff format and lint checks, Pyright for every package, quality
+  guards (including the rules registry), Docker build, and local integration
+  tests. Two jobs run separately, each from its own workflow, on every PR,
+  stacked ones included: `PR description` checks that every open PR's
+  description starts with `Fixes #ISSUE_NUMBER` for an open issue in this
+  repository, and `Changelog fragment` checks that the PR adds a changelog
+  fragment relative to its own base branch.
 - Every PR adds a changelog fragment, `changelog.d/<name>.<type>.md`, and never
   edits `CHANGELOG.md` to describe a change (rule
   `sim.process.changelog-fragment`). A merge to `main` doesn't deploy itself:
@@ -132,6 +133,10 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`.
 
 - Checked: `sim.stage12.deployment-scope`, `sim.stage12.deployment-uses-api-schema`, `sim.stage12.no-migration-chain`, `sim.stage12.no-production-user-associations`, `sim.stage12.no-schema-ddl`, `sim.stage12.persistence-typed-dml-only`, `sim.stage12.production-lock-has-db-runtime`
 
+### Simulation service database
+
+- Checked: `sim.persistence.missing-privileges-reported`
+
 ### Deployment pipeline
 
 - Checked: `sim.deploy.container-context-minimal`, `sim.deploy.docker-images-build`, `sim.deploy.gcloud-without-terraform`, `sim.deploy.modal-image-smoke`, `sim.deploy.numerical-sentinel-gate`, `sim.deploy.permission-enforcement-wired`, `sim.deploy.promotion-order-explicit`, `sim.deploy.release-commit-deploys-once`, `sim.deploy.release-identity-smoke`, `sim.deploy.revision-tags-valid`, `sim.deploy.routing-publication-journaled`, `sim.deploy.shell-scripts-valid`, `sim.deploy.smoke-tests-strict`, `sim.deploy.staging-promotes-tested-revision`, `sim.deploy.traffic-changes-exact`, `sim.deploy.workflow-shell-safety`

Makefile

@@ -91,6 +91,7 @@ test:
 		projects/policyengine-simulation-gateway \
 		libs/policyengine-simulation-contract \
 		libs/policyengine-simulation-observability \
+		libs/policyengine-simulation-persistence \
 		libs/policyengine-stage12-persistence \
 		libs/policyengine-fastapi; do \
 		echo "Testing $$proj..."; \

changelog.d/897.added.md

@@ -0,0 +1 @@
+The Simulation Entrypoint connects to the Public API's database with its own role, mounted as `SIMULATION_SERVICE_DATABASE_URL` on every deploy, and models the tables it writes there (simulations, reports, their runs, budget window calculations and household jobs), with their foreign keys to the Public API's own tables. `GET /ready` reports whether it reached the database and whether its role holds every privilege it needs, the service logs any it lacks, a check that reached the database is reused for 30 seconds, and the candidate check requires them all. Every runtime secret is now resolved in its own assignment, so one that cannot be resolved stops the deploy.

docs/engineering/skills/database-access.md

@@ -29,17 +29,21 @@ It builds on the Stage 12 rules:
 
 ## Who owns the schema
 
-- **Today**, policyengine-api's Alembic migrations own the only tables this
-  repository uses, the Stage 12 comparison tables
-  (`sim.stage12.no-migration-chain`). This repository's code reads and
-  writes their rows, and deployment validation compares their live schema
-  with its SQLAlchemy mappings; nothing here changes them.
-- **Once #847 merges**, this repository's own Alembic chain at `migrations/`
-  owns the `simulation` schema, and the Stage 12 tables stay with
-  policyengine-api.
-
-The rule holds either way: whichever chain owns a table, only that chain's
-migrations change it.
+policyengine-api's Alembic migrations own every table this repository uses,
+and this repository has no migration chain of its own
+(`sim.stage12.no-migration-chain`):
+
+- the Stage 12 comparison tables, whose live schema deployment validation
+  compares with the Stage 12 library's SQLAlchemy mappings;
+- the tables the simulation service writes (`simulations`, `reports`,
+  `simulation_runs`, `report_runs`, `budget_window_runs` and
+  `household_jobs`), modelled in `libs/policyengine-simulation-persistence`.
+  The service connects as its own role, which policyengine-api's privilege
+  manifest grants.
+
+This repository's code reads and writes their rows; nothing here changes
+them. A change to one of those tables starts in policyengine-api, and the
+models here follow it.
 
 ## What the guard checks
 

docs/engineering/skills/type-checking.md

@@ -11,9 +11,9 @@ Every uv package is type-checked with Pyright on every pull request, by the
 
 - `sim.quality.pyright-libraries-and-entry`, `sim.quality.pyright-gateway`,
   `sim.quality.pyright-executor` and `sim.quality.pyright-apis-integ`: the
-  contract, observability, Stage 12 persistence and fastapi libraries, the
-  simulation entry, the gateway, the executor and the generated-client
-  integration tests have no Pyright errors.
+  contract, observability, simulation persistence, Stage 12 persistence and
+  fastapi libraries, the simulation entry, the gateway, the executor and the
+  generated-client integration tests have no Pyright errors.
 - `sim.quality.every-package-type-checked`: a Pyright rule lists every uv
   package under `libs/` and `projects/`. When you add a package, give it a
   `[tool.pyright] include`, add it to a Pyright rule's `enforcer.projects`, and