diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28d701f..e5f405c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,9 @@ jobs: run: | scripts/test-deploy-classifier.sh scripts/test-deploy.sh + scripts/test-image-deploy.sh + scripts/test-production-repair.sh + scripts/test-deploy-workflow.sh scripts/test-backup-postgres.sh - name: Validate CI branch coverage diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..f502394 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,207 @@ +name: Production deploy + +on: + workflow_run: + workflows: [CI] + branches: [master] + types: [completed] + workflow_dispatch: + inputs: + ref: + description: Commit, branch, or tag to deploy. Defaults to master. + required: false + type: string + +permissions: + contents: read + packages: write + +concurrency: + group: production-deploy + cancel-in-progress: false + +jobs: + resolve: + name: Resolve release source + if: >- + github.event_name == 'workflow_dispatch' + || ( + github.event.workflow_run.conclusion == 'success' + && github.event.workflow_run.head_branch == 'master' + ) + runs-on: ubuntu-24.04 + outputs: + source_sha: ${{ steps.source.outputs.source_sha }} + steps: + - name: Check out release source + uses: actions/checkout@v4 + with: + ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || inputs.ref || 'master' }} + fetch-depth: 0 + + - name: Resolve exact commit + id: source + run: echo "source_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + + image: + name: Build immutable image + needs: resolve + runs-on: ubuntu-24.04 + outputs: + image_ref: ${{ steps.image.outputs.image_ref }} + steps: + - name: Check out exact release source + uses: actions/checkout@v4 + with: + ref: ${{ needs.resolve.outputs.source_sha }} + + - uses: docker/setup-buildx-action@v3 + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Build and publish release image with retries + id: build + env: + IMAGE_TAG: ghcr.io/${{ github.repository }}:${{ needs.resolve.outputs.source_sha }} + SOURCE_SHA: ${{ needs.resolve.outputs.source_sha }} + run: | + for attempt in 1 2 3; do + if docker buildx build . \ + --build-arg "SOURCE_SHA=$SOURCE_SHA" \ + --cache-from type=gha,scope=production-image \ + --cache-to type=gha,mode=max,scope=production-image \ + --metadata-file /tmp/build-metadata.json \ + --push \ + --tag "$IMAGE_TAG"; then + digest=$(jq -r '."containerimage.digest" // empty' /tmp/build-metadata.json) + [[ "$digest" =~ ^sha256:[0-9a-f]{64}$ ]] + echo "digest=$digest" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [ "$attempt" -lt 3 ]; then + echo "Image build attempt $attempt failed. Retrying in 30 seconds." >&2 + sleep 30 + fi + done + exit 1 + + - name: Record digest-qualified image + id: image + env: + IMAGE_REPOSITORY: ghcr.io/${{ github.repository }} + IMAGE_DIGEST: ${{ steps.build.outputs.digest }} + run: | + [[ "$IMAGE_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] + echo "image_ref=$IMAGE_REPOSITORY@$IMAGE_DIGEST" >> "$GITHUB_OUTPUT" + + deploy: + name: Activate production release + needs: [resolve, image] + runs-on: ubuntu-24.04 + timeout-minutes: 30 + environment: production + env: + APP_DIR: /opt/letscube/runtime + ENV_FILE: /home/www/letscube/.env.prod + PRIVACY_EMAIL_CUTOVER_FILE: /home/www/letscube/.privacy-email-cutover + SERVER_IP: ${{ secrets.SERVER_IP }} + SERVER_USER: www + SOURCE_SHA: ${{ needs.resolve.outputs.source_sha }} + IMAGE_REF: ${{ needs.image.outputs.image_ref }} + steps: + - name: Check out exact release source + uses: actions/checkout@v4 + with: + ref: ${{ needs.resolve.outputs.source_sha }} + fetch-depth: 0 + + - name: Configure deployment SSH + env: + DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }} + DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }} + run: | + install -m 700 -d "$HOME/.ssh" + printf '%s\n' "$DEPLOY_SSH_KEY" > "$HOME/.ssh/id_ed25519" + printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > "$HOME/.ssh/known_hosts" + chmod 600 "$HOME/.ssh/id_ed25519" "$HOME/.ssh/known_hosts" + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Pull and validate the immutable image + run: | + docker pull "$IMAGE_REF" + test "$(docker image inspect "$IMAGE_REF" --format '{{index .RepoDigests 0}}')" = "$IMAGE_REF" + test "$(docker image inspect "$IMAGE_REF" --format '{{index .Config.Labels "org.opencontainers.image.revision"}}')" = "$SOURCE_SHA" + docker tag "$IMAGE_REF" "letscube-app:$SOURCE_SHA" + + - name: Enforce the production privacy floor + run: | + cutover=$(ssh -q "$SERVER_USER@$SERVER_IP" \ + "test ! -f '$PRIVACY_EMAIL_CUTOVER_FILE' || cat '$PRIVACY_EMAIL_CUTOVER_FILE'") + if [ -n "$cutover" ]; then + [[ "$cutover" =~ ^[0-9a-f]{40}$ ]] + git merge-base --is-ancestor "$cutover" "$SOURCE_SHA" + fi + + - name: Stage release configuration + run: | + remote_release_dir="/tmp/letscube-release-$SOURCE_SHA" + ssh -q "$SERVER_USER@$SERVER_IP" "mkdir -p '$remote_release_dir'" + scp -q compose.yml compose.prod.yml nginx.conf \ + scripts/deploy-image-release.sh scripts/check-production-health.sh \ + scripts/repair-production-services.sh \ + scripts/disable-legacy-deploy-cron.sh \ + "$SERVER_USER@$SERVER_IP:$remote_release_dir/" + + - name: Transfer the release image + run: | + set -o pipefail + for attempt in 1 2 3; do + if docker save "letscube-app:$SOURCE_SHA" | gzip | \ + ssh -q "$SERVER_USER@$SERVER_IP" 'gzip -d | docker load'; then + exit 0 + fi + + if [ "$attempt" -lt 3 ]; then + echo "Image transfer attempt $attempt failed. Retrying in 30 seconds." >&2 + sleep 30 + fi + done + exit 1 + + - name: Deploy with bounded automatic retries + run: | + remote_release_dir="/tmp/letscube-release-$SOURCE_SHA" + for attempt in 1 2 3; do + if ssh -q "$SERVER_USER@$SERVER_IP" \ + "APP_DIR='$APP_DIR' ENV_FILE='$ENV_FILE' bash '$remote_release_dir/deploy-image-release.sh' '$SOURCE_SHA' '$remote_release_dir'"; then + exit 0 + fi + + if [ "$attempt" -lt 3 ]; then + echo "Deployment attempt $attempt failed. Retrying the same image in 30 seconds." >&2 + sleep 30 + fi + done + exit 1 + + - name: Replace the legacy deployment cron with health repair + run: | + remote_release_dir="/tmp/letscube-release-$SOURCE_SHA" + ssh -q "$SERVER_USER@$SERVER_IP" \ + "APP_DIR='$APP_DIR' ENV_FILE='$ENV_FILE' bash '$remote_release_dir/disable-legacy-deploy-cron.sh'" + + - name: Remove staged release files + if: always() + run: | + remote_release_dir="/tmp/letscube-release-$SOURCE_SHA" + ssh -q "$SERVER_USER@$SERVER_IP" "rm -rf -- '$remote_release_dir'" || true diff --git a/Dockerfile b/Dockerfile index e7bdecb..9f51745 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,10 @@ FROM node:22-bookworm-slim AS client-build WORKDIR /app +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates openssl \ + && rm -rf /var/lib/apt/lists/* + RUN corepack enable COPY package.json yarn.lock ./ @@ -27,7 +31,7 @@ RUN yarn workspace letscube-client build FROM node:22-bookworm-slim AS server-deps RUN apt-get update \ - && apt-get install -y --no-install-recommends python3 make g++ \ + && apt-get install -y --no-install-recommends ca-certificates openssl python3 make g++ \ && rm -rf /var/lib/apt/lists/* WORKDIR /app @@ -42,6 +46,8 @@ RUN yarn install --frozen-lockfile --production=true --network-concurrency 1 --n && mkdir -p server/node_modules FROM node:22-bookworm-slim AS runtime +ARG SOURCE_SHA=local +LABEL org.opencontainers.image.revision=${SOURCE_SHA} ENV NODE_ENV=prod ENV PORT=8080 ENV SOCKETIO_PORT=9000 diff --git a/compose.prod.yml b/compose.prod.yml index 2d6b0d5..65a3caf 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -48,5 +48,12 @@ services: - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro - ${LETSENCRYPT_DIR:-/etc/letsencrypt}:/etc/letsencrypt:ro - ${CERTBOT_WWW_DIR:-/var/www/certbot}:/var/www/certbot:ro + healthcheck: + test: ["CMD-SHELL", "wget --quiet --spider --no-check-certificate https://127.0.0.1/health/api || exit 1"] + interval: 30s + start_interval: 2s + timeout: 5s + retries: 5 + start_period: 10s logging: *production-logging restart: unless-stopped diff --git a/docs/operations.md b/docs/operations.md index a3e1a84..909290f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -7,15 +7,50 @@ backfill tool. ## Deployment -Use the Compose deploy script from the active checkout: +Production deploys run through the `Production deploy` GitHub Actions workflow. +The workflow starts after a successful `CI` run on `master`. It performs these +steps: + +1. Build one application image from the exact `master` commit. +2. Push the image to GHCR and retain its immutable digest. +3. Transfer that exact image and the production Compose files to the server. +4. Apply migrations, replace Socket.IO before the API, and verify all five + production services. +5. Retry image builds, transfers, and activation up to three times. Each failed + activation restores the previous application images and Compose files before + the next attempt. + +The server does not fetch or merge Git commits during this process. A manual +workflow run can deploy a specific commit, branch, or tag. The workflow uses the +GitHub `production` environment, so configure its approval policy to match the +release policy. + +The repository needs these GitHub Actions secrets: + +- `SERVER_IP`: the production server address. +- `DEPLOY_SSH_KEY`: the private key for the `www` account. +- `DEPLOY_KNOWN_HOSTS`: the pinned SSH host key entry. + +The server needs Docker access for `www`, the production `.env.prod` file at +`/home/www/letscube`, and the existing certificate directories. Actions stores +runtime Compose files in `/opt/letscube/runtime`, outside the old Git checkout. +After the first successful image activation, it removes only the old +`scripts/deploy.sh` cron entry. It saves the prior crontab at +`/opt/letscube/legacy-deploy-crontab.backup`. The PostgreSQL backup cron remains +active. A replacement cron checks all five services every ten minutes. It +recreates only missing or unhealthy services from the last successful image. +It does not fetch source code. + +The checkout-based script remains available as a temporary recovery path: ```sh -APP_DIR=/opt/letscube ENV_FILE=.env.prod scripts/deploy.sh +APP_DIR=/home/www/letscube ENV_FILE=.env.prod scripts/deploy.sh ``` -The script builds an immutable image, starts PostgreSQL and Redis, applies -committed Prisma migrations, then replaces and health-checks Socket.IO and the -API. It does not start or depend on MongoDB. +It records the last successful commit separately from the checkout. A failed +deployment therefore retries on the next run even when Git already moved to the +new commit. It also treats a missing or unhealthy PostgreSQL, Redis, Socket.IO, +API, or nginx container as a full repair deployment. Before a big-bang cutover, take a PostgreSQL backup and run the MongoDB backfill tool during a maintenance window: @@ -34,6 +69,9 @@ not copy sessions or OAuth tokens; users authenticate again after cutover. ```sh docker compose -f compose.yml -f compose.prod.yml --env-file .env.prod ps +APP_DIR=/opt/letscube/runtime \ + ENV_FILE=/home/www/letscube/.env.prod \ + /opt/letscube/runtime/scripts/check-production-health.sh docker compose -f compose.yml -f compose.prod.yml --env-file .env.prod logs --tail=200 api socket postgres curl -fsS https://letscube.net/health/api curl -fsS https://letscube.net/health/socket @@ -42,6 +80,10 @@ curl -fsS https://letscube.net/health/socket Both health endpoints require PostgreSQL. The socket endpoint also requires Redis. A failed required dependency returns HTTP 503. +nginx also has a container health check. It calls the API health route through +the local TLS listener. This detects a broken proxy configuration and an +unavailable API upstream. + ## Backups and restore Create a PostgreSQL custom-format backup and test restores against a separate diff --git a/docs/release-workflow.md b/docs/release-workflow.md index e0cc89a..687c011 100644 --- a/docs/release-workflow.md +++ b/docs/release-workflow.md @@ -1,8 +1,8 @@ # Release Workflow `dev` is the integration branch. `master` is the reviewed release branch. -Production deployment remains a separately authorized operation described in -[Production operations](operations.md). +Production deployment runs through the protected GitHub `production` +environment described in [Production operations](operations.md). ## Branch Flow @@ -11,8 +11,8 @@ Production deployment remains a separately authorized operation described in 3. Treat the resulting `dev` merge commit as the release candidate. Its push CI and CodeQL push runs must finish before promotion is considered. 4. Open a reviewed promotion from that exact `dev` commit to `master`. -5. After the promotion is merged and its `master` push CI passes, perform the - separately authorized deployment procedure. +5. After the promotion is merged, the successful `master` CI run starts the + production deployment workflow. Do not bypass `dev` by merging feature work directly to `master`. A new commit on `dev` after release-candidate validation requires a new validation run. @@ -42,10 +42,11 @@ explicit operational authorization required by the operations guide. ## Deployment Handoff -After `master` CI is green, follow the operations guide in order: +After `master` CI is green, GitHub Actions follows the operations guide in +order: 1. complete the [preflight](operations.md#preflight) checks; -2. deploy with the documented procedure; +2. build and deploy the immutable release image; 3. perform the [verification and monitoring](operations.md#verification-and-monitoring) checks; and 4. use the documented rollback procedure if readiness or verification fails. diff --git a/scripts/check-production-health.sh b/scripts/check-production-health.sh new file mode 100755 index 0000000..bf5ab68 --- /dev/null +++ b/scripts/check-production-health.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +set -euo pipefail + +APP_DIR="${APP_DIR:-/opt/letscube/runtime}" +ENV_FILE="${ENV_FILE:-/home/www/letscube/.env.prod}" +HEALTH_ATTEMPTS="${HEALTH_ATTEMPTS:-10}" +HEALTH_RETRY_SECONDS="${HEALTH_RETRY_SECONDS:-3}" + +if ! [[ "$HEALTH_ATTEMPTS" =~ ^[1-9][0-9]*$ ]]; then + echo "HEALTH_ATTEMPTS must be a positive integer." >&2 + exit 2 +fi + +if [[ "$ENV_FILE" = /* ]]; then + ENV_PATH="$ENV_FILE" +else + ENV_PATH="$APP_DIR/$ENV_FILE" +fi + +COMPOSE=( + docker compose + --project-directory "$APP_DIR" + -f "$APP_DIR/compose.yml" + -f "$APP_DIR/compose.prod.yml" + --env-file "$ENV_PATH" +) +services=(postgres redis socket api nginx) + +for ((attempt = 1; attempt <= HEALTH_ATTEMPTS; attempt += 1)); do + failed=() + + for service in "${services[@]}"; do + container_id="$("${COMPOSE[@]}" ps --status running -q "$service")" + if [ -z "$container_id" ]; then + failed+=("$service:missing") + continue + fi + + health_status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}' "$container_id")" + if [ "$health_status" != "healthy" ]; then + failed+=("$service:$health_status") + fi + done + + if [ "${#failed[@]}" -eq 0 ]; then + echo "All production services are healthy." + exit 0 + fi + + echo "Health check $attempt/$HEALTH_ATTEMPTS failed: ${failed[*]}" >&2 + if [ "$attempt" -lt "$HEALTH_ATTEMPTS" ]; then + sleep "$HEALTH_RETRY_SECONDS" + fi +done + +"${COMPOSE[@]}" ps >&2 || true +"${COMPOSE[@]}" logs --tail=120 api socket postgres redis nginx >&2 || true +exit 1 diff --git a/scripts/deploy-image-release.sh b/scripts/deploy-image-release.sh new file mode 100755 index 0000000..9764ced --- /dev/null +++ b/scripts/deploy-image-release.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +set -euo pipefail + +SOURCE_SHA="${1:-}" +RELEASE_DIR="${2:-}" +APP_DIR="${APP_DIR:-/opt/letscube/runtime}" +ENV_FILE="${ENV_FILE:-/home/www/letscube/.env.prod}" +DEPLOY_WAIT_TIMEOUT_SECONDS="${DEPLOY_WAIT_TIMEOUT_SECONDS:-90}" +LOCK_FILE="${LOCK_FILE:-/tmp/letscube-deploy.lock}" + +if ! [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]]; then + echo "A full source commit SHA is required." >&2 + exit 2 +fi +if [ -z "$RELEASE_DIR" ] || [ ! -d "$RELEASE_DIR" ]; then + echo "A release directory is required." >&2 + exit 2 +fi +if [[ "$ENV_FILE" = /* ]]; then + ENV_PATH="$ENV_FILE" +else + ENV_PATH="$APP_DIR/$ENV_FILE" +fi +if [ ! -f "$ENV_PATH" ]; then + echo "Missing production environment file: $ENV_PATH" >&2 + exit 1 +fi + +release_files=(compose.yml compose.prod.yml nginx.conf) +for file in "${release_files[@]}"; do + if [ ! -f "$RELEASE_DIR/$file" ]; then + echo "Missing release file: $RELEASE_DIR/$file" >&2 + exit 1 + fi +done +if [ ! -x "$RELEASE_DIR/check-production-health.sh" ]; then + echo "Missing executable production health check." >&2 + exit 1 +fi +if [ ! -x "$RELEASE_DIR/repair-production-services.sh" ]; then + echo "Missing executable production repair script." >&2 + exit 1 +fi + +image="letscube-app:$SOURCE_SHA" +docker image inspect "$image" > /dev/null +image_source="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image")" +if [ "$image_source" != "$SOURCE_SHA" ]; then + echo "Image source label does not match $SOURCE_SHA." >&2 + exit 1 +fi + +exec 9> "$LOCK_FILE" +if ! flock -w 60 9; then + echo "Another production deployment holds $LOCK_FILE." >&2 + exit 1 +fi + +mkdir -p "$APP_DIR/scripts" +rollback_dir="$(mktemp -d "${TMPDIR:-/tmp}/letscube-release-rollback.XXXXXX")" +config_staged=false +deployment_complete=false +declare -A rollback_tags=() + +COMPOSE=( + docker compose + --project-directory "$APP_DIR" + -f "$APP_DIR/compose.yml" + -f "$APP_DIR/compose.prod.yml" + --env-file "$ENV_PATH" +) +ROLLBACK_COMPOSE=() + +restore_release() { + local service + local rollback_tag + + set +e + echo "Restoring the previous production release." >&2 + + if [ "$config_staged" = "true" ]; then + for file in "${release_files[@]}"; do + if [ -f "$rollback_dir/$file" ]; then + cp -p "$rollback_dir/$file" "$APP_DIR/$file" + fi + done + fi + + if [ -f "$rollback_dir/compose.yml" ] && [ -f "$rollback_dir/compose.prod.yml" ]; then + ROLLBACK_COMPOSE=( + docker compose + --project-directory "$APP_DIR" + -f "$rollback_dir/compose.yml" + -f "$rollback_dir/compose.prod.yml" + --env-file "$ENV_PATH" + ) + + for service in socket api; do + rollback_tag="${rollback_tags[$service]-}" + if [ -n "$rollback_tag" ]; then + APP_IMAGE_TAG="$rollback_tag" "${ROLLBACK_COMPOSE[@]}" up -d --no-deps \ + --force-recreate --wait --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" "$service" + fi + done + + "${ROLLBACK_COMPOSE[@]}" up -d --no-deps --force-recreate --wait \ + --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" nginx + fi +} + +cleanup() { + local status=$? + + if [ "$status" -ne 0 ] && [ "$deployment_complete" != "true" ]; then + restore_release || true + fi + rm -rf -- "$rollback_dir" + exit "$status" +} +trap cleanup EXIT + +for file in "${release_files[@]}"; do + if [ -f "$APP_DIR/$file" ]; then + cp -p "$APP_DIR/$file" "$rollback_dir/$file" + fi +done + +deploy_id="$(date -u +%Y%m%dT%H%M%SZ)-$$" +for service in socket api; do + container_id="$("${COMPOSE[@]}" ps -q "$service" 2> /dev/null || true)" + if [ -n "$container_id" ]; then + image_id="$(docker inspect --format '{{.Image}}' "$container_id")" + rollback_tag="rollback-$service-${SOURCE_SHA:0:12}-$deploy_id" + docker image tag "$image_id" "letscube-app:$rollback_tag" + rollback_tags["$service"]="$rollback_tag" + fi +done + +for file in "${release_files[@]}"; do + install -m 0644 "$RELEASE_DIR/$file" "$APP_DIR/$file.new" + mv "$APP_DIR/$file.new" "$APP_DIR/$file" +done +install -m 0755 "$RELEASE_DIR/check-production-health.sh" \ + "$APP_DIR/scripts/check-production-health.sh" +install -m 0755 "$RELEASE_DIR/repair-production-services.sh" \ + "$APP_DIR/scripts/repair-production-services.sh" +config_staged=true + +export APP_IMAGE_TAG="$SOURCE_SHA" +"${COMPOSE[@]}" config --quiet +"${COMPOSE[@]}" up -d --wait --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" postgres redis +"${COMPOSE[@]}" run --rm --no-deps migrate + +# Socket.IO is replaced first so it accepts the browser client in the new API image. +"${COMPOSE[@]}" up -d --no-deps --force-recreate --wait \ + --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" socket +"${COMPOSE[@]}" up -d --no-deps --force-recreate --wait \ + --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" api +"${COMPOSE[@]}" up -d --no-deps --force-recreate --wait \ + --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" nginx + +APP_DIR="$APP_DIR" ENV_FILE="$ENV_FILE" \ + "$RELEASE_DIR/check-production-health.sh" + +printf '%s\n' "$SOURCE_SHA" > "$APP_DIR/.last-successful-deploy" +deployment_complete=true +echo "Deployment complete: all services use release $SOURCE_SHA." +"${COMPOSE[@]}" ps + +docker image prune -af --filter "until=168h" || \ + echo "Deployment succeeded, but unused image cleanup failed." >&2 diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 9a1539b..e123cee 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -9,6 +9,7 @@ DEPLOY_STRATEGY="${DEPLOY_STRATEGY:-ff-only}" DEPLOY_FORCE="${DEPLOY_FORCE:-false}" DEPLOY_TARGET="${DEPLOY_TARGET:-auto}" DEPLOY_WAIT_TIMEOUT_SECONDS="${DEPLOY_WAIT_TIMEOUT_SECONDS:-60}" +LAST_SUCCESSFUL_DEPLOY_FILE="${LAST_SUCCESSFUL_DEPLOY_FILE:-$APP_DIR/.last-successful-deploy}" COMPOSE=(docker compose -f compose.yml -f compose.prod.yml --env-file "$ENV_FILE") ROLLBACK_COMPOSE=() @@ -39,7 +40,18 @@ fi cd "$APP_DIR" -previous_commit="$(git rev-parse HEAD)" +checkout_commit="$(git rev-parse HEAD)" +if [ -f "$LAST_SUCCESSFUL_DEPLOY_FILE" ]; then + previous_commit="$(tr -d '[:space:]' < "$LAST_SUCCESSFUL_DEPLOY_FILE")" + if ! [[ "$previous_commit" =~ ^[0-9a-f]{40}$ ]]; then + echo "Invalid commit in $LAST_SUCCESSFUL_DEPLOY_FILE" >&2 + exit 2 + fi +else + previous_commit="$checkout_commit" + printf '%s\n' "$previous_commit" > "$LAST_SUCCESSFUL_DEPLOY_FILE" +fi + git fetch "$REMOTE" "$BRANCH" case "$DEPLOY_STRATEGY" in @@ -87,20 +99,44 @@ if [ -f "$privacy_cutover_file" ]; then fi fi +unhealthy_backing_services=() +unhealthy_runtime_services=() if [ "$DEPLOY_TARGET" = "auto" ]; then missing_services=() - for service in api socket nginx; do + unhealthy_services=() + for service in postgres redis api socket nginx; do if ! container_id="$("${COMPOSE[@]}" ps --status running -q "$service")"; then echo "Failed to inspect the $service container." >&2 exit 1 fi if [ -z "$container_id" ]; then missing_services+=("$service") + if [ "$service" = "postgres" ] || [ "$service" = "redis" ]; then + unhealthy_backing_services+=("$service") + else + unhealthy_runtime_services+=("$service") + fi + continue + fi + + health_status="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}' "$container_id")" + if [ "$health_status" != "healthy" ]; then + unhealthy_services+=("$service:$health_status") + if [ "$service" = "postgres" ] || [ "$service" = "redis" ]; then + unhealthy_backing_services+=("$service") + else + unhealthy_runtime_services+=("$service") + fi fi done - if [ "${#missing_services[@]}" -gt 0 ]; then - echo "Missing runtime services: ${missing_services[*]}; deploying all." + if [ "${#missing_services[@]}" -gt 0 ] || [ "${#unhealthy_services[@]}" -gt 0 ]; then + if [ "${#missing_services[@]}" -gt 0 ]; then + echo "Missing production services: ${missing_services[*]}; deploying all." + fi + if [ "${#unhealthy_services[@]}" -gt 0 ]; then + echo "Unhealthy production services: ${unhealthy_services[*]}; deploying all." + fi resolved_target="all" elif [ "$previous_commit" = "$current_commit" ]; then if [ "$DEPLOY_FORCE" != "true" ]; then @@ -135,6 +171,7 @@ case "$resolved_target" in ;; none) echo "Updated checkout to $current_commit; no application services selected." + printf '%s\n' "$current_commit" > "$LAST_SUCCESSFUL_DEPLOY_FILE" "${COMPOSE[@]}" ps exit 0 ;; @@ -182,9 +219,13 @@ fi # Compose usage compatible with the default `local` tag. export APP_IMAGE_TAG="$current_commit" -compose_recreate_args=() +backing_recreate_args=() +runtime_recreate_args=() if [ "$production_compose_changed" = "true" ]; then - compose_recreate_args=(--force-recreate) + backing_recreate_args=(--force-recreate) + runtime_recreate_args=(--force-recreate) +elif [ "${#unhealthy_runtime_services[@]}" -gt 0 ]; then + runtime_recreate_args=(--force-recreate) fi if ! "${COMPOSE[@]}" build api; then @@ -193,7 +234,16 @@ if ! "${COMPOSE[@]}" build api; then exit 1 fi -if ! "${COMPOSE[@]}" up -d "${compose_recreate_args[@]}" --wait \ +for service in "${unhealthy_backing_services[@]}"; do + if ! "${COMPOSE[@]}" up -d --force-recreate --wait \ + --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" "$service"; then + echo "Failed to repair unhealthy $service; no application containers were replaced." >&2 + print_retry_hint + exit 1 + fi +done + +if ! "${COMPOSE[@]}" up -d "${backing_recreate_args[@]}" --wait \ --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" postgres redis; then echo "Failed to start backing services; no application containers were replaced." >&2 print_retry_hint @@ -241,7 +291,7 @@ reload_nginx() { echo "nginx is not running; starting it without replacing dependencies." fi - "${compose_command[@]}" up -d --no-deps "${compose_recreate_args[@]}" --wait \ + "${compose_command[@]}" up -d --no-deps "${runtime_recreate_args[@]}" --wait \ --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" nginx || return 1 "${compose_command[@]}" exec -T nginx nginx -s reload @@ -306,7 +356,7 @@ for service_index in "${!selected_services[@]}"; do service="${selected_services[$service_index]}" attempted_services+=("$service") - if ! "${COMPOSE[@]}" up -d --no-deps --wait \ + if ! "${COMPOSE[@]}" up -d --no-deps "${runtime_recreate_args[@]}" --wait \ --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" "$service"; then echo "$service deployment failed. Recent logs:" >&2 "${COMPOSE[@]}" logs --tail=120 "$service" >&2 || true @@ -336,6 +386,7 @@ for service_index in "${!selected_services[@]}"; do done echo "Deployment complete: $resolved_target now uses letscube-app:$current_commit" +printf '%s\n' "$current_commit" > "$LAST_SUCCESSFUL_DEPLOY_FILE" "${COMPOSE[@]}" ps if ! docker image prune -af --filter "until=168h"; then diff --git a/scripts/disable-legacy-deploy-cron.sh b/scripts/disable-legacy-deploy-cron.sh new file mode 100755 index 0000000..c423b4d --- /dev/null +++ b/scripts/disable-legacy-deploy-cron.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +BACKUP_FILE="${BACKUP_FILE:-/opt/letscube/legacy-deploy-crontab.backup}" +APP_DIR="${APP_DIR:-/opt/letscube/runtime}" +ENV_FILE="${ENV_FILE:-/home/www/letscube/.env.prod}" +temporary_dir="$(mktemp -d "${TMPDIR:-/tmp}/letscube-crontab.XXXXXX")" +marker='# letscube production health repair' +repair_entry="*/10 * * * * APP_DIR=$APP_DIR ENV_FILE=$ENV_FILE $APP_DIR/scripts/repair-production-services.sh 2>&1 | logger -t letscube-repair" + +cleanup() { + rm -rf -- "$temporary_dir" +} +trap cleanup EXIT + +current="$temporary_dir/current" +filtered="$temporary_dir/filtered" +crontab -l > "$current" 2> /dev/null || : + +if ! grep -Fq 'scripts/deploy.sh' "$current" \ + && grep -Fqx "$repair_entry" "$current"; then + echo "Production health repair cron is already configured." + exit 0 +fi + +if [ ! -f "$BACKUP_FILE" ]; then + install -m 0600 "$current" "$BACKUP_FILE" +fi + +grep -Fv 'scripts/deploy.sh' "$current" \ + | grep -Fv "$marker" \ + | grep -Fv 'scripts/repair-production-services.sh' > "$filtered" || : +printf '%s\n%s\n' "$marker" "$repair_entry" >> "$filtered" +crontab "$filtered" +echo "Replaced the legacy deploy cron with image-safe health repair. Backup: $BACKUP_FILE" diff --git a/scripts/repair-production-services.sh b/scripts/repair-production-services.sh new file mode 100755 index 0000000..0f44b74 --- /dev/null +++ b/scripts/repair-production-services.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +set -euo pipefail + +APP_DIR="${APP_DIR:-/opt/letscube/runtime}" +ENV_FILE="${ENV_FILE:-/home/www/letscube/.env.prod}" +DEPLOY_WAIT_TIMEOUT_SECONDS="${DEPLOY_WAIT_TIMEOUT_SECONDS:-90}" +LOCK_FILE="${LOCK_FILE:-/tmp/letscube-deploy.lock}" + +if [[ "$ENV_FILE" = /* ]]; then + ENV_PATH="$ENV_FILE" +else + ENV_PATH="$APP_DIR/$ENV_FILE" +fi + +COMPOSE=( + docker compose + --project-directory "$APP_DIR" + -f "$APP_DIR/compose.yml" + -f "$APP_DIR/compose.prod.yml" + --env-file "$ENV_PATH" +) + +exec 9> "$LOCK_FILE" +if ! flock -n 9; then + exit 0 +fi + +release_file="$APP_DIR/.last-successful-deploy" +if [ ! -f "$release_file" ]; then + echo "Cannot repair production before the first successful image deployment." >&2 + exit 1 +fi + +source_sha="$(tr -d '[:space:]' < "$release_file")" +if ! [[ "$source_sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "Invalid release in $release_file" >&2 + exit 1 +fi + +image="letscube-app:$source_sha" +image_source="$(docker image inspect \ + --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$image")" +if [ "$image_source" != "$source_sha" ]; then + echo "Release image $image does not match its source label." >&2 + exit 1 +fi + +service_health() { + local service="$1" + local container_id + + container_id="$("${COMPOSE[@]}" ps --status running -q "$service")" + if [ -z "$container_id" ]; then + printf 'missing\n' + return + fi + + docker inspect \ + --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}missing{{end}}' \ + "$container_id" +} + +services=(postgres redis socket api nginx) +unhealthy=() +for service in "${services[@]}"; do + status="$(service_health "$service")" + if [ "$status" != healthy ]; then + unhealthy+=("$service:$status") + fi +done + +if [ "${#unhealthy[@]}" -eq 0 ]; then + exit 0 +fi + +echo "Repairing unhealthy production services: ${unhealthy[*]}" +export APP_IMAGE_TAG="$source_sha" +"${COMPOSE[@]}" config --quiet + +for service in "${services[@]}"; do + if [ "$(service_health "$service")" = healthy ]; then + continue + fi + + "${COMPOSE[@]}" up -d --no-deps --force-recreate --wait \ + --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" "$service" +done + +APP_DIR="$APP_DIR" ENV_FILE="$ENV_FILE" \ + "$APP_DIR/scripts/check-production-health.sh" diff --git a/scripts/test-deploy-workflow.sh b/scripts/test-deploy-workflow.sh new file mode 100755 index 0000000..5195651 --- /dev/null +++ b/scripts/test-deploy-workflow.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd -- "$SCRIPT_DIR/.." && pwd)" +WORKFLOW="$ROOT_DIR/.github/workflows/deploy.yml" +IMAGE_DEPLOY="$SCRIPT_DIR/deploy-image-release.sh" +HEALTH_CHECK="$SCRIPT_DIR/check-production-health.sh" + +assert_contains() { + local file="$1" + local expected="$2" + + if ! grep -Fq -- "$expected" "$file"; then + echo "$file must contain: $expected" >&2 + exit 1 + fi +} + +assert_contains "$WORKFLOW" 'workflows: [CI]' +assert_contains "$WORKFLOW" 'branches: [master]' +assert_contains "$WORKFLOW" "github.event.workflow_run.conclusion == 'success'" +assert_contains "$WORKFLOW" 'image_ref=$IMAGE_REPOSITORY@$IMAGE_DIGEST' +assert_contains "$WORKFLOW" 'for attempt in 1 2 3; do' +assert_contains "$WORKFLOW" '--cache-to type=gha,mode=max,scope=production-image' +assert_contains "$WORKFLOW" 'Image build attempt $attempt failed.' +assert_contains "$WORKFLOW" 'Image transfer attempt $attempt failed.' +assert_contains "$WORKFLOW" 'docker save "letscube-app:$SOURCE_SHA"' +assert_contains "$WORKFLOW" 'scripts/deploy-image-release.sh scripts/check-production-health.sh' +assert_contains "$WORKFLOW" 'scripts/repair-production-services.sh' +assert_contains "$WORKFLOW" 'scripts/disable-legacy-deploy-cron.sh' +assert_contains "$WORKFLOW" 'APP_DIR: /opt/letscube/runtime' +assert_contains "$WORKFLOW" 'ENV_FILE: /home/www/letscube/.env.prod' + +assert_contains "$IMAGE_DEPLOY" 'docker image inspect "$image"' +assert_contains "$IMAGE_DEPLOY" 'org.opencontainers.image.revision' +assert_contains "$IMAGE_DEPLOY" 'run --rm --no-deps migrate' +assert_contains "$IMAGE_DEPLOY" 'printf '\''%s\n'\'' "$SOURCE_SHA" > "$APP_DIR/.last-successful-deploy"' +assert_contains "$HEALTH_CHECK" 'services=(postgres redis socket api nginx)' + +if grep -Eq 'git (pull|fetch|merge)' "$IMAGE_DEPLOY"; then + echo 'Image deployment must not update a production Git checkout.' >&2 + exit 1 +fi + +bash -n "$IMAGE_DEPLOY" "$HEALTH_CHECK" +echo 'GitHub Actions deployment checks passed.' diff --git a/scripts/test-deploy.sh b/scripts/test-deploy.sh index 342794c..97e5754 100755 --- a/scripts/test-deploy.sh +++ b/scripts/test-deploy.sh @@ -55,7 +55,15 @@ fake_docker() { case "$1" in inspect) - printf 'sha256:old-image\n' + if [[ "$*" == *'.State.Health'* ]]; then + if [ "${FAKE_UNHEALTHY_CONTAINER:-}" = "${*: -1}" ]; then + printf 'unhealthy\n' + else + printf 'healthy\n' + fi + else + printf 'sha256:old-image\n' + fi return 0 ;; builder|image) @@ -100,6 +108,12 @@ fake_docker() { nginx) [ -z "${FAKE_RUNNING_NGINX:-}" ] || printf '%s\n' "$FAKE_RUNNING_NGINX" ;; + postgres) + [ -z "${FAKE_RUNNING_POSTGRES:-}" ] || printf '%s\n' "$FAKE_RUNNING_POSTGRES" + ;; + redis) + [ -z "${FAKE_RUNNING_REDIS:-}" ] || printf '%s\n' "$FAKE_RUNNING_REDIS" + ;; esac return 0 fi @@ -207,13 +221,17 @@ run_bootstrap_test() { TMPDIR="$scenario/tmp" \ bash "$app_dir/scripts/deploy.sh" > "$output" 2>&1 - assert_contains "$(<"$output")" 'Missing runtime services: api socket nginx; deploying all.' + assert_contains "$(<"$output")" \ + 'Missing production services: postgres redis api socket nginx; deploying all.' assert_contains "$(<"$output")" '(target: all)' - backing_line="$(command_line "$log" 'up -d --wait --wait-timeout 17 postgres redis')" + backing_line="$(command_line "$log" \ + 'up -d --force-recreate --wait --wait-timeout 17 postgres')" migration_line="$(command_line "$log" 'run --rm --no-deps migrate')" - socket_line="$(command_line "$log" 'up -d --no-deps --wait --wait-timeout 17 socket')" - api_line="$(command_line "$log" 'up -d --no-deps --wait --wait-timeout 17 api')" + socket_line="$(command_line "$log" \ + 'up -d --no-deps --force-recreate --wait --wait-timeout 17 socket')" + api_line="$(command_line "$log" \ + 'up -d --no-deps --force-recreate --wait --wait-timeout 17 api')" if [ "$backing_line" -ge "$migration_line" ]; then echo 'Migrations ran before backing services became ready.' >&2 @@ -309,10 +327,77 @@ run_rollback_test() { assert_contains "$(<"$log")" "git show $previous_commit:compose.yml" assert_contains "$(<"$log")" "git show $previous_commit:compose.prod.yml" + if [ "$(<"$app_dir/.last-successful-deploy")" != "$previous_commit" ]; then + echo 'Failed deployment changed the last successful commit.' >&2 + exit 1 + fi if grep -Eq 'docker (image|builder) prune' "$log"; then echo 'Failed deployment removed Docker data.' >&2 exit 1 fi + + PATH="$FAKE_BIN:$PATH" \ + APP_DIR="$app_dir" \ + DEPLOY_TEST_LOG="$log" \ + DEPLOY_WAIT_TIMEOUT_SECONDS=17 \ + ENV_FILE=.env.prod \ + FAKE_CHANGED_PATH=server/index.js \ + FAKE_CURRENT_COMMIT="$current_commit" \ + FAKE_GIT_STATE="$git_state" \ + FAKE_PREVIOUS_COMMIT="$previous_commit" \ + FAKE_ROLLBACK_MARKER="$rollback_marker" \ + FAKE_RUNNING_API=api-container \ + FAKE_RUNNING_NGINX=nginx-container \ + FAKE_RUNNING_POSTGRES=postgres-container \ + FAKE_RUNNING_REDIS=redis-container \ + FAKE_RUNNING_SOCKET=socket-container \ + TMPDIR="$scenario/tmp" \ + bash "$app_dir/scripts/deploy.sh" >> "$output" 2>&1 + + if [ "$(<"$app_dir/.last-successful-deploy")" != "$current_commit" ]; then + echo 'Successful retry did not record the deployed commit.' >&2 + exit 1 + fi + +} + +run_unhealthy_service_test() { + local scenario="$TEST_ROOT/unhealthy" + local app_dir="$scenario/app" + local log="$scenario/commands.log" + local output="$scenario/output.log" + local git_state="$scenario/git-state" + local rollback_marker="$scenario/rollback" + local commit="3333333333333333333333333333333333333333" + + mkdir -p "$scenario/tmp" + prepare_app "$app_dir" + + PATH="$FAKE_BIN:$PATH" \ + APP_DIR="$app_dir" \ + DEPLOY_TEST_LOG="$log" \ + DEPLOY_WAIT_TIMEOUT_SECONDS=17 \ + ENV_FILE=.env.prod \ + FAKE_CURRENT_COMMIT="$commit" \ + FAKE_GIT_STATE="$git_state" \ + FAKE_PREVIOUS_COMMIT="$commit" \ + FAKE_ROLLBACK_MARKER="$rollback_marker" \ + FAKE_RUNNING_API=api-container \ + FAKE_RUNNING_NGINX=nginx-container \ + FAKE_RUNNING_POSTGRES=postgres-container \ + FAKE_RUNNING_REDIS=redis-container \ + FAKE_RUNNING_SOCKET=socket-container \ + FAKE_UNHEALTHY_CONTAINER=api-container \ + TMPDIR="$scenario/tmp" \ + bash "$app_dir/scripts/deploy.sh" > "$output" 2>&1 + + assert_contains "$(<"$output")" 'Unhealthy production services: api:unhealthy; deploying all.' + assert_contains "$(<"$log")" \ + 'up -d --no-deps --force-recreate --wait --wait-timeout 17 socket' + if grep -Fq 'up -d --force-recreate --wait --wait-timeout 17 postgres redis' "$log"; then + echo 'An unhealthy API caused healthy backing services to be recreated.' >&2 + exit 1 + fi } run_privacy_floor_test() { @@ -354,6 +439,7 @@ run_privacy_floor_test() { run_bootstrap_test run_compose_change_test run_rollback_test +run_unhealthy_service_test run_privacy_floor_test echo 'Deploy script checks passed.' diff --git a/scripts/test-image-deploy.sh b/scripts/test-image-deploy.sh new file mode 100755 index 0000000..d42d059 --- /dev/null +++ b/scripts/test-image-deploy.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +set -euo pipefail + +log_command() { + { + printf 'APP_IMAGE_TAG=%s docker' "${APP_IMAGE_TAG:-}" + printf ' %q' "$@" + printf '\n' + } >> "$IMAGE_DEPLOY_TEST_LOG" +} + +fake_docker() { + log_command "$@" + + if [ "$1" = image ]; then + case "$2" in + inspect) + if [[ "$*" == *org.opencontainers.image.revision* ]]; then + printf '%s\n' "$IMAGE_DEPLOY_TEST_SHA" + fi + return 0 + ;; + tag|prune) return 0 ;; + esac + fi + + if [ "$1" = inspect ]; then + if [[ "$*" == *org.opencontainers.image.revision* ]]; then + printf '%s\n' "$IMAGE_DEPLOY_TEST_SHA" + elif [[ "$*" == *'.State.Health'* ]]; then + printf 'healthy\n' + else + printf 'sha256:old-image\n' + fi + return 0 + fi + + if [ "$1" != compose ]; then + echo "Unexpected fake Docker command: $*" >&2 + return 1 + fi + + local subcommand="" + local service="${*: -1}" + local arg + for arg in "$@"; do + case "$arg" in + config|logs|ps|run|up) + if [ -z "$subcommand" ]; then + subcommand="$arg" + fi + ;; + esac + done + + if [ "$subcommand" = ps ] && [[ "$*" == *' -q '* ]]; then + printf '%s-container\n' "$service" + fi + if [ "$subcommand" = up ] && [ "$service" = api ] \ + && [ "${FAKE_FAIL_CANDIDATE_API:-false}" = true ] \ + && [ "${APP_IMAGE_TAG:-}" = "$IMAGE_DEPLOY_TEST_SHA" ]; then + return 1 + fi +} + +fake_crontab() { + if [ "${1:-}" = -l ]; then + cat "$CRONTAB_TEST_STATE" + else + cp "$1" "$CRONTAB_TEST_STATE" + fi +} + +case "$(basename -- "$0")" in + docker) + fake_docker "$@" + exit + ;; + crontab) + fake_crontab "$@" + exit + ;; +esac + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/letscube-image-deploy-test.XXXXXX")" +SOURCE_SHA="4444444444444444444444444444444444444444" + +cleanup() { + rm -rf -- "$TEST_ROOT" +} +trap cleanup EXIT + +FAKE_BIN="$TEST_ROOT/bin" +mkdir -p "$FAKE_BIN" +ln -s "$SCRIPT_DIR/test-image-deploy.sh" "$FAKE_BIN/docker" +ln -s "$SCRIPT_DIR/test-image-deploy.sh" "$FAKE_BIN/crontab" + +prepare_scenario() { + local scenario="$1" + local app_dir="$scenario/app" + local release_dir="$scenario/release" + + mkdir -p "$app_dir/scripts" "$release_dir" + printf 'production secrets\n' > "$app_dir/.env.prod" + printf 'old compose\n' > "$app_dir/compose.yml" + printf 'old production compose\n' > "$app_dir/compose.prod.yml" + printf 'old nginx\n' > "$app_dir/nginx.conf" + printf 'new compose\n' > "$release_dir/compose.yml" + printf 'new production compose\n' > "$release_dir/compose.prod.yml" + printf 'new nginx\n' > "$release_dir/nginx.conf" + cp "$SCRIPT_DIR/check-production-health.sh" "$release_dir/check-production-health.sh" + cp "$SCRIPT_DIR/repair-production-services.sh" "$release_dir/repair-production-services.sh" + chmod +x "$release_dir/check-production-health.sh" \ + "$release_dir/repair-production-services.sh" +} + +assert_order() { + local log="$1" + local first="$2" + local second="$3" + local first_line + local second_line + + first_line="$(grep -n -m1 -F -- "$first" "$log" | cut -d: -f1)" + second_line="$(grep -n -m1 -F -- "$second" "$log" | cut -d: -f1)" + if [ -z "$first_line" ] || [ -z "$second_line" ] || [ "$first_line" -ge "$second_line" ]; then + echo "Expected '$first' before '$second'." >&2 + exit 1 + fi +} + +run_success_test() { + local scenario="$TEST_ROOT/success" + local log="$scenario/docker.log" + prepare_scenario "$scenario" + + PATH="$FAKE_BIN:$PATH" \ + APP_DIR="$scenario/app" \ + ENV_FILE=.env.prod \ + IMAGE_DEPLOY_TEST_LOG="$log" \ + IMAGE_DEPLOY_TEST_SHA="$SOURCE_SHA" \ + HEALTH_ATTEMPTS=1 \ + LOCK_FILE="$scenario/deploy.lock" \ + bash "$SCRIPT_DIR/deploy-image-release.sh" "$SOURCE_SHA" "$scenario/release" + + test "$(<"$scenario/app/.last-successful-deploy")" = "$SOURCE_SHA" + test "$(<"$scenario/app/compose.yml")" = 'new compose' + assert_order "$log" 'run --rm --no-deps migrate' \ + "up -d --no-deps --force-recreate --wait --wait-timeout 90 socket" + assert_order "$log" \ + "up -d --no-deps --force-recreate --wait --wait-timeout 90 socket" \ + "up -d --no-deps --force-recreate --wait --wait-timeout 90 api" +} + +run_rollback_test() { + local scenario="$TEST_ROOT/rollback" + local log="$scenario/docker.log" + prepare_scenario "$scenario" + + if PATH="$FAKE_BIN:$PATH" \ + APP_DIR="$scenario/app" \ + ENV_FILE=.env.prod \ + FAKE_FAIL_CANDIDATE_API=true \ + IMAGE_DEPLOY_TEST_LOG="$log" \ + IMAGE_DEPLOY_TEST_SHA="$SOURCE_SHA" \ + HEALTH_ATTEMPTS=1 \ + LOCK_FILE="$scenario/deploy.lock" \ + bash "$SCRIPT_DIR/deploy-image-release.sh" "$SOURCE_SHA" "$scenario/release"; then + echo 'Expected the candidate API deployment to fail.' >&2 + exit 1 + fi + + test ! -e "$scenario/app/.last-successful-deploy" + test "$(<"$scenario/app/compose.yml")" = 'old compose' + grep -Fq 'APP_IMAGE_TAG=rollback-socket-' "$log" + grep -Fq 'APP_IMAGE_TAG=rollback-api-' "$log" +} + +run_disable_cron_test() { + local scenario="$TEST_ROOT/disable-cron" + local state="$scenario/crontab" + local backup="$scenario/crontab.backup" + + mkdir -p "$scenario" + cat > "$state" <<'CRONTAB' +*/10 7-14 * * * cd /home/www/letscube && ./scripts/deploy.sh +17 8 * * * cd /home/www/letscube && ./scripts/backup-postgres.sh +CRONTAB + + PATH="$FAKE_BIN:$PATH" CRONTAB_TEST_STATE="$state" BACKUP_FILE="$backup" \ + bash "$SCRIPT_DIR/disable-legacy-deploy-cron.sh" + + grep -Fq 'scripts/deploy.sh' "$backup" + grep -Fq 'scripts/backup-postgres.sh' "$state" + grep -Fq 'scripts/repair-production-services.sh' "$state" + if grep -Fq 'scripts/deploy.sh' "$state"; then + echo 'Legacy deployment cron was not removed.' >&2 + exit 1 + fi + + PATH="$FAKE_BIN:$PATH" CRONTAB_TEST_STATE="$state" BACKUP_FILE="$backup" \ + bash "$SCRIPT_DIR/disable-legacy-deploy-cron.sh" +} + +run_success_test +run_rollback_test +run_disable_cron_test +echo 'Image deployment checks passed.' diff --git a/scripts/test-production-repair.sh b/scripts/test-production-repair.sh new file mode 100755 index 0000000..e75956e --- /dev/null +++ b/scripts/test-production-repair.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +set -euo pipefail + +log_command() { + { + printf 'APP_IMAGE_TAG=%s docker' "${APP_IMAGE_TAG:-}" + printf ' %q' "$@" + printf '\n' + } >> "$REPAIR_TEST_LOG" +} + +fake_docker() { + log_command "$@" + + if [ "$1" = image ]; then + printf '%s\n' "$REPAIR_TEST_SHA" + return 0 + fi + + if [ "$1" = inspect ]; then + if [[ "$*" == *org.opencontainers.image.revision* ]]; then + printf '%s\n' "$REPAIR_TEST_SHA" + elif grep -Fqx "${*: -1}" "$REPAIR_TEST_REPAIRED" 2> /dev/null; then + printf 'healthy\n' + elif [ "${*: -1}" = "${REPAIR_TEST_UNHEALTHY:-}" ]; then + printf 'unhealthy\n' + else + printf 'healthy\n' + fi + return 0 + fi + + if [ "$1" != compose ]; then + echo "Unexpected fake Docker command: $*" >&2 + return 1 + fi + + local service="${*: -1}" + if [[ "$*" == *' ps '* ]]; then + printf '%s-container\n' "$service" + elif [[ "$*" == *' up '* ]]; then + printf '%s-container\n' "$service" >> "$REPAIR_TEST_REPAIRED" + fi +} + +case "$(basename -- "$0")" in + docker) + fake_docker "$@" + exit + ;; +esac + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/letscube-repair-test.XXXXXX")" +SOURCE_SHA="5555555555555555555555555555555555555555" + +cleanup() { + rm -rf -- "$TEST_ROOT" +} +trap cleanup EXIT + +mkdir -p "$TEST_ROOT/bin" "$TEST_ROOT/app/scripts" +ln -s "$SCRIPT_DIR/test-production-repair.sh" "$TEST_ROOT/bin/docker" +cp "$SCRIPT_DIR/check-production-health.sh" "$TEST_ROOT/app/scripts/" +chmod +x "$TEST_ROOT/app/scripts/check-production-health.sh" +touch "$TEST_ROOT/app/compose.yml" "$TEST_ROOT/app/compose.prod.yml" \ + "$TEST_ROOT/app/.env.prod" "$TEST_ROOT/repaired" +printf '%s\n' "$SOURCE_SHA" > "$TEST_ROOT/app/.last-successful-deploy" + +PATH="$TEST_ROOT/bin:$PATH" \ + APP_DIR="$TEST_ROOT/app" \ + ENV_FILE=.env.prod \ + HEALTH_ATTEMPTS=1 \ + LOCK_FILE="$TEST_ROOT/deploy.lock" \ + REPAIR_TEST_LOG="$TEST_ROOT/docker.log" \ + REPAIR_TEST_REPAIRED="$TEST_ROOT/repaired" \ + REPAIR_TEST_SHA="$SOURCE_SHA" \ + REPAIR_TEST_UNHEALTHY=api-container \ + bash "$SCRIPT_DIR/repair-production-services.sh" + +grep -Fq 'up -d --no-deps --force-recreate --wait --wait-timeout 90 api' \ + "$TEST_ROOT/docker.log" +if grep -Fq 'up -d --no-deps --force-recreate --wait --wait-timeout 90 postgres' \ + "$TEST_ROOT/docker.log"; then + echo 'Repair replaced a healthy PostgreSQL container.' >&2 + exit 1 +fi + +echo 'Production service repair checks passed.' diff --git a/server/middlewares/clientAssets.js b/server/middlewares/clientAssets.js index ceb2ab2..b69e24a 100644 --- a/server/middlewares/clientAssets.js +++ b/server/middlewares/clientAssets.js @@ -1,12 +1,9 @@ const express = require('express'); const path = require('path'); -const rateLimit = require('express-rate-limit'); -const { apiRateLimitOptions } = require('./apiRateLimit'); -const createClientAssetsRouter = (buildPath, rateLimitOptions = apiRateLimitOptions()) => { +const createClientAssetsRouter = (buildPath) => { const router = express.Router(); - router.use(rateLimit(rateLimitOptions)); router.use(express.static(buildPath)); const indexPath = path.join(buildPath, 'index.html'); diff --git a/server/middlewares/clientAssets.test.js b/server/middlewares/clientAssets.test.js index 4987eb0..9ae0d07 100644 --- a/server/middlewares/clientAssets.test.js +++ b/server/middlewares/clientAssets.test.js @@ -5,7 +5,6 @@ const http = require('http'); const path = require('path'); const express = require('express'); const { createClientAssetsRouter } = require('./clientAssets'); -const { apiRateLimitOptions } = require('./apiRateLimit'); const request = (server) => new Promise((resolve, reject) => { const req = http.request({ @@ -22,15 +21,12 @@ const request = (server) => new Promise((resolve, reject) => { req.end(); }); -describe('client asset rate limiting', () => { +describe('client assets', () => { let server; beforeEach((done) => { const app = express(); - app.use(createClientAssetsRouter( - path.join(__dirname, '../../client'), - apiRateLimitOptions({ limit: 1, windowMs: 60_000 }), - )); + app.use(createClientAssetsRouter(path.join(__dirname, '../../client'))); server = app.listen(0, '127.0.0.1', () => done()); }); @@ -38,14 +34,11 @@ describe('client asset rate limiting', () => { server.close(() => done()); }); - it('serves the client fallback', async () => { + it('serves repeated client fallback requests without API rate limiting', async () => { const response = await request(server); expect(response.status).toBe(200); expect(response.body).toContain(''); - expect(await request(server)).toEqual({ - body: '{"code":"rate_limited","message":"Too many requests. Please try again later."}', - status: 429, - }); + expect((await request(server)).status).toBe(200); }); }); diff --git a/server/postgres/index.js b/server/postgres/index.js index 47242f4..11f520e 100644 --- a/server/postgres/index.js +++ b/server/postgres/index.js @@ -4,6 +4,9 @@ const config = require('../runtimeConfig'); const logger = require('../logger'); const RETRY_DELAY_MS = 30 * 1000; +const MAINTENANCE_INTERVAL_MS = 60 * 60 * 1000; +const SESSION_CLEANUP_BATCH_SIZE = 10_000; +const SESSION_CLEANUP_MAX_BATCHES = 100; const poolConfig = config.postgres.connectionString ? { connectionString: config.postgres.connectionString } @@ -36,6 +39,7 @@ pool.on('error', (err) => { let initialized = false; let initializationPromise; let lastFailureAt = 0; +let maintenancePromise; const initializePostgres = async () => { if (!config.postgres.enabled) { @@ -109,17 +113,67 @@ const deleteExpiredMetrics = () => query( 'DELETE FROM analytics.events WHERE expires_at <= now()', ); +const deleteExpiredSessions = async () => { + let deleted = 0; + + for (let batch = 0; batch < SESSION_CLEANUP_MAX_BATCHES; batch += 1) { + // Small transactions keep cleanup from holding one large table lock when + // a server starts with a large expired-session backlog. + const result = await query(` + WITH expired AS ( + SELECT sid + FROM app.sessions + WHERE expire <= now() + ORDER BY expire + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + DELETE FROM app.sessions AS sessions + USING expired + WHERE sessions.sid = expired.sid + `, [SESSION_CLEANUP_BATCH_SIZE]); + + if (!result) { + return deleted; + } + + deleted += result.rowCount; + if (result.rowCount < SESSION_CLEANUP_BATCH_SIZE) { + return deleted; + } + } + + return deleted; +}; + +const runPostgresMaintenance = () => { + if (maintenancePromise) { + return maintenancePromise; + } + + maintenancePromise = Promise.all([ + deleteExpiredMetrics(), + deleteExpiredSessions(), + ]).finally(() => { + maintenancePromise = undefined; + }); + + return maintenancePromise; +}; + const startPostgresMaintenance = () => { - deleteExpiredMetrics(); - const timer = setInterval(deleteExpiredMetrics, 6 * 60 * 60 * 1000); + runPostgresMaintenance(); + const timer = setInterval(runPostgresMaintenance, MAINTENANCE_INTERVAL_MS); timer.unref(); }; module.exports = { deleteExpiredMetrics, + deleteExpiredSessions, initializePostgres, pool, query, + runPostgresMaintenance, startPostgresMaintenance, withTransaction, }; diff --git a/server/postgres/index.test.js b/server/postgres/index.test.js new file mode 100644 index 0000000..ce3e26d --- /dev/null +++ b/server/postgres/index.test.js @@ -0,0 +1,46 @@ +/** @jest-environment node */ +/* eslint-env jest */ + +const mockQuery = jest.fn(); +const mockOn = jest.fn(); + +jest.mock('pg', () => ({ + Pool: jest.fn(() => ({ + on: mockOn, + query: mockQuery, + })), +})); +jest.mock('../runtimeConfig', () => ({ + postgres: { + enabled: true, + connectionString: 'postgres://test', + ssl: false, + }, +})); +jest.mock('../logger', () => ({ + error: jest.fn(), + info: jest.fn(), +})); + +const { deleteExpiredSessions } = require('./index'); + +describe('PostgreSQL maintenance', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockQuery.mockResolvedValueOnce({ rows: [{ '?column?': 1 }] }); + }); + + it('deletes expired sessions in bounded batches until the backlog is clear', async () => { + mockQuery + .mockResolvedValueOnce({ rowCount: 10_000 }) + .mockResolvedValueOnce({ rowCount: 4 }); + + await expect(deleteExpiredSessions()).resolves.toBe(10_004); + + expect(mockQuery).toHaveBeenCalledTimes(3); + expect(mockQuery.mock.calls[1][0]).toContain('WHERE expire <= now()'); + expect(mockQuery.mock.calls[1][0]).toContain('LIMIT $1'); + expect(mockQuery.mock.calls[1][0]).toContain('FOR UPDATE SKIP LOCKED'); + expect(mockQuery.mock.calls[1][1]).toEqual([10_000]); + }); +});