diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5c790fe..28d701f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,12 @@ jobs: - name: Validate production metrics configuration run: scripts/test-compose-metrics-config.sh + - name: Validate production deployment + run: | + scripts/test-deploy-classifier.sh + scripts/test-deploy.sh + scripts/test-backup-postgres.sh + - name: Validate CI branch coverage run: scripts/test-ci-branches.sh diff --git a/README_DEPLOYMENT.md b/README_DEPLOYMENT.md index 57d986c..527152e 100644 --- a/README_DEPLOYMENT.md +++ b/README_DEPLOYMENT.md @@ -260,23 +260,32 @@ connection in addition to the HTTP readiness endpoint. ## Backups And Restore -Run a PostgreSQL backup from the host or a trusted administration environment: +Run the backup script from the production checkout: ```bash -BACKUP_DIR=/opt/letscube/backups -mkdir -p "$BACKUP_DIR" -docker compose -f compose.yml -f compose.prod.yml --env-file .env.prod \ - exec -T postgres pg_dump -U "$PGUSER" -d "$PGDATABASE" -Fc \ - > "$BACKUP_DIR/letscube-postgres-$(date -u +%Y%m%dT%H%M%SZ).dump" +APP_DIR=/opt/letscube \ +BACKUP_DIR=/opt/letscube/backups \ +BACKUP_RETENTION_DAYS=14 \ + ./scripts/backup-postgres.sh ``` -Retain encrypted PostgreSQL backups according to the hosting policy and test -restores against a separate database before relying on them. +The script writes a temporary file before it publishes the backup. It removes +PostgreSQL backups that are older than the retention period. It does not remove +other files. + +Copy encrypted backups to a different host. Test restores against a separate +database before you rely on the backups. Example cron: +Create the log directory before you install the cron entry: + +```bash +mkdir -p /opt/letscube/logs +``` + ```cron -15 3 * * * cd /opt/letscube && docker compose -f compose.yml -f compose.prod.yml --env-file .env.prod exec -T postgres pg_dump -U "$PGUSER" -d "$PGDATABASE" -Fc > "/opt/letscube/backups/letscube-postgres-$(date -u +\%Y\%m\%dT\%H\%M\%SZ).dump" +17 8 * * * flock -n /tmp/letscube-postgres-backup.lock sh -c 'cd /opt/letscube && APP_DIR=/opt/letscube BACKUP_DIR=/opt/letscube/backups BACKUP_RETENTION_DAYS=14 ./scripts/backup-postgres.sh' >> /opt/letscube/logs/backup-postgres.log 2>&1 ``` Restore into a separate PostgreSQL database for verification: diff --git a/compose.prod.yml b/compose.prod.yml index 2ff522b..2d6b0d5 100644 --- a/compose.prod.yml +++ b/compose.prod.yml @@ -6,22 +6,32 @@ x-app-build: &app-build REACT_APP_WCA_ORIGIN: ${REACT_APP_WCA_ORIGIN:-https://www.worldcubeassociation.org} REACT_APP_WCA_CLIENT_ID: ${REACT_APP_WCA_CLIENT_ID:-} +x-production-logging: &production-logging + driver: json-file + options: + max-size: "20m" + max-file: "3" + services: migrate: build: *app-build api: build: *app-build + logging: *production-logging restart: unless-stopped socket: build: *app-build + logging: *production-logging restart: unless-stopped postgres: + logging: *production-logging restart: unless-stopped redis: + logging: *production-logging restart: unless-stopped nginx: @@ -38,4 +48,5 @@ services: - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro - ${LETSENCRYPT_DIR:-/etc/letsencrypt}:/etc/letsencrypt:ro - ${CERTBOT_WWW_DIR:-/var/www/certbot}:/var/www/certbot:ro + logging: *production-logging restart: unless-stopped diff --git a/scripts/backup-postgres.sh b/scripts/backup-postgres.sh new file mode 100755 index 0000000..15f5b1e --- /dev/null +++ b/scripts/backup-postgres.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +APP_DIR="${APP_DIR:-/opt/letscube}" +ENV_FILE="${ENV_FILE:-.env.prod}" +BACKUP_DIR="${BACKUP_DIR:-$APP_DIR/backups}" +BACKUP_RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-14}" + +if ! [[ "$BACKUP_RETENTION_DAYS" =~ ^[1-9][0-9]*$ ]]; then + echo "BACKUP_RETENTION_DAYS must be a positive integer." >&2 + exit 2 +fi + +if [ "$BACKUP_DIR" = "/" ]; then + echo "BACKUP_DIR cannot be the root directory." >&2 + exit 2 +fi + +COMPOSE=(docker compose -f compose.yml -f compose.prod.yml --env-file "$ENV_FILE") + +cd "$APP_DIR" +mkdir -p -- "$BACKUP_DIR" +umask 077 + +timestamp="$(date -u +%Y%m%dT%H%M%SZ)" +backup_file="$BACKUP_DIR/letscube-postgres-$timestamp.dump" +temporary_file="$(mktemp "$BACKUP_DIR/.letscube-postgres-$timestamp.XXXXXX")" + +cleanup() { + rm -f -- "$temporary_file" +} + +trap cleanup EXIT + +"${COMPOSE[@]}" exec -T postgres \ + sh -c 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc' \ + > "$temporary_file" + +mv -- "$temporary_file" "$backup_file" +trap - EXIT + +find "$BACKUP_DIR" -maxdepth 1 -type f \ + -name 'letscube-postgres-*.dump' \ + -mtime "+$BACKUP_RETENTION_DAYS" \ + -print -delete + +echo "Created PostgreSQL backup: $backup_file" diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 1b9dd73..cfc2259 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -175,7 +175,7 @@ if ! "${COMPOSE[@]}" build api; then exit 1 fi -if ! "${COMPOSE[@]}" up -d --no-recreate --wait \ +if ! "${COMPOSE[@]}" up -d --wait \ --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" postgres redis; then echo "Failed to start backing services; no application containers were replaced." >&2 print_retry_hint @@ -221,10 +221,11 @@ reload_nginx() { container_id="$("${compose_command[@]}" ps --status running -q nginx)" if [ -z "$container_id" ]; then echo "nginx is not running; starting it without replacing dependencies." - "${compose_command[@]}" up -d --no-deps --wait \ - --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" nginx || return 1 fi + "${compose_command[@]}" up -d --no-deps --wait \ + --wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" nginx || return 1 + "${compose_command[@]}" exec -T nginx nginx -s reload } @@ -318,3 +319,7 @@ done echo "Deployment complete: $resolved_target now uses letscube-app:$current_commit" "${COMPOSE[@]}" ps + +if ! docker image prune -af --filter "until=168h"; then + echo "Deployment succeeded, but unused image cleanup failed." >&2 +fi diff --git a/scripts/test-backup-postgres.sh b/scripts/test-backup-postgres.sh new file mode 100755 index 0000000..a7ad811 --- /dev/null +++ b/scripts/test-backup-postgres.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +case "$(basename -- "$0")" in + docker) + printf '%q ' docker "$@" >> "$BACKUP_TEST_LOG" + printf '\n' >> "$BACKUP_TEST_LOG" + printf 'test PostgreSQL backup\n' + exit + ;; +esac + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +BACKUP_SCRIPT="$SCRIPT_DIR/backup-postgres.sh" +TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/letscube-backup-test.XXXXXX")" + +cleanup() { + rm -rf -- "$TEST_ROOT" +} + +trap cleanup EXIT + +APP_DIR="$TEST_ROOT/app" +BACKUP_DIR="$TEST_ROOT/backups" +FAKE_BIN="$TEST_ROOT/bin" +BACKUP_TEST_LOG="$TEST_ROOT/docker.log" + +mkdir -p "$APP_DIR" "$BACKUP_DIR" "$FAKE_BIN" +touch "$APP_DIR/compose.yml" "$APP_DIR/compose.prod.yml" "$APP_DIR/.env.prod" +ln -s "$SCRIPT_DIR/test-backup-postgres.sh" "$FAKE_BIN/docker" + +stale_backup="$BACKUP_DIR/letscube-postgres-20261001T000000Z.dump" +unrelated_file="$BACKUP_DIR/keep.txt" +touch -d '16 days ago' "$stale_backup" +touch -d '16 days ago' "$unrelated_file" + +PATH="$FAKE_BIN:$PATH" \ + APP_DIR="$APP_DIR" \ + BACKUP_DIR="$BACKUP_DIR" \ + BACKUP_RETENTION_DAYS=14 \ + BACKUP_TEST_LOG="$BACKUP_TEST_LOG" \ + "$BACKUP_SCRIPT" > "$TEST_ROOT/output.log" + +mapfile -t backups < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name 'letscube-postgres-*.dump') + +if [ "${#backups[@]}" -ne 1 ]; then + echo "Expected one current PostgreSQL backup." >&2 + exit 1 +fi + +if [ "$(<"${backups[0]}")" != 'test PostgreSQL backup' ]; then + echo "The PostgreSQL backup content is incorrect." >&2 + exit 1 +fi + +if [ -e "$stale_backup" ]; then + echo "The stale PostgreSQL backup remains." >&2 + exit 1 +fi + +if [ ! -e "$unrelated_file" ]; then + echo "The retention rule removed an unrelated file." >&2 + exit 1 +fi + +if ! grep -q 'exec -T postgres' "$BACKUP_TEST_LOG"; then + echo "The backup did not use the PostgreSQL service." >&2 + exit 1 +fi + +if APP_DIR="$APP_DIR" BACKUP_DIR="$BACKUP_DIR" BACKUP_RETENTION_DAYS=invalid \ + "$BACKUP_SCRIPT" > /dev/null 2>&1; then + echo "The backup accepted an invalid retention period." >&2 + exit 1 +fi + +echo "PostgreSQL backup script checks passed." diff --git a/scripts/test-compose-metrics-config.sh b/scripts/test-compose-metrics-config.sh index d0cb929..db15a16 100755 --- a/scripts/test-compose-metrics-config.sh +++ b/scripts/test-compose-metrics-config.sh @@ -32,4 +32,13 @@ jq -e \ ) ' "$RENDERED_CONFIG" > /dev/null -echo "Production metrics Compose configuration checks passed." +jq -e ' + [.services.api, .services.socket, .services.postgres, .services.redis, .services.nginx] + | all( + .logging.driver == "json-file" + and .logging.options["max-size"] == "20m" + and .logging.options["max-file"] == "3" + ) +' "$RENDERED_CONFIG" > /dev/null + +echo "Production Compose configuration checks passed." diff --git a/scripts/test-deploy.sh b/scripts/test-deploy.sh index d40dfd1..805e303 100755 --- a/scripts/test-deploy.sh +++ b/scripts/test-deploy.sh @@ -210,7 +210,7 @@ run_bootstrap_test() { assert_contains "$(<"$output")" 'Missing runtime services: api socket nginx; deploying all.' assert_contains "$(<"$output")" '(target: all)' - backing_line="$(command_line "$log" 'up -d --no-recreate --wait --wait-timeout 17 postgres redis')" + backing_line="$(command_line "$log" 'up -d --wait --wait-timeout 17 postgres redis')" migration_line="$(command_line "$log" 'run --rm --no-deps migrate')" socket_line="$(command_line "$log" 'up -d --no-deps --wait --wait-timeout 17 socket')" api_line="$(command_line "$log" 'up -d --no-deps --wait --wait-timeout 17 api')" @@ -223,6 +223,8 @@ run_bootstrap_test() { echo 'Bootstrap did not deploy socket before API.' >&2 exit 1 fi + + assert_contains "$(<"$log")" 'docker image prune -af --filter until=168h' } run_rollback_test() { @@ -270,6 +272,11 @@ run_rollback_test() { assert_contains "$rollback_details" 'APP_IMAGE_TAG=rollback-socket-111111111111-' assert_contains "$(<"$log")" "git show $previous_commit:compose.yml" assert_contains "$(<"$log")" "git show $previous_commit:compose.prod.yml" + + if grep -q 'docker image prune' "$log"; then + echo 'Failed deployment removed images.' >&2 + exit 1 + fi } run_privacy_floor_test() {