Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,12 @@ jobs:
- name: Validate production metrics configuration
run: scripts/test-compose-metrics-config.sh

- name: Validate production deployment
run: |
scripts/test-deploy-classifier.sh
scripts/test-deploy.sh
scripts/test-backup-postgres.sh

- name: Validate CI branch coverage
run: scripts/test-ci-branches.sh

Expand Down
27 changes: 18 additions & 9 deletions README_DEPLOYMENT.md
Original file line number Diff line number Diff line change
Expand Up @@ -260,23 +260,32 @@ connection in addition to the HTTP readiness endpoint.

## Backups And Restore

Run a PostgreSQL backup from the host or a trusted administration environment:
Run the backup script from the production checkout:

```bash
BACKUP_DIR=/opt/letscube/backups
mkdir -p "$BACKUP_DIR"
docker compose -f compose.yml -f compose.prod.yml --env-file .env.prod \
exec -T postgres pg_dump -U "$PGUSER" -d "$PGDATABASE" -Fc \
> "$BACKUP_DIR/letscube-postgres-$(date -u +%Y%m%dT%H%M%SZ).dump"
APP_DIR=/opt/letscube \
BACKUP_DIR=/opt/letscube/backups \
BACKUP_RETENTION_DAYS=14 \
./scripts/backup-postgres.sh
```

Retain encrypted PostgreSQL backups according to the hosting policy and test
restores against a separate database before relying on them.
The script writes a temporary file before it publishes the backup. It removes
PostgreSQL backups that are older than the retention period. It does not remove
other files.

Copy encrypted backups to a different host. Test restores against a separate
database before you rely on the backups.

Example cron:

Create the log directory before you install the cron entry:

```bash
mkdir -p /opt/letscube/logs
```

```cron
15 3 * * * cd /opt/letscube && docker compose -f compose.yml -f compose.prod.yml --env-file .env.prod exec -T postgres pg_dump -U "$PGUSER" -d "$PGDATABASE" -Fc > "/opt/letscube/backups/letscube-postgres-$(date -u +\%Y\%m\%dT\%H\%M\%SZ).dump"
17 8 * * * flock -n /tmp/letscube-postgres-backup.lock sh -c 'cd /opt/letscube && APP_DIR=/opt/letscube BACKUP_DIR=/opt/letscube/backups BACKUP_RETENTION_DAYS=14 ./scripts/backup-postgres.sh' >> /opt/letscube/logs/backup-postgres.log 2>&1
```

Restore into a separate PostgreSQL database for verification:
Expand Down
11 changes: 11 additions & 0 deletions compose.prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,32 @@ x-app-build: &app-build
REACT_APP_WCA_ORIGIN: ${REACT_APP_WCA_ORIGIN:-https://www.worldcubeassociation.org}
REACT_APP_WCA_CLIENT_ID: ${REACT_APP_WCA_CLIENT_ID:-}

x-production-logging: &production-logging
driver: json-file
options:
max-size: "20m"
max-file: "3"

services:
migrate:
build: *app-build

api:
build: *app-build
logging: *production-logging
restart: unless-stopped

socket:
build: *app-build
logging: *production-logging
restart: unless-stopped

postgres:
logging: *production-logging
restart: unless-stopped

redis:
logging: *production-logging
restart: unless-stopped

nginx:
Expand All @@ -38,4 +48,5 @@ services:
- ./nginx.conf:/etc/nginx/conf.d/default.conf:ro
- ${LETSENCRYPT_DIR:-/etc/letsencrypt}:/etc/letsencrypt:ro
- ${CERTBOT_WWW_DIR:-/var/www/certbot}:/var/www/certbot:ro
logging: *production-logging
restart: unless-stopped
47 changes: 47 additions & 0 deletions scripts/backup-postgres.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
#!/usr/bin/env bash
set -euo pipefail

APP_DIR="${APP_DIR:-/opt/letscube}"
ENV_FILE="${ENV_FILE:-.env.prod}"
BACKUP_DIR="${BACKUP_DIR:-$APP_DIR/backups}"
BACKUP_RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-14}"

if ! [[ "$BACKUP_RETENTION_DAYS" =~ ^[1-9][0-9]*$ ]]; then
echo "BACKUP_RETENTION_DAYS must be a positive integer." >&2
exit 2
fi

if [ "$BACKUP_DIR" = "/" ]; then
echo "BACKUP_DIR cannot be the root directory." >&2
exit 2
fi

COMPOSE=(docker compose -f compose.yml -f compose.prod.yml --env-file "$ENV_FILE")

cd "$APP_DIR"
mkdir -p -- "$BACKUP_DIR"
umask 077

timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
backup_file="$BACKUP_DIR/letscube-postgres-$timestamp.dump"
temporary_file="$(mktemp "$BACKUP_DIR/.letscube-postgres-$timestamp.XXXXXX")"

cleanup() {
rm -f -- "$temporary_file"
}

trap cleanup EXIT

"${COMPOSE[@]}" exec -T postgres \
sh -c 'exec pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Fc' \
> "$temporary_file"

mv -- "$temporary_file" "$backup_file"
trap - EXIT

find "$BACKUP_DIR" -maxdepth 1 -type f \
-name 'letscube-postgres-*.dump' \
-mtime "+$BACKUP_RETENTION_DAYS" \
-print -delete

echo "Created PostgreSQL backup: $backup_file"
11 changes: 8 additions & 3 deletions scripts/deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ if ! "${COMPOSE[@]}" build api; then
exit 1
fi

if ! "${COMPOSE[@]}" up -d --no-recreate --wait \
if ! "${COMPOSE[@]}" up -d --wait \
--wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" postgres redis; then
echo "Failed to start backing services; no application containers were replaced." >&2
print_retry_hint
Expand Down Expand Up @@ -221,10 +221,11 @@ reload_nginx() {
container_id="$("${compose_command[@]}" ps --status running -q nginx)"
if [ -z "$container_id" ]; then
echo "nginx is not running; starting it without replacing dependencies."
"${compose_command[@]}" up -d --no-deps --wait \
--wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" nginx || return 1
fi

"${compose_command[@]}" up -d --no-deps --wait \
--wait-timeout "$DEPLOY_WAIT_TIMEOUT_SECONDS" nginx || return 1

"${compose_command[@]}" exec -T nginx nginx -s reload
}

Expand Down Expand Up @@ -318,3 +319,7 @@ done

echo "Deployment complete: $resolved_target now uses letscube-app:$current_commit"
"${COMPOSE[@]}" ps

if ! docker image prune -af --filter "until=168h"; then
echo "Deployment succeeded, but unused image cleanup failed." >&2
fi
77 changes: 77 additions & 0 deletions scripts/test-backup-postgres.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
#!/usr/bin/env bash
set -euo pipefail

case "$(basename -- "$0")" in
docker)
printf '%q ' docker "$@" >> "$BACKUP_TEST_LOG"
printf '\n' >> "$BACKUP_TEST_LOG"
printf 'test PostgreSQL backup\n'
exit
;;
esac

SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
BACKUP_SCRIPT="$SCRIPT_DIR/backup-postgres.sh"
TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/letscube-backup-test.XXXXXX")"

cleanup() {
rm -rf -- "$TEST_ROOT"
}

trap cleanup EXIT

APP_DIR="$TEST_ROOT/app"
BACKUP_DIR="$TEST_ROOT/backups"
FAKE_BIN="$TEST_ROOT/bin"
BACKUP_TEST_LOG="$TEST_ROOT/docker.log"

mkdir -p "$APP_DIR" "$BACKUP_DIR" "$FAKE_BIN"
touch "$APP_DIR/compose.yml" "$APP_DIR/compose.prod.yml" "$APP_DIR/.env.prod"
ln -s "$SCRIPT_DIR/test-backup-postgres.sh" "$FAKE_BIN/docker"

stale_backup="$BACKUP_DIR/letscube-postgres-20261001T000000Z.dump"
unrelated_file="$BACKUP_DIR/keep.txt"
touch -d '16 days ago' "$stale_backup"
touch -d '16 days ago' "$unrelated_file"

PATH="$FAKE_BIN:$PATH" \
APP_DIR="$APP_DIR" \
BACKUP_DIR="$BACKUP_DIR" \
BACKUP_RETENTION_DAYS=14 \
BACKUP_TEST_LOG="$BACKUP_TEST_LOG" \
"$BACKUP_SCRIPT" > "$TEST_ROOT/output.log"

mapfile -t backups < <(find "$BACKUP_DIR" -maxdepth 1 -type f -name 'letscube-postgres-*.dump')

if [ "${#backups[@]}" -ne 1 ]; then
echo "Expected one current PostgreSQL backup." >&2
exit 1
fi

if [ "$(<"${backups[0]}")" != 'test PostgreSQL backup' ]; then
echo "The PostgreSQL backup content is incorrect." >&2
exit 1
fi

if [ -e "$stale_backup" ]; then
echo "The stale PostgreSQL backup remains." >&2
exit 1
fi

if [ ! -e "$unrelated_file" ]; then
echo "The retention rule removed an unrelated file." >&2
exit 1
fi

if ! grep -q 'exec -T postgres' "$BACKUP_TEST_LOG"; then
echo "The backup did not use the PostgreSQL service." >&2
exit 1
fi

if APP_DIR="$APP_DIR" BACKUP_DIR="$BACKUP_DIR" BACKUP_RETENTION_DAYS=invalid \
"$BACKUP_SCRIPT" > /dev/null 2>&1; then
echo "The backup accepted an invalid retention period." >&2
exit 1
fi

echo "PostgreSQL backup script checks passed."
11 changes: 10 additions & 1 deletion scripts/test-compose-metrics-config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,13 @@ jq -e \
)
' "$RENDERED_CONFIG" > /dev/null

echo "Production metrics Compose configuration checks passed."
jq -e '
[.services.api, .services.socket, .services.postgres, .services.redis, .services.nginx]
| all(
.logging.driver == "json-file"
and .logging.options["max-size"] == "20m"
and .logging.options["max-file"] == "3"
)
' "$RENDERED_CONFIG" > /dev/null

echo "Production Compose configuration checks passed."
9 changes: 8 additions & 1 deletion scripts/test-deploy.sh
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ run_bootstrap_test() {
assert_contains "$(<"$output")" 'Missing runtime services: api socket nginx; deploying all.'
assert_contains "$(<"$output")" '(target: all)'

backing_line="$(command_line "$log" 'up -d --no-recreate --wait --wait-timeout 17 postgres redis')"
backing_line="$(command_line "$log" 'up -d --wait --wait-timeout 17 postgres redis')"
migration_line="$(command_line "$log" 'run --rm --no-deps migrate')"
socket_line="$(command_line "$log" 'up -d --no-deps --wait --wait-timeout 17 socket')"
api_line="$(command_line "$log" 'up -d --no-deps --wait --wait-timeout 17 api')"
Expand All @@ -223,6 +223,8 @@ run_bootstrap_test() {
echo 'Bootstrap did not deploy socket before API.' >&2
exit 1
fi

assert_contains "$(<"$log")" 'docker image prune -af --filter until=168h'
}

run_rollback_test() {
Expand Down Expand Up @@ -270,6 +272,11 @@ run_rollback_test() {
assert_contains "$rollback_details" 'APP_IMAGE_TAG=rollback-socket-111111111111-'
assert_contains "$(<"$log")" "git show $previous_commit:compose.yml"
assert_contains "$(<"$log")" "git show $previous_commit:compose.prod.yml"

if grep -q 'docker image prune' "$log"; then
echo 'Failed deployment removed images.' >&2
exit 1
fi
}

run_privacy_floor_test() {
Expand Down
Loading