diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5f405c..8a517dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,7 @@ jobs: - name: Validate production deployment run: | + scripts/test-public-health.sh scripts/test-deploy-classifier.sh scripts/test-deploy.sh scripts/test-image-deploy.sh @@ -117,6 +118,9 @@ jobs: PGDATABASE: letscube PGUSER: letscube PGPASSWORD: letscube + CORS_ORIGINS: http://localhost:3000,http://localhost:8080 + LETSCUBE_TEST_AUTH: true + SOCIAL_FEATURES_ENABLED: true services: postgres: image: postgres:17-alpine @@ -160,10 +164,15 @@ jobs: run: yarn workspace letscube-server postgres:migrate - name: Start server - run: CORS_ORIGINS=http://localhost:3000,http://localhost:8080 LETSCUBE_TEST_AUTH=true SOCIAL_FEATURES_ENABLED=true yarn workspace letscube-server node index.js > server.log 2>&1 & + run: | + mkdir -p .tmp + node server/index.js > server.log 2>&1 & + echo $! > .tmp/ci-api.pid - name: Start socket server - run: SOCIAL_FEATURES_ENABLED=true yarn workspace letscube-server node socket/ > socket.log 2>&1 & + run: | + node server/socket/index.js > socket.log 2>&1 & + echo $! > .tmp/ci-socket.pid - name: Wait for local stack run: | @@ -194,6 +203,8 @@ jobs: done - name: Run Cypress + env: + CYPRESS_faultInjection: true run: yarn cypress:run - name: Print app logs on failure diff --git a/.github/workflows/production-health.yml b/.github/workflows/production-health.yml new file mode 100644 index 0000000..3979527 --- /dev/null +++ b/.github/workflows/production-health.yml @@ -0,0 +1,66 @@ +name: Production health monitor + +on: + schedule: + - cron: '*/5 * * * *' + workflow_dispatch: + +permissions: + contents: read + issues: write + +concurrency: + group: production-health-monitor + cancel-in-progress: true + +jobs: + monitor: + name: Verify public production health + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Verify public routes + id: health + continue-on-error: true + run: scripts/check-public-health.sh + + - name: Open one outage issue + if: steps.health.outcome == 'failure' + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + label=production-health + gh label create "$label" \ + --color D73A4A \ + --description "The external production health monitor found an outage." \ + --force + issue_number="$(gh issue list --state open --label "$label" --json number --jq '.[0].number // empty')" + if [ -z "$issue_number" ]; then + gh issue create \ + --label "$label" \ + --title "Production health monitor failed" \ + --body "The external monitor could not verify the homepage and both health endpoints. [Open the failed run]($RUN_URL)." + fi + + - name: Close the outage issue after recovery + if: steps.health.outcome == 'success' + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + label_count="$(gh label list --json name --jq 'map(select(.name == "production-health")) | length')" + if [ "$label_count" -gt 0 ]; then + issue_number="$(gh issue list --state open --label production-health --json number --jq '.[0].number // empty')" + if [ -n "$issue_number" ]; then + gh issue close "$issue_number" \ + --comment "The external monitor reports that production recovered. [Open the successful run]($RUN_URL)." + fi + fi + + - name: Fail an unhealthy monitor run + if: steps.health.outcome == 'failure' + run: exit 1 diff --git a/client/src/components/ErrorBoundary.jsx b/client/src/components/ErrorBoundary.jsx new file mode 100644 index 0000000..a4f9e5e --- /dev/null +++ b/client/src/components/ErrorBoundary.jsx @@ -0,0 +1,46 @@ +import React from 'react'; +import PropTypes from 'prop-types'; + +class ErrorBoundary extends React.Component { + constructor(props) { + super(props); + this.state = { failed: false }; + } + + static getDerivedStateFromError() { + return { failed: true }; + } + + componentDidCatch(error, details) { + this.props.reportError(error, details); + } + + render() { + if (!this.state.failed) { + return this.props.children; + } + + return ( +
+

Let's Cube stopped unexpectedly.

+

If you have a saved result, it remains on this device. Reload the app to recover.

+ +
+ ); + } +} + +ErrorBoundary.propTypes = { + children: PropTypes.node.isRequired, + onReload: PropTypes.func, + reportError: PropTypes.func, +}; + +ErrorBoundary.defaultProps = { + onReload: () => window.location.reload(), + // The browser console is the only error sink configured in this client. + // eslint-disable-next-line no-console + reportError: (error, details) => console.error(error, details), +}; + +export default ErrorBoundary; diff --git a/client/src/components/ErrorBoundary.test.jsx b/client/src/components/ErrorBoundary.test.jsx new file mode 100644 index 0000000..a4763c6 --- /dev/null +++ b/client/src/components/ErrorBoundary.test.jsx @@ -0,0 +1,23 @@ +import React from 'react'; +import { fireEvent, render, screen } from '@testing-library/react'; +import ErrorBoundary from './ErrorBoundary'; + +const Broken = () => { + throw new Error('render failed'); +}; + +it('offers a reload action after a render failure', () => { + const onReload = jest.fn(); + const reportError = jest.fn(); + + render( + + + , + ); + + expect(screen.getByRole('alert')).toHaveTextContent("Let's Cube stopped unexpectedly."); + expect(reportError).toHaveBeenCalledWith(expect.any(Error), expect.any(Object)); + fireEvent.click(screen.getByRole('button', { name: 'Reload app' })); + expect(onReload).toHaveBeenCalledTimes(1); +}); diff --git a/client/src/index.jsx b/client/src/index.jsx index ccfceef..be15cfc 100644 --- a/client/src/index.jsx +++ b/client/src/index.jsx @@ -2,7 +2,9 @@ import React from 'react'; import { createRoot } from 'react-dom/client'; import './css/index.css'; import App from './components/App'; +import ErrorBoundary from './components/ErrorBoundary'; import * as serviceWorker from './registerServiceWorker'; +import { promptForAppUpdate } from './lib/appUpdate'; import store from './store'; import { connectSocket as connectToDefault } from './store/default/actions'; import { connectSocket as connectToRooms } from './store/rooms/actions'; @@ -14,8 +16,12 @@ store.dispatch(fetchUser()); /* CssBaseline kickstart an elegant, consistent, and simple baseline to build upon. */ const root = createRoot(document.querySelector('#root')); -root.render(); +root.render( + + + , +); // If you want your app to work offline and load faster, you can change // unregister() to register() below. Note this comes with some pitfalls. -serviceWorker.register(); +serviceWorker.register({ onUpdate: promptForAppUpdate }); diff --git a/client/src/lib/appUpdate.js b/client/src/lib/appUpdate.js new file mode 100644 index 0000000..04d9be1 --- /dev/null +++ b/client/src/lib/appUpdate.js @@ -0,0 +1,14 @@ +const UPDATE_MESSAGE = 'A new version of Let\'s Cube is ready. Reload now?'; + +export const promptForAppUpdate = ( + applyUpdate, + confirmUpdate = (message) => window.confirm(message), +) => { + if (confirmUpdate(UPDATE_MESSAGE)) { + applyUpdate(); + return true; + } + return false; +}; + +export { UPDATE_MESSAGE }; diff --git a/client/src/lib/appUpdate.test.js b/client/src/lib/appUpdate.test.js new file mode 100644 index 0000000..a858bf4 --- /dev/null +++ b/client/src/lib/appUpdate.test.js @@ -0,0 +1,17 @@ +import { promptForAppUpdate, UPDATE_MESSAGE } from './appUpdate'; + +it('applies a waiting update after user approval', () => { + const applyUpdate = jest.fn(); + const confirmUpdate = jest.fn().mockReturnValue(true); + + expect(promptForAppUpdate(applyUpdate, confirmUpdate)).toBe(true); + expect(confirmUpdate).toHaveBeenCalledWith(UPDATE_MESSAGE); + expect(applyUpdate).toHaveBeenCalledTimes(1); +}); + +it('keeps the current client after user rejection', () => { + const applyUpdate = jest.fn(); + + expect(promptForAppUpdate(applyUpdate, () => false)).toBe(false); + expect(applyUpdate).not.toHaveBeenCalled(); +}); diff --git a/client/src/lib/fetch.js b/client/src/lib/fetch.js index b61c509..7eef7c6 100644 --- a/client/src/lib/fetch.js +++ b/client/src/lib/fetch.js @@ -3,11 +3,37 @@ export const apiOrigin = process.env.REACT_APP_API_ORIGIN; const UNSAFE_METHODS = new Set(['DELETE', 'PATCH', 'POST', 'PUT']); +const DEFAULT_TIMEOUT_MS = 15000; let csrfTokenRequest; +export const fetchWithTimeout = async (url, options = {}, timeoutMs = DEFAULT_TIMEOUT_MS) => { + const controller = new AbortController(); + const upstreamSignal = options.signal; + const abortFromUpstream = () => controller.abort(upstreamSignal.reason); + if (upstreamSignal) { + if (upstreamSignal.aborted) { + abortFromUpstream(); + } else { + upstreamSignal.addEventListener('abort', abortFromUpstream, { once: true }); + } + } + const timeout = setTimeout(() => { + controller.abort(new Error('Request timed out.')); + }, timeoutMs); + + try { + return await fetch(url, { ...options, signal: controller.signal }); + } finally { + clearTimeout(timeout); + if (upstreamSignal) { + upstreamSignal.removeEventListener('abort', abortFromUpstream); + } + } +}; + const requestCsrfToken = async () => { if (!csrfTokenRequest) { - csrfTokenRequest = fetch(`${apiOrigin}/api/csrf-token`, { + csrfTokenRequest = fetchWithTimeout(`${apiOrigin}/api/csrf-token`, { credentials: 'include', }).then(async (response) => { if (!response.ok) { @@ -38,7 +64,7 @@ export const lcFetch = async (url, options = {}) => { const method = (options.method || 'GET').toUpperCase(); const csrfToken = UNSAFE_METHODS.has(method) ? await requestCsrfToken() : null; - return fetch(`${apiOrigin}${url}`, { + return fetchWithTimeout(`${apiOrigin}${url}`, { ...options, credentials: 'include', ...(csrfToken ? { headers: withCsrfToken(options.headers, csrfToken) } : {}), diff --git a/client/src/lib/fetch.test.js b/client/src/lib/fetch.test.js index 3fcd87a..beed232 100644 --- a/client/src/lib/fetch.test.js +++ b/client/src/lib/fetch.test.js @@ -1,16 +1,16 @@ -import { lcFetch } from './fetch'; +import { fetchWithTimeout, lcFetch } from './fetch'; describe('lcFetch', () => { beforeEach(() => { - window.fetch = jest.fn(); + globalThis.fetch = jest.fn(); }); afterEach(() => { - jest.resetModules(); + jest.restoreAllMocks(); }); it('adds a server-issued CSRF token to unsafe requests', async () => { - window.fetch + globalThis.fetch .mockResolvedValueOnce({ json: () => Promise.resolve({ csrfToken: 'csrf-token' }), ok: true, @@ -22,27 +22,75 @@ describe('lcFetch', () => { method: 'PUT', }); - expect(window.fetch).toHaveBeenNthCalledWith(1, 'undefined/api/csrf-token', { - credentials: 'include', - }); - expect(window.fetch).toHaveBeenNthCalledWith(2, 'undefined/api/updatePreference', { - credentials: 'include', - headers: { - 'Content-Type': 'application/json', - 'x-csrf-token': 'csrf-token', - }, - method: 'PUT', - }); + expect(globalThis.fetch).toHaveBeenNthCalledWith( + 1, + 'undefined/api/csrf-token', + expect.objectContaining({ + credentials: 'include', + signal: expect.any(AbortSignal), + }), + ); + expect(globalThis.fetch).toHaveBeenNthCalledWith( + 2, + 'undefined/api/updatePreference', + expect.objectContaining({ + credentials: 'include', + headers: { + 'Content-Type': 'application/json', + 'x-csrf-token': 'csrf-token', + }, + method: 'PUT', + signal: expect.any(AbortSignal), + }), + ); }); it('does not fetch a CSRF token for safe requests', async () => { - window.fetch.mockResolvedValue({ ok: true }); + globalThis.fetch.mockResolvedValue({ ok: true }); await lcFetch('/api/me'); - expect(window.fetch).toHaveBeenCalledTimes(1); - expect(window.fetch).toHaveBeenCalledWith('undefined/api/me', { - credentials: 'include', - }); + expect(globalThis.fetch).toHaveBeenCalledTimes(1); + expect(globalThis.fetch).toHaveBeenCalledWith( + 'undefined/api/me', + expect.objectContaining({ + credentials: 'include', + signal: expect.any(AbortSignal), + }), + ); + }); +}); + +describe('fetchWithTimeout', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + jest.restoreAllMocks(); + }); + + it('aborts a request after its deadline', async () => { + globalThis.fetch = jest.fn((_url, { signal }) => new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason)); + })); + + const request = fetchWithTimeout('/slow', {}, 5000); + jest.advanceTimersByTime(5000); + + await expect(request).rejects.toThrow('Request timed out.'); + }); + + it('preserves a caller abort signal', async () => { + const upstream = new AbortController(); + globalThis.fetch = jest.fn((_url, { signal }) => new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason)); + })); + + const request = fetchWithTimeout('/cancelled', { signal: upstream.signal }); + upstream.abort(new Error('Caller cancelled.')); + + await expect(request).rejects.toThrow('Caller cancelled.'); }); }); diff --git a/client/src/registerServiceWorker.js b/client/src/registerServiceWorker.js index af5bebd..232f385 100644 --- a/client/src/registerServiceWorker.js +++ b/client/src/registerServiceWorker.js @@ -48,48 +48,15 @@ export function register(config) { function registerValidSW(swUrl, config) { const wb = new Workbox(swUrl); + wb.addEventListener('waiting', () => { + if (config && config.onUpdate) { + config.onUpdate(() => wb.messageSkipWaiting()); + } + }); + wb.addEventListener('controlling', () => { + window.location.reload(); + }); wb.register(); - // navigator.serviceWorker - // .register(swUrl) - // .then((registration) => { - // registration.onupdatefound = () => { - // const installingWorker = registration.installing; - // if (installingWorker == null) { - // return; - // } - // installingWorker.onstatechange = () => { - // if (installingWorker.state === 'installed') { - // if (navigator.serviceWorker.controller) { - // // At this point, the updated precached content has been fetched, - // // but the previous service worker will still serve the older - // // content until all client tabs are closed. - // console.log( - // 'New content is available and will be used when all ' - // + 'tabs for this page are closed. See https://bit.ly/CRA-PWA.', - // ); - - // // Execute callback - // if (config && config.onUpdate) { - // config.onUpdate(registration); - // } - // } else { - // // At this point, everything has been precached. - // // It's the perfect time to display a - // // "Content is cached for offline use." message. - // console.log('Content is cached for offline use.'); - - // // Execute callback - // if (config && config.onSuccess) { - // config.onSuccess(registration); - // } - // } - // } - // }; - // }; - // }) - // .catch((error) => { - // console.error('Error during service worker registration:', error); - // }); } function checkValidServiceWorker(swUrl, config) { diff --git a/client/src/store/middlewares/rooms.js b/client/src/store/middlewares/rooms.js index 8897304..b96c57e 100644 --- a/client/src/store/middlewares/rooms.js +++ b/client/src/store/middlewares/rooms.js @@ -100,6 +100,7 @@ export const createRoomsNamespaceMiddleware = ({ let roomsConnected = false; let joinedRoomId = null; let pendingJoinRequest = null; + let pendingChatMessages = []; let joinGeneration = 0; let submissionGeneration = 0; let submissionAckTimer = null; @@ -274,6 +275,18 @@ export const createRoomsNamespaceMiddleware = ({ let namespace; + const flushPendingChatMessages = (roomId) => { + const remainingMessages = []; + pendingChatMessages.forEach((pendingMessage) => { + if (String(pendingMessage.roomId) === String(roomId)) { + namespace.emit(Protocol.MESSAGE, pendingMessage.message); + } else { + remainingMessages.push(pendingMessage); + } + }); + pendingChatMessages = remainingMessages; + }; + flushPendingResult = () => { const submission = currentSubmission(); let pendingResult = submission && submission.pendingResult; @@ -646,6 +659,7 @@ export const createRoomsNamespaceMiddleware = ({ password: room.private ? roomPassword || null : null, })); joinedRoomId = joinedRoomIdValue; + flushPendingChatMessages(joinedRoomIdValue); flushPendingResult(); if (room.accessCode) { @@ -675,6 +689,7 @@ export const createRoomsNamespaceMiddleware = ({ password: room.private ? options.password : null, })); joinedRoomId = room._id; + flushPendingChatMessages(room._id); history.push(`/rooms/${room._id}`); flushPendingResult(); }); @@ -682,6 +697,7 @@ export const createRoomsNamespaceMiddleware = ({ [LEAVE_ROOM]: () => { joinedRoomId = null; pendingJoinRequest = null; + pendingChatMessages = []; joinGeneration += 1; if (store.getState().room._id) { namespace.emit(Protocol.LEAVE_ROOM); @@ -784,7 +800,12 @@ export const createRoomsNamespaceMiddleware = ({ }); }, [SEND_CHAT]: ({ message }) => { - namespace.emit(Protocol.MESSAGE, message); + const roomId = store.getState().room._id; + if (roomsConnected && String(joinedRoomId) === String(roomId)) { + namespace.emit(Protocol.MESSAGE, message); + } else if (roomId) { + pendingChatMessages.push({ roomId, message }); + } }, [SEND_STATUS]: ({ status }) => { namespace.emit(Protocol.UPDATE_STATUS, { diff --git a/client/src/store/middlewares/rooms.test.js b/client/src/store/middlewares/rooms.test.js index a3f6237..cb21ecd 100644 --- a/client/src/store/middlewares/rooms.test.js +++ b/client/src/store/middlewares/rooms.test.js @@ -11,6 +11,7 @@ import { joinRoom, submitResult, } from '../room/actions'; +import { sendChat } from '../chat/actions'; import { connectSocket, createRoom } from '../rooms/actions'; import { PENDING_RESULT_STORAGE_KEY, @@ -177,6 +178,27 @@ describe('rooms middleware', () => { expect(store.getState().room.attempts).toBe(attempts); }); + it('sends chat only after the room rejoin succeeds', () => { + const { namespace, store } = buildStore(); + store.dispatch(connectSocket()); + namespace.options.onConnected(); + acknowledgeJoin(namespace); + + namespace.options.onDisconnected(); + namespace.options.onConnected(); + store.dispatch(sendChat({ text: 'Sent during reconnect' })); + + expect(emissionsFor(namespace, Protocol.MESSAGE)).toHaveLength(0); + + acknowledgeJoin(namespace); + + expect(emissionsFor(namespace, Protocol.MESSAGE)).toEqual([ + expect.objectContaining({ + args: [{ text: 'Sent during reconnect' }], + }), + ]); + }); + it('waits for the socket connection before emitting the initial room join', () => { const { namespace, store } = buildStore({ roomState: { diff --git a/compose.yml b/compose.yml index 02dcfb5..30c58c7 100644 --- a/compose.yml +++ b/compose.yml @@ -45,6 +45,7 @@ services: api: image: letscube-app:${APP_IMAGE_TAG:-local} command: ["node", "server/index.js"] + stop_grace_period: 30s environment: <<: *app-environment depends_on: @@ -65,6 +66,7 @@ services: socket: image: letscube-app:${APP_IMAGE_TAG:-local} command: ["node", "server/socket/index.js"] + stop_grace_period: 30s environment: <<: *app-environment depends_on: diff --git a/cypress.config.js b/cypress.config.js index e760be7..3702dda 100644 --- a/cypress.config.js +++ b/cypress.config.js @@ -1,4 +1,5 @@ const { defineConfig } = require('cypress'); +const { restartService } = require('./cypress/serviceControl'); module.exports = defineConfig({ e2e: { @@ -6,5 +7,15 @@ module.exports = defineConfig({ specPattern: 'cypress/e2e/**/*.cy.js', supportFile: false, video: false, + setupNodeEvents(on, config) { + if (config.env.faultInjection) { + on('task', { + restartApi: () => restartService('api'), + restartSocket: () => restartService('socket'), + }); + } + + return config; + }, }, }); diff --git a/cypress/e2e/restart_recovery.cy.js b/cypress/e2e/restart_recovery.cy.js new file mode 100644 index 0000000..3d41eb8 --- /dev/null +++ b/cypress/e2e/restart_recovery.cy.js @@ -0,0 +1,56 @@ +describe('service restart recovery', () => { + const appOrigin = Cypress.config('baseUrl') || 'http://localhost:3000'; + + const post = (url, body) => cy.request(`${appOrigin}/api/csrf-token`) + .then(({ body: { csrfToken } }) => cy.request({ + body, + headers: { 'x-csrf-token': csrfToken }, + method: 'POST', + url, + })); + + const login = () => { + post(`${appOrigin}/auth/code`, { + code: 'cypress-test-code', + redirectUri: 'http://localhost:3000/wca-redirect', + }); + cy.request(`${appOrigin}/api/me`).its('body.id').should('eq', 990001); + }; + + beforeEach(() => { + if (!Cypress.env('faultInjection')) { + cy.log('Fault injection is only enabled in CI.'); + return; + } + cy.clearCookies(); + }); + + it('keeps the login session when the API restarts', function testApiRestart() { + if (!Cypress.env('faultInjection')) this.skip(); + + login(); + cy.task('restartApi'); + cy.request(`${appOrigin}/api/me`).its('body.id').should('eq', 990001); + }); + + it('rejoins the active room when the socket service restarts', function testSocketRestart() { + if (!Cypress.env('faultInjection')) this.skip(); + + login(); + cy.visit('/'); + cy.contains('Create Room', { timeout: 10000 }).click(); + cy.get('#roomName').type(`Restart Recovery ${Date.now()}`); + cy.get('[role="dialog"]').contains('button', 'Create').click(); + + cy.location('pathname', { timeout: 10000 }).should('match', /^\/rooms\/[0-9a-f-]{36}$/); + cy.contains('Waiting For:', { timeout: 10000 }).should('be.visible'); + cy.location('pathname').then((pathname) => { + const message = `Recovered after restart ${Date.now()}`; + cy.task('restartSocket'); + cy.location('pathname').should('eq', pathname); + cy.contains('Waiting For:', { timeout: 20000 }).should('be.visible'); + cy.get('input[placeholder="Send Message"]').type(`${message}{enter}`); + cy.contains(message, { timeout: 20000 }).should('be.visible'); + }); + }); +}); diff --git a/cypress/serviceControl.js b/cypress/serviceControl.js new file mode 100644 index 0000000..a2cceac --- /dev/null +++ b/cypress/serviceControl.js @@ -0,0 +1,82 @@ +const fs = require('fs'); +const path = require('path'); +const { spawn } = require('child_process'); + +const root = path.resolve(__dirname, '..'); +const services = { + api: { + entry: 'server/index.js', + healthUrl: 'http://127.0.0.1:8080/health/api', + log: 'server.log', + pidFile: '.tmp/ci-api.pid', + }, + socket: { + entry: 'server/socket/index.js', + healthUrl: 'http://127.0.0.1:9000/health/socket', + log: 'socket.log', + pidFile: '.tmp/ci-socket.pid', + }, +}; + +const wait = (delayMs) => new Promise((resolve) => setTimeout(resolve, delayMs)); + +const isRunning = (pid) => { + try { + process.kill(pid, 0); + return true; + } catch (error) { + if (error.code === 'ESRCH') return false; + throw error; + } +}; + +const waitForStop = async (pid) => { + for (let attempt = 0; attempt < 300; attempt += 1) { + if (!isRunning(pid)) return; + await wait(100); + } + throw new Error(`Process ${pid} did not stop gracefully.`); +}; + +const waitForHealth = async (url) => { + for (let attempt = 0; attempt < 300; attempt += 1) { + try { + const response = await fetch(url); + if (response.ok) return; + } catch (error) { + // The service can refuse connections while it starts. + } + await wait(100); + } + throw new Error(`Service did not become healthy at ${url}.`); +}; + +const restartService = async (name) => { + const service = services[name]; + if (!service) throw new Error(`Unknown service: ${name}`); + + const pidPath = path.join(root, service.pidFile); + const previousPid = Number(fs.readFileSync(pidPath, 'utf8').trim()); + if (!Number.isInteger(previousPid) || previousPid <= 1) { + throw new Error(`Invalid PID in ${service.pidFile}.`); + } + if (isRunning(previousPid)) { + process.kill(previousPid, 'SIGTERM'); + await waitForStop(previousPid); + } + + const logFd = fs.openSync(path.join(root, service.log), 'a'); + const child = spawn(process.execPath, [service.entry], { + cwd: root, + env: process.env, + stdio: ['ignore', logFd, logFd], + }); + fs.closeSync(logFd); + child.unref(); + fs.writeFileSync(pidPath, `${child.pid}\n`); + + await waitForHealth(service.healthUrl); + return null; +}; + +module.exports = { restartService }; diff --git a/docs/operations.md b/docs/operations.md index 909290f..07cf1c5 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -84,6 +84,20 @@ nginx also has a container health check. It calls the API health route through the local TLS listener. This detects a broken proxy configuration and an unavailable API upstream. +The `Production health monitor` GitHub Actions workflow checks the homepage and +both health endpoints every five minutes. It opens one `production-health` +issue during an outage. It closes that issue after the public routes recover. +This external check still runs when the production host or its network fails. + +API and Socket.IO containers allow 30 seconds for a clean stop. On `SIGTERM` or +`SIGINT`, they stop accepting connections and close their PostgreSQL and Redis +clients before they exit. The Socket.IO process also closes its active sockets. +This keeps rolling deployments from cutting off work in the middle of cleanup. + +The Cypress CI job restarts both Node services during browser tests. It verifies +that an API restart keeps the PostgreSQL-backed login session and that a browser +in a room reconnects after a Socket.IO restart. + ## Backups and restore Create a PostgreSQL custom-format backup and test restores against a separate diff --git a/docs/realtime.md b/docs/realtime.md index 00a652c..ceaecd3 100644 --- a/docs/realtime.md +++ b/docs/realtime.md @@ -63,9 +63,11 @@ adapter. This prevents a deploy or second browser tab from producing a false leave. The server also reconciles durable room presence after startup and every five -minutes. Reconciliation processes two departures at a time so it does not -exhaust the PostgreSQL connection pool. A departure that throws an error gets -two retries. +minutes. A narrow PostgreSQL query reads only active room IDs, user IDs, and +per-user presence revisions. It does not load full rooms, attempts, or solves. +Reconciliation processes two departures at a time so it does not exhaust the +PostgreSQL connection pool. Per-user revision checks let one pass remove several +stale users from the same room. A departure that throws an error gets two retries. Each retry checks live socket presence and the durable presence revision again. This prevents an old cleanup attempt from removing a user who reconnected. diff --git a/scripts/check-public-health.sh b/scripts/check-public-health.sh new file mode 100755 index 0000000..bf71b63 --- /dev/null +++ b/scripts/check-public-health.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${PRODUCTION_BASE_URL:-https://letscube.net}" +CURL=( + curl + --fail + --silent + --show-error + --location + --connect-timeout 10 + --max-time 20 + --retry 2 + --retry-delay 2 + --retry-connrefused +) + +"${CURL[@]}" --output /dev/null "$BASE_URL/" + +verify_health() { + local path="$1" + local service="$2" + local response + + response="$("${CURL[@]}" "$BASE_URL$path")" + if ! grep -Eq '"status"[[:space:]]*:[[:space:]]*"ok"' <<< "$response" \ + || ! grep -Eq "\"service\"[[:space:]]*:[[:space:]]*\"$service\"" <<< "$response"; then + echo "Unexpected health response from $path" >&2 + return 1 + fi +} + +verify_health /health/api api +verify_health /health/socket socket + +echo "The public production routes are healthy." diff --git a/scripts/test-public-health.sh b/scripts/test-public-health.sh new file mode 100755 index 0000000..6498f2c --- /dev/null +++ b/scripts/test-public-health.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail + +case "$(basename -- "$0")" in + curl) + url="${!#}" + printf '%s\n' "$url" >> "$PUBLIC_HEALTH_TEST_LOG" + if [ "${PUBLIC_HEALTH_FAIL_PATH:-}" != "" ] \ + && [[ "$url" == *"$PUBLIC_HEALTH_FAIL_PATH" ]]; then + exit 22 + fi + case "$url" in + */health/api) + printf '{"status":"ok","service":"api"}\n' + ;; + */health/socket) + printf '{"status":"ok","service":"socket"}\n' + ;; + esac + exit 0 + ;; +esac + +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +TEST_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/letscube-public-health.XXXXXX")" + +cleanup() { + rm -rf -- "$TEST_ROOT" +} +trap cleanup EXIT + +PUBLIC_HEALTH_TEST_LOG="$TEST_ROOT/curl.log" +mkdir -p "$TEST_ROOT/bin" +ln -s "$SCRIPT_DIR/test-public-health.sh" "$TEST_ROOT/bin/curl" + +PATH="$TEST_ROOT/bin:$PATH" \ + PUBLIC_HEALTH_TEST_LOG="$PUBLIC_HEALTH_TEST_LOG" \ + PRODUCTION_BASE_URL=https://example.test \ + "$SCRIPT_DIR/check-public-health.sh" > /dev/null + +for path in / /health/api /health/socket; do + if ! grep -Fqx "https://example.test$path" "$PUBLIC_HEALTH_TEST_LOG"; then + echo "The public health test did not request $path." >&2 + exit 1 + fi +done + +if PATH="$TEST_ROOT/bin:$PATH" \ + PUBLIC_HEALTH_TEST_LOG="$PUBLIC_HEALTH_TEST_LOG" \ + PUBLIC_HEALTH_FAIL_PATH=/health/socket \ + PRODUCTION_BASE_URL=https://example.test \ + "$SCRIPT_DIR/check-public-health.sh" > /dev/null 2>&1; then + echo "The public health test accepted an unavailable socket service." >&2 + exit 1 +fi + +echo "Public production health checks passed." diff --git a/server/auth/index.js b/server/auth/index.js index 04de5db..ac0f2e6 100644 --- a/server/auth/index.js +++ b/server/auth/index.js @@ -8,6 +8,13 @@ const { buildWcaUserUpdate } = require('./wcaProfile'); const { upsertTestUser } = require('./testUser'); const { apiRateLimitOptions } = require('../middlewares/apiRateLimit'); +const AUTH_PROVIDER_TIMEOUT_MS = 10000; + +const fetchFromAuthProvider = (url, options = {}) => fetch(url, { + ...options, + signal: AbortSignal.timeout(AUTH_PROVIDER_TIMEOUT_MS), +}); + const checkStatus = async (res) => { if (res.ok) { // res.status >= 200 && res.status < 300 return res; @@ -68,7 +75,7 @@ module.exports = (app, passport, rateLimitOptions = apiRateLimitOptions()) => { params.append('code', code); try { - const tokenRes = await fetch(tokenURL, { + const tokenRes = await fetchFromAuthProvider(tokenURL, { method: 'POST', body: params, headers: { @@ -76,7 +83,7 @@ module.exports = (app, passport, rateLimitOptions = apiRateLimitOptions()) => { }, }).then(checkStatus).then((data) => data.json()); - const meRes = await fetch(userProfileURL, { + const meRes = await fetchFromAuthProvider(userProfileURL, { headers: { 'Content-Type': 'application/x-www-form-urlencoded', Authorization: `Bearer ${tokenRes.access_token}`, @@ -154,3 +161,4 @@ module.exports = (app, passport, rateLimitOptions = apiRateLimitOptions()) => { }; module.exports.deserializeUser = deserializeUser; +module.exports.fetchFromAuthProvider = fetchFromAuthProvider; diff --git a/server/auth/index.test.js b/server/auth/index.test.js index 61b8873..b1217f8 100644 --- a/server/auth/index.test.js +++ b/server/auth/index.test.js @@ -1,7 +1,7 @@ /** @jest-environment node */ /* eslint-env jest */ -const { deserializeUser } = require('./index'); +const { deserializeUser, fetchFromAuthProvider } = require('./index'); const http = require('http'); const express = require('express'); const createAuthRouter = require('./index'); @@ -49,6 +49,27 @@ describe('session user deserialization', () => { }); }); +describe('authentication provider requests', () => { + it('sets a deadline on provider requests', async () => { + const originalFetch = global.fetch; + global.fetch = jest.fn().mockResolvedValue({ ok: true }); + + try { + await fetchFromAuthProvider('https://wca.example/oauth/token', { method: 'POST' }); + + expect(global.fetch).toHaveBeenCalledWith( + 'https://wca.example/oauth/token', + expect.objectContaining({ + method: 'POST', + signal: expect.any(AbortSignal), + }), + ); + } finally { + global.fetch = originalFetch; + } + }); +}); + describe('authentication rate limiting', () => { let server; diff --git a/server/gracefulShutdown.js b/server/gracefulShutdown.js new file mode 100644 index 0000000..4e08c61 --- /dev/null +++ b/server/gracefulShutdown.js @@ -0,0 +1,49 @@ +const registerGracefulShutdown = ({ + name, + close, + logger, + timeoutMs = 25000, + processRef = process, + setTimer = setTimeout, + clearTimer = clearTimeout, +}) => { + let shutdownPromise; + + const shutDown = (signal) => { + if (shutdownPromise) { + return shutdownPromise; + } + + logger.info(`[${name}] Shutting down`, { signal }); + shutdownPromise = new Promise((resolve) => { + const timeout = setTimer(() => { + logger.error(`[${name}] Graceful shutdown timed out`); + processRef.exit(1); + resolve(false); + }, timeoutMs); + + Promise.resolve() + .then(close) + .then(() => { + clearTimer(timeout); + processRef.exit(0); + resolve(true); + }) + .catch((err) => { + clearTimer(timeout); + logger.error(err); + processRef.exit(1); + resolve(false); + }); + }); + + return shutdownPromise; + }; + + processRef.once('SIGTERM', () => shutDown('SIGTERM')); + processRef.once('SIGINT', () => shutDown('SIGINT')); + + return shutDown; +}; + +module.exports = { registerGracefulShutdown }; diff --git a/server/gracefulShutdown.test.js b/server/gracefulShutdown.test.js new file mode 100644 index 0000000..a4ac938 --- /dev/null +++ b/server/gracefulShutdown.test.js @@ -0,0 +1,70 @@ +/* eslint-env jest */ + +const { registerGracefulShutdown } = require('./gracefulShutdown'); + +const flushPromises = async () => { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); +}; + +describe('graceful shutdown', () => { + let close; + let logger; + let processRef; + let signals; + + beforeEach(() => { + jest.useFakeTimers(); + close = jest.fn().mockResolvedValue(undefined); + logger = { info: jest.fn(), error: jest.fn() }; + signals = {}; + processRef = { + once: jest.fn((signal, handler) => { signals[signal] = handler; }), + exit: jest.fn(), + }; + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('closes once and exits successfully', async () => { + const shutDown = registerGracefulShutdown({ + name: 'TEST', close, logger, processRef, + }); + + signals.SIGTERM(); + signals.SIGINT(); + await shutDown('SIGTERM'); + + expect(close).toHaveBeenCalledTimes(1); + expect(processRef.exit).toHaveBeenCalledWith(0); + }); + + it('exits with a failure when cleanup fails', async () => { + close.mockRejectedValue(new Error('cleanup failed')); + const shutDown = registerGracefulShutdown({ + name: 'TEST', close, logger, processRef, + }); + + await expect(shutDown('SIGTERM')).resolves.toBe(false); + + expect(logger.error).toHaveBeenCalledWith(expect.any(Error)); + expect(processRef.exit).toHaveBeenCalledWith(1); + }); + + it('forces an exit when cleanup exceeds the deadline', async () => { + close.mockReturnValue(new Promise(() => {})); + const shutDown = registerGracefulShutdown({ + name: 'TEST', close, logger, processRef, timeoutMs: 1000, + }); + + shutDown('SIGTERM'); + await flushPromises(); + jest.advanceTimersByTime(1000); + + expect(processRef.exit).toHaveBeenCalledWith(1); + expect(logger.error).toHaveBeenCalledWith('[TEST] Graceful shutdown timed out'); + }); +}); diff --git a/server/index.js b/server/index.js index 7ead393..da6b421 100644 --- a/server/index.js +++ b/server/index.js @@ -9,7 +9,12 @@ const rateLimit = require('express-rate-limit'); const config = require('./runtimeConfig'); const { createHealthHandler, createHealthReporter } = require('./health'); -const { initializePostgres, pool, startPostgresMaintenance } = require('./postgres'); +const { + initializePostgres, + pool, + startPostgresMaintenance, + stopPostgresMaintenance, +} = require('./postgres'); const session = require('./middlewares/session'); const { createCorsOptions } = require('./middlewares/cors'); const { createClientAssetsRouter } = require('./middlewares/clientAssets'); @@ -18,6 +23,7 @@ const logger = require('./logger'); const auth = require('./auth'); const api = require('./api'); const { isUserApiRequest } = require('./requestLogging'); +const { registerGracefulShutdown } = require('./gracefulShutdown'); Error.stackTraceLimit = 100; @@ -93,7 +99,7 @@ const init = async () => { app.use(createClientAssetsRouter(path.join(__dirname, '../client/build'))); - app.listen(config.server.port, '0.0.0.0', (err) => { + const server = app.listen(config.server.port, '0.0.0.0', (err) => { if (err) { logger.error(err); } @@ -103,12 +109,25 @@ const init = async () => { }); }); + registerGracefulShutdown({ + name: 'API', + logger, + close: async () => { + stopPostgresMaintenance(); + await new Promise((resolve, reject) => { + server.close((err) => (err ? reject(err) : resolve())); + }); + await pool.end(); + }, + }); + return app; }; -try { - const app = init(); - module.exports = app; -} catch (e) { +const app = init().catch((e) => { logger.error(e); -} + process.exitCode = 1; + throw e; +}); + +module.exports = app; diff --git a/server/postgres/index.js b/server/postgres/index.js index 11f520e..cfa60a4 100644 --- a/server/postgres/index.js +++ b/server/postgres/index.js @@ -40,6 +40,7 @@ let initialized = false; let initializationPromise; let lastFailureAt = 0; let maintenancePromise; +let maintenanceTimer; const initializePostgres = async () => { if (!config.postgres.enabled) { @@ -163,8 +164,17 @@ const runPostgresMaintenance = () => { const startPostgresMaintenance = () => { runPostgresMaintenance(); - const timer = setInterval(runPostgresMaintenance, MAINTENANCE_INTERVAL_MS); - timer.unref(); + if (!maintenanceTimer) { + maintenanceTimer = setInterval(runPostgresMaintenance, MAINTENANCE_INTERVAL_MS); + maintenanceTimer.unref(); + } +}; + +const stopPostgresMaintenance = () => { + if (maintenanceTimer) { + clearInterval(maintenanceTimer); + maintenanceTimer = undefined; + } }; module.exports = { @@ -175,5 +185,6 @@ module.exports = { query, runPostgresMaintenance, startPostgresMaintenance, + stopPostgresMaintenance, withTransaction, }; diff --git a/server/postgres/sqlRoomModel.js b/server/postgres/sqlRoomModel.js index 3f04027..5638ab7 100644 --- a/server/postgres/sqlRoomModel.js +++ b/server/postgres/sqlRoomModel.js @@ -459,6 +459,26 @@ const find = (criteria = {}) => new SqlQuery(async () => { const findById = (id) => new SqlQuery(() => loadRoom(id)); +const listActiveParticipants = async () => { + const result = await pool.query(` + SELECT r.mongo_id AS room_id, u.wca_user_id AS user_id, + rp.presence_revision + FROM app.room_participants rp + JOIN app.rooms r ON r.id = rp.room_id + JOIN app.users u ON u.id = rp.user_id + WHERE rp.in_room = true + AND r.deleted_at IS NULL + AND (r.expires_at IS NULL OR r.expires_at > now()) + ORDER BY r.id, rp.user_id + `); + + return result.rows.map((row) => ({ + roomId: row.room_id, + userId: row.user_id, + presenceRevision: row.presence_revision || 0, + })); +}; + const deleteOne = async (criteria) => { const id = externalRoomId(criteria._id || criteria.id); const result = await pool.query(` @@ -472,8 +492,18 @@ const deleteOne = async (criteria) => { const Room = RoomDocument; Room.find = find; Room.findById = findById; +Room.listActiveParticipants = listActiveParticipants; Room.deleteOne = deleteOne; Room.isSqlRepository = true; Room.cache = roomCache; -module.exports = { Room, RoomDocument, find, findById, loadRoom, saveRoom, selectRoomAdmin }; +module.exports = { + Room, + RoomDocument, + find, + findById, + listActiveParticipants, + loadRoom, + saveRoom, + selectRoomAdmin, +}; diff --git a/server/postgres/sqlRoomModel.test.js b/server/postgres/sqlRoomModel.test.js new file mode 100644 index 0000000..2b280a1 --- /dev/null +++ b/server/postgres/sqlRoomModel.test.js @@ -0,0 +1,36 @@ +/* eslint-env jest */ + +const mockQuery = jest.fn(); + +jest.mock('./index', () => ({ + pool: { query: mockQuery }, +})); + +jest.mock('letscube-scrambles', () => ({ + generateScramble: jest.fn(), +})); + +const { listActiveParticipants } = require('./sqlRoomModel'); + +describe('SQL room presence queries', () => { + beforeEach(() => { + mockQuery.mockReset(); + }); + + it('loads only the fields needed to reconcile active participants', async () => { + mockQuery.mockResolvedValue({ + rows: [ + { room_id: 'room-one', user_id: 101, presence_revision: 4 }, + { room_id: 'room-one', user_id: 202, presence_revision: null }, + ], + }); + + await expect(listActiveParticipants()).resolves.toEqual([ + { roomId: 'room-one', userId: 101, presenceRevision: 4 }, + { roomId: 'room-one', userId: 202, presenceRevision: 0 }, + ]); + + expect(mockQuery).toHaveBeenCalledWith(expect.stringContaining('WHERE rp.in_room = true')); + expect(mockQuery.mock.calls[0][0]).not.toContain('race_sessions'); + }); +}); diff --git a/server/socket/index.js b/server/socket/index.js index b063bcc..731d7ba 100644 --- a/server/socket/index.js +++ b/server/socket/index.js @@ -15,6 +15,7 @@ const { isFeatureEnabled } = require('../features'); const wrapExpressMiddleware = require('./middlewares/wrapExpressMiddleware'); const initRooms = require('./namespaces/rooms'); const initDefault = require('./namespaces/default'); +const { registerGracefulShutdown } = require('../gracefulShutdown'); const logSocketError = (source) => (err) => { if (err) { @@ -107,7 +108,30 @@ const init = async () => { watchNamespaceErrors(io, '/'); watchNamespaceErrors(io, '/rooms'); - return server.listen(process.env.SOCKETIO_PORT || config.socketio.port); + server.listen(process.env.SOCKETIO_PORT || config.socketio.port); + + registerGracefulShutdown({ + name: 'SOCKET', + logger, + close: async () => { + await new Promise((resolve) => io.close(resolve)); + const redisResults = await Promise.allSettled([ + pubClient.quit(), + subClient.quit(), + socialSubClient ? socialSubClient.quit() : Promise.resolve(), + ]); + await pool.end(); + const redisFailure = redisResults.find(({ status }) => status === 'rejected'); + if (redisFailure) { + throw redisFailure.reason; + } + }, + }); + + return server; }; -init(); +init().catch((err) => { + logger.error(err); + process.exit(1); +}); diff --git a/server/socket/namespaces/rooms.js b/server/socket/namespaces/rooms.js index f83ce01..2dd9a9e 100644 --- a/server/socket/namespaces/rooms.js +++ b/server/socket/namespaces/rooms.js @@ -86,18 +86,11 @@ module.exports = (io, middlewares) => { ); async function listActiveRoomUsers() { - const rooms = await Room.find() - .populate('users') - .populate('admin') - .populate('owner'); - - return rooms.flatMap((room) => room.usersInRoom.map((user) => ({ - roomId: room._id, - userId: user.id, - membershipRevision: room.membershipRevision, - presenceRevision: room.presenceRevision.get(user.id.toString()), + const participants = await Room.listActiveParticipants(); + return participants.map((participant) => ({ + ...participant, leaveReason: 'disconnect', - }))); + })); } async function createAndSendNewScramble(room) { @@ -140,8 +133,9 @@ module.exports = (io, middlewares) => { const room = await fetchRoom(roomId); if (!room || !room.inRoom.get(userKey) - || room.membershipRevision !== membershipRevision - || room.presenceRevision.get(userKey) !== presenceRevision) { + || (membershipRevision !== undefined && room.membershipRevision !== membershipRevision) + || (presenceRevision !== undefined + && room.presenceRevision.get(userKey) !== presenceRevision)) { return false; } diff --git a/server/socket/namespaces/rooms.test.js b/server/socket/namespaces/rooms.test.js index 3d47d61..178eca3 100644 --- a/server/socket/namespaces/rooms.test.js +++ b/server/socket/namespaces/rooms.test.js @@ -26,6 +26,7 @@ jest.mock('../../models', () => ({ Room: { find: jest.fn(), findById: jest.fn(), + listActiveParticipants: jest.fn(), }, User: { find: jest.fn(), @@ -139,6 +140,7 @@ const setup = (rooms) => { }); Room.find.mockImplementation(() => queryResult([])); Room.findById.mockImplementation(({ _id }) => queryResult(rooms.get(String(_id)))); + Room.listActiveParticipants.mockResolvedValue([]); User.find.mockResolvedValue([]); initRooms(io, []); @@ -459,6 +461,33 @@ describe('room namespace departures and moderation', () => { expect(connect.roomChannel.emit).toHaveBeenCalledWith(Protocol.USER_LEFT, 101); }); + it('uses the per-user revision when reconciliation sees a room-wide change', async () => { + const room = makeRoom({ + private: false, + password: null, + inRoom: new Map([['101', true]]), + membershipRevision: 9, + presenceRevision: new Map([['101', 7]]), + doneWithScramble: jest.fn().mockReturnValue(false), + }); + room.dropUserAtomically = jest.fn().mockResolvedValue({ + room: { ...room, usersLength: 0 }, + adminChanged: false, + }); + const connect = setup(new Map([[room._id, room]])); + await connect(makeSocket({ id: 'spectator-socket', session: {} })); + const reconnectGrace = createReconnectGrace.mock.results[0].value; + + await expect(reconnectGrace.finalizeDeparture({ + roomId: room._id, + userId: 101, + presenceRevision: 7, + leaveReason: 'disconnect', + })).resolves.toBe(true); + + expect(room.dropUserAtomically).toHaveBeenCalledWith({ id: 101 }, undefined, 7); + }); + it('fences an active duplicate join before an older departure can finalize', async () => { const room = makeRoom({ private: false,