From ab57937c800003313e29d48aab3a69bf79981c43 Mon Sep 17 00:00:00 2001 From: Cailyn Sinclair Date: Tue, 17 Nov 2026 16:58:35 -0800 Subject: [PATCH] Make room presence cleanup resilient Limit reconciliation concurrency to protect the PostgreSQL pool. Retry transient departure errors and repeat reconciliation so stale presence cannot persist indefinitely. --- docs/realtime.md | 7 ++ docs/room-ownership.md | 6 +- server/socket/lib/reconnectGrace.js | 55 ++++++++++-- server/socket/lib/reconnectGrace.test.js | 107 ++++++++++++++++++++++- 4 files changed, 166 insertions(+), 9 deletions(-) diff --git a/docs/realtime.md b/docs/realtime.md index 83ddceab..00a652cf 100644 --- a/docs/realtime.md +++ b/docs/realtime.md @@ -62,6 +62,13 @@ checks for another active socket for the same user and room through the Redis adapter. This prevents a deploy or second browser tab from producing a false leave. +The server also reconciles durable room presence after startup and every five +minutes. Reconciliation processes two departures at a time so it does not +exhaust the PostgreSQL connection pool. A departure that throws an error gets +two retries. +Each retry checks live socket presence and the durable presence revision again. +This prevents an old cleanup attempt from removing a user who reconnected. + During the grace window, preserve: - room membership; diff --git a/docs/room-ownership.md b/docs/room-ownership.md index 3b1297ee..fa8a6cdf 100644 --- a/docs/room-ownership.md +++ b/docs/room-ownership.md @@ -34,13 +34,17 @@ itself from the per-user socket group before the remaining-tab check, so two simultaneous explicit leaves cannot strand membership. If the owner later rejoins, they reclaim admin before the join is acknowledged. -The final persisted departure is a MongoDB compare-and-set keyed by room +The final persisted departure is a PostgreSQL compare-and-set keyed by room membership and the active user's presence revision. This applies across Socket.IO processes: exactly one process can claim a leave, emit its room events, and record its metrics. Every authenticated duplicate join advances that user's durable presence revision before acknowledgment, so an older leave is terminal and cannot overwrite an active new tab. +Startup and periodic reconciliation remove durable presence left by a replaced +Socket.IO process. Reconciliation limits concurrent departures and retries +transient failures. Every retry repeats the socket and revision safeguards. + An empty room persists `admin: null`, but the established `UPDATE_ADMIN` Socket.IO event is emitted only for a non-null active admin. Spectators and other clients therefore never receive a null admin payload. diff --git a/server/socket/lib/reconnectGrace.js b/server/socket/lib/reconnectGrace.js index 162b2ed9..23fbe586 100644 --- a/server/socket/lib/reconnectGrace.js +++ b/server/socket/lib/reconnectGrace.js @@ -1,11 +1,20 @@ const timerKey = (roomId, userId) => `${roomId}:${userId}`; +const DEFAULT_RECONCILIATION_CONCURRENCY = 2; +const DEFAULT_RECONCILIATION_INTERVAL_MS = 5 * 60 * 1000; +const DEFAULT_MAX_FINALIZATION_RETRIES = 2; +const DEFAULT_FINALIZATION_RETRY_DELAY_MS = 1000; + const createReconnectGrace = ({ graceMs, hasActiveSockets, finalizeDeparture, listActiveUsers, logger, + reconciliationConcurrency = DEFAULT_RECONCILIATION_CONCURRENCY, + reconciliationIntervalMs = DEFAULT_RECONCILIATION_INTERVAL_MS, + maxFinalizationRetries = DEFAULT_MAX_FINALIZATION_RETRIES, + finalizationRetryDelayMs = DEFAULT_FINALIZATION_RETRY_DELAY_MS, setTimer = setTimeout, clearTimer = clearTimeout, }) => { @@ -48,8 +57,28 @@ const createReconnectGrace = ({ return current; }; - const finalizeIfInactive = (departure) => enqueueFinalization(departure, true); - const finalize = (departure) => enqueueFinalization(departure, false); + const wait = (delayMs) => new Promise((resolve) => { + const timer = setTimer(resolve, delayMs); + if (timer && timer.unref) { + timer.unref(); + } + }); + + const finalizeWithRetry = async (departure, requireInactive, retry = 0) => { + try { + return await enqueueFinalization(departure, requireInactive); + } catch (err) { + if (retry >= maxFinalizationRetries) { + throw err; + } + + await wait(finalizationRetryDelayMs * (retry + 1)); + return finalizeWithRetry(departure, requireInactive, retry + 1); + } + }; + + const finalizeIfInactive = (departure) => finalizeWithRetry(departure, true); + const finalize = (departure) => finalizeWithRetry(departure, false); const schedule = (departure) => { cancel(departure.roomId, departure.userId); @@ -69,13 +98,23 @@ const createReconnectGrace = ({ const reconcile = async () => { const activeUsers = await listActiveUsers(); - await Promise.all(activeUsers.map((departure) => finalizeIfInactive(departure))); + let nextIndex = 0; + const worker = async () => { + while (nextIndex < activeUsers.length) { + const departure = activeUsers[nextIndex]; + nextIndex += 1; + await finalizeIfInactive(departure).catch((err) => logger.error(err)); + } + }; + const workerCount = Math.min(reconciliationConcurrency, activeUsers.length); + await Promise.all(Array.from({ length: workerCount }, worker)); }; - const startReconciliation = () => { - const timer = setTimer(() => { - reconcile().catch((err) => logger.error(err)); - }, graceMs); + const scheduleReconciliation = (delayMs) => { + const timer = setTimer(async () => { + await reconcile().catch((err) => logger.error(err)); + scheduleReconciliation(reconciliationIntervalMs); + }, delayMs); if (timer && timer.unref) { timer.unref(); @@ -84,6 +123,8 @@ const createReconnectGrace = ({ return timer; }; + const startReconciliation = () => scheduleReconciliation(graceMs); + return { cancel, finalize, diff --git a/server/socket/lib/reconnectGrace.test.js b/server/socket/lib/reconnectGrace.test.js index 0d5cf8b3..e43be508 100644 --- a/server/socket/lib/reconnectGrace.test.js +++ b/server/socket/lib/reconnectGrace.test.js @@ -14,14 +14,24 @@ describe('reconnect grace', () => { let listActiveUsers; let logger; - const createManager = (graceMs = 30000) => createReconnectGrace({ + const createManager = (graceMs = 30000, options = {}) => createReconnectGrace({ graceMs, hasActiveSockets, finalizeDeparture, listActiveUsers, logger, + ...options, }); + const flushPromises = async () => { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }; + beforeEach(() => { jest.useFakeTimers(); hasActiveSockets = jest.fn().mockResolvedValue(false); @@ -142,6 +152,83 @@ describe('reconnect grace', () => { expect(finalizeDeparture).toHaveBeenCalledWith(departure); }); + it('limits reconciliation concurrency across rooms', async () => { + const departures = Array.from({ length: 5 }, (_, index) => ({ + ...departure, + roomId: `room-${index}`, + userId: 200 + index, + })); + let releaseFinalizations; + const finalizationsBlocked = new Promise((resolve) => { + releaseFinalizations = resolve; + }); + let activeFinalizations = 0; + let peakFinalizations = 0; + listActiveUsers.mockResolvedValue(departures); + finalizeDeparture.mockImplementation(async () => { + activeFinalizations += 1; + peakFinalizations = Math.max(peakFinalizations, activeFinalizations); + await finalizationsBlocked; + activeFinalizations -= 1; + return true; + }); + const manager = createManager(30000, { reconciliationConcurrency: 2 }); + + const reconciliation = manager.reconcile(); + await flushPromises(); + + expect(finalizeDeparture).toHaveBeenCalledTimes(2); + + releaseFinalizations(); + await reconciliation; + + expect(finalizeDeparture).toHaveBeenCalledTimes(5); + expect(peakFinalizations).toBe(2); + }); + + it('retries a failed disconnect cleanup after checking socket activity again', async () => { + finalizeDeparture + .mockRejectedValueOnce(new Error('database unavailable')) + .mockResolvedValueOnce(true); + const manager = createManager(30000, { + maxFinalizationRetries: 1, + finalizationRetryDelayMs: 5000, + }); + + manager.schedule(departure); + jest.advanceTimersByTime(30000); + await flushPromises(); + + expect(finalizeDeparture).toHaveBeenCalledTimes(1); + expect(hasActiveSockets).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(5000); + await flushPromises(); + + expect(finalizeDeparture).toHaveBeenCalledTimes(2); + expect(hasActiveSockets).toHaveBeenCalledTimes(2); + }); + + it('does not retry departure after the user reconnects', async () => { + hasActiveSockets + .mockResolvedValueOnce(false) + .mockResolvedValueOnce(true); + finalizeDeparture.mockRejectedValueOnce(new Error('database unavailable')); + const manager = createManager(30000, { + maxFinalizationRetries: 1, + finalizationRetryDelayMs: 5000, + }); + + manager.schedule(departure); + jest.advanceTimersByTime(30000); + await flushPromises(); + jest.advanceTimersByTime(5000); + await flushPromises(); + + expect(hasActiveSockets).toHaveBeenCalledTimes(2); + expect(finalizeDeparture).toHaveBeenCalledTimes(1); + }); + it('serializes cleanup for users in the same room', async () => { let finishFirstDeparture; let markFirstStarted; @@ -184,4 +271,22 @@ describe('reconnect grace', () => { expect(listActiveUsers).toHaveBeenCalledTimes(1); }); + + it('repeats reconciliation after the configured interval', async () => { + const manager = createManager(30000, { reconciliationIntervalMs: 60000 }); + + manager.startReconciliation(); + jest.advanceTimersByTime(30000); + await flushPromises(); + + expect(listActiveUsers).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(59999); + expect(listActiveUsers).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(1); + await flushPromises(); + + expect(listActiveUsers).toHaveBeenCalledTimes(2); + }); });