diff --git a/docs/realtime.md b/docs/realtime.md index 83ddcea..00a652c 100644 --- a/docs/realtime.md +++ b/docs/realtime.md @@ -62,6 +62,13 @@ checks for another active socket for the same user and room through the Redis adapter. This prevents a deploy or second browser tab from producing a false leave. +The server also reconciles durable room presence after startup and every five +minutes. Reconciliation processes two departures at a time so it does not +exhaust the PostgreSQL connection pool. A departure that throws an error gets +two retries. +Each retry checks live socket presence and the durable presence revision again. +This prevents an old cleanup attempt from removing a user who reconnected. + During the grace window, preserve: - room membership; diff --git a/docs/room-ownership.md b/docs/room-ownership.md index 3b1297e..fa8a6cd 100644 --- a/docs/room-ownership.md +++ b/docs/room-ownership.md @@ -34,13 +34,17 @@ itself from the per-user socket group before the remaining-tab check, so two simultaneous explicit leaves cannot strand membership. If the owner later rejoins, they reclaim admin before the join is acknowledged. -The final persisted departure is a MongoDB compare-and-set keyed by room +The final persisted departure is a PostgreSQL compare-and-set keyed by room membership and the active user's presence revision. This applies across Socket.IO processes: exactly one process can claim a leave, emit its room events, and record its metrics. Every authenticated duplicate join advances that user's durable presence revision before acknowledgment, so an older leave is terminal and cannot overwrite an active new tab. +Startup and periodic reconciliation remove durable presence left by a replaced +Socket.IO process. Reconciliation limits concurrent departures and retries +transient failures. Every retry repeats the socket and revision safeguards. + An empty room persists `admin: null`, but the established `UPDATE_ADMIN` Socket.IO event is emitted only for a non-null active admin. Spectators and other clients therefore never receive a null admin payload. diff --git a/server/socket/lib/reconnectGrace.js b/server/socket/lib/reconnectGrace.js index 162b2ed..23fbe58 100644 --- a/server/socket/lib/reconnectGrace.js +++ b/server/socket/lib/reconnectGrace.js @@ -1,11 +1,20 @@ const timerKey = (roomId, userId) => `${roomId}:${userId}`; +const DEFAULT_RECONCILIATION_CONCURRENCY = 2; +const DEFAULT_RECONCILIATION_INTERVAL_MS = 5 * 60 * 1000; +const DEFAULT_MAX_FINALIZATION_RETRIES = 2; +const DEFAULT_FINALIZATION_RETRY_DELAY_MS = 1000; + const createReconnectGrace = ({ graceMs, hasActiveSockets, finalizeDeparture, listActiveUsers, logger, + reconciliationConcurrency = DEFAULT_RECONCILIATION_CONCURRENCY, + reconciliationIntervalMs = DEFAULT_RECONCILIATION_INTERVAL_MS, + maxFinalizationRetries = DEFAULT_MAX_FINALIZATION_RETRIES, + finalizationRetryDelayMs = DEFAULT_FINALIZATION_RETRY_DELAY_MS, setTimer = setTimeout, clearTimer = clearTimeout, }) => { @@ -48,8 +57,28 @@ const createReconnectGrace = ({ return current; }; - const finalizeIfInactive = (departure) => enqueueFinalization(departure, true); - const finalize = (departure) => enqueueFinalization(departure, false); + const wait = (delayMs) => new Promise((resolve) => { + const timer = setTimer(resolve, delayMs); + if (timer && timer.unref) { + timer.unref(); + } + }); + + const finalizeWithRetry = async (departure, requireInactive, retry = 0) => { + try { + return await enqueueFinalization(departure, requireInactive); + } catch (err) { + if (retry >= maxFinalizationRetries) { + throw err; + } + + await wait(finalizationRetryDelayMs * (retry + 1)); + return finalizeWithRetry(departure, requireInactive, retry + 1); + } + }; + + const finalizeIfInactive = (departure) => finalizeWithRetry(departure, true); + const finalize = (departure) => finalizeWithRetry(departure, false); const schedule = (departure) => { cancel(departure.roomId, departure.userId); @@ -69,13 +98,23 @@ const createReconnectGrace = ({ const reconcile = async () => { const activeUsers = await listActiveUsers(); - await Promise.all(activeUsers.map((departure) => finalizeIfInactive(departure))); + let nextIndex = 0; + const worker = async () => { + while (nextIndex < activeUsers.length) { + const departure = activeUsers[nextIndex]; + nextIndex += 1; + await finalizeIfInactive(departure).catch((err) => logger.error(err)); + } + }; + const workerCount = Math.min(reconciliationConcurrency, activeUsers.length); + await Promise.all(Array.from({ length: workerCount }, worker)); }; - const startReconciliation = () => { - const timer = setTimer(() => { - reconcile().catch((err) => logger.error(err)); - }, graceMs); + const scheduleReconciliation = (delayMs) => { + const timer = setTimer(async () => { + await reconcile().catch((err) => logger.error(err)); + scheduleReconciliation(reconciliationIntervalMs); + }, delayMs); if (timer && timer.unref) { timer.unref(); @@ -84,6 +123,8 @@ const createReconnectGrace = ({ return timer; }; + const startReconciliation = () => scheduleReconciliation(graceMs); + return { cancel, finalize, diff --git a/server/socket/lib/reconnectGrace.test.js b/server/socket/lib/reconnectGrace.test.js index 0d5cf8b..e43be50 100644 --- a/server/socket/lib/reconnectGrace.test.js +++ b/server/socket/lib/reconnectGrace.test.js @@ -14,14 +14,24 @@ describe('reconnect grace', () => { let listActiveUsers; let logger; - const createManager = (graceMs = 30000) => createReconnectGrace({ + const createManager = (graceMs = 30000, options = {}) => createReconnectGrace({ graceMs, hasActiveSockets, finalizeDeparture, listActiveUsers, logger, + ...options, }); + const flushPromises = async () => { + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }; + beforeEach(() => { jest.useFakeTimers(); hasActiveSockets = jest.fn().mockResolvedValue(false); @@ -142,6 +152,83 @@ describe('reconnect grace', () => { expect(finalizeDeparture).toHaveBeenCalledWith(departure); }); + it('limits reconciliation concurrency across rooms', async () => { + const departures = Array.from({ length: 5 }, (_, index) => ({ + ...departure, + roomId: `room-${index}`, + userId: 200 + index, + })); + let releaseFinalizations; + const finalizationsBlocked = new Promise((resolve) => { + releaseFinalizations = resolve; + }); + let activeFinalizations = 0; + let peakFinalizations = 0; + listActiveUsers.mockResolvedValue(departures); + finalizeDeparture.mockImplementation(async () => { + activeFinalizations += 1; + peakFinalizations = Math.max(peakFinalizations, activeFinalizations); + await finalizationsBlocked; + activeFinalizations -= 1; + return true; + }); + const manager = createManager(30000, { reconciliationConcurrency: 2 }); + + const reconciliation = manager.reconcile(); + await flushPromises(); + + expect(finalizeDeparture).toHaveBeenCalledTimes(2); + + releaseFinalizations(); + await reconciliation; + + expect(finalizeDeparture).toHaveBeenCalledTimes(5); + expect(peakFinalizations).toBe(2); + }); + + it('retries a failed disconnect cleanup after checking socket activity again', async () => { + finalizeDeparture + .mockRejectedValueOnce(new Error('database unavailable')) + .mockResolvedValueOnce(true); + const manager = createManager(30000, { + maxFinalizationRetries: 1, + finalizationRetryDelayMs: 5000, + }); + + manager.schedule(departure); + jest.advanceTimersByTime(30000); + await flushPromises(); + + expect(finalizeDeparture).toHaveBeenCalledTimes(1); + expect(hasActiveSockets).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(5000); + await flushPromises(); + + expect(finalizeDeparture).toHaveBeenCalledTimes(2); + expect(hasActiveSockets).toHaveBeenCalledTimes(2); + }); + + it('does not retry departure after the user reconnects', async () => { + hasActiveSockets + .mockResolvedValueOnce(false) + .mockResolvedValueOnce(true); + finalizeDeparture.mockRejectedValueOnce(new Error('database unavailable')); + const manager = createManager(30000, { + maxFinalizationRetries: 1, + finalizationRetryDelayMs: 5000, + }); + + manager.schedule(departure); + jest.advanceTimersByTime(30000); + await flushPromises(); + jest.advanceTimersByTime(5000); + await flushPromises(); + + expect(hasActiveSockets).toHaveBeenCalledTimes(2); + expect(finalizeDeparture).toHaveBeenCalledTimes(1); + }); + it('serializes cleanup for users in the same room', async () => { let finishFirstDeparture; let markFirstStarted; @@ -184,4 +271,22 @@ describe('reconnect grace', () => { expect(listActiveUsers).toHaveBeenCalledTimes(1); }); + + it('repeats reconciliation after the configured interval', async () => { + const manager = createManager(30000, { reconciliationIntervalMs: 60000 }); + + manager.startReconciliation(); + jest.advanceTimersByTime(30000); + await flushPromises(); + + expect(listActiveUsers).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(59999); + expect(listActiveUsers).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(1); + await flushPromises(); + + expect(listActiveUsers).toHaveBeenCalledTimes(2); + }); });