Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions docs/realtime.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,13 @@ checks for another active socket for the same user and room through the Redis
adapter. This prevents a deploy or second browser tab from producing a false
leave.

The server also reconciles durable room presence after startup and every five
minutes. Reconciliation processes two departures at a time so it does not
exhaust the PostgreSQL connection pool. A departure that throws an error gets
two retries.
Each retry checks live socket presence and the durable presence revision again.
This prevents an old cleanup attempt from removing a user who reconnected.

During the grace window, preserve:

- room membership;
Expand Down
6 changes: 5 additions & 1 deletion docs/room-ownership.md
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,17 @@ itself from the per-user socket group before the remaining-tab check, so two
simultaneous explicit leaves cannot strand membership. If the owner later
rejoins, they reclaim admin before the join is acknowledged.

The final persisted departure is a MongoDB compare-and-set keyed by room
The final persisted departure is a PostgreSQL compare-and-set keyed by room
membership and the active user's presence revision. This applies across
Socket.IO processes: exactly one process can claim a leave, emit its room
events, and record its metrics. Every authenticated duplicate join advances
that user's durable presence revision before acknowledgment, so an older leave
is terminal and cannot overwrite an active new tab.

Startup and periodic reconciliation remove durable presence left by a replaced
Socket.IO process. Reconciliation limits concurrent departures and retries
transient failures. Every retry repeats the socket and revision safeguards.

An empty room persists `admin: null`, but the established `UPDATE_ADMIN`
Socket.IO event is emitted only for a non-null active admin. Spectators and
other clients therefore never receive a null admin payload.
Expand Down
55 changes: 48 additions & 7 deletions server/socket/lib/reconnectGrace.js
Original file line number Diff line number Diff line change
@@ -1,11 +1,20 @@
const timerKey = (roomId, userId) => `${roomId}:${userId}`;

const DEFAULT_RECONCILIATION_CONCURRENCY = 2;
const DEFAULT_RECONCILIATION_INTERVAL_MS = 5 * 60 * 1000;
const DEFAULT_MAX_FINALIZATION_RETRIES = 2;
const DEFAULT_FINALIZATION_RETRY_DELAY_MS = 1000;

const createReconnectGrace = ({
graceMs,
hasActiveSockets,
finalizeDeparture,
listActiveUsers,
logger,
reconciliationConcurrency = DEFAULT_RECONCILIATION_CONCURRENCY,
reconciliationIntervalMs = DEFAULT_RECONCILIATION_INTERVAL_MS,
maxFinalizationRetries = DEFAULT_MAX_FINALIZATION_RETRIES,
finalizationRetryDelayMs = DEFAULT_FINALIZATION_RETRY_DELAY_MS,
setTimer = setTimeout,
clearTimer = clearTimeout,
}) => {
Expand Down Expand Up @@ -48,8 +57,28 @@ const createReconnectGrace = ({
return current;
};

const finalizeIfInactive = (departure) => enqueueFinalization(departure, true);
const finalize = (departure) => enqueueFinalization(departure, false);
const wait = (delayMs) => new Promise((resolve) => {
const timer = setTimer(resolve, delayMs);
if (timer && timer.unref) {
timer.unref();
}
});

const finalizeWithRetry = async (departure, requireInactive, retry = 0) => {
try {
return await enqueueFinalization(departure, requireInactive);
} catch (err) {
if (retry >= maxFinalizationRetries) {
throw err;
}

await wait(finalizationRetryDelayMs * (retry + 1));
return finalizeWithRetry(departure, requireInactive, retry + 1);
}
};

const finalizeIfInactive = (departure) => finalizeWithRetry(departure, true);
const finalize = (departure) => finalizeWithRetry(departure, false);

const schedule = (departure) => {
cancel(departure.roomId, departure.userId);
Expand All @@ -69,13 +98,23 @@ const createReconnectGrace = ({

const reconcile = async () => {
const activeUsers = await listActiveUsers();
await Promise.all(activeUsers.map((departure) => finalizeIfInactive(departure)));
let nextIndex = 0;
const worker = async () => {
while (nextIndex < activeUsers.length) {
const departure = activeUsers[nextIndex];
nextIndex += 1;
await finalizeIfInactive(departure).catch((err) => logger.error(err));
}
};
const workerCount = Math.min(reconciliationConcurrency, activeUsers.length);
await Promise.all(Array.from({ length: workerCount }, worker));
};

const startReconciliation = () => {
const timer = setTimer(() => {
reconcile().catch((err) => logger.error(err));
}, graceMs);
const scheduleReconciliation = (delayMs) => {
const timer = setTimer(async () => {
await reconcile().catch((err) => logger.error(err));
scheduleReconciliation(reconciliationIntervalMs);
}, delayMs);

if (timer && timer.unref) {
timer.unref();
Expand All @@ -84,6 +123,8 @@ const createReconnectGrace = ({
return timer;
};

const startReconciliation = () => scheduleReconciliation(graceMs);

return {
cancel,
finalize,
Expand Down
107 changes: 106 additions & 1 deletion server/socket/lib/reconnectGrace.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,24 @@ describe('reconnect grace', () => {
let listActiveUsers;
let logger;

const createManager = (graceMs = 30000) => createReconnectGrace({
const createManager = (graceMs = 30000, options = {}) => createReconnectGrace({
graceMs,
hasActiveSockets,
finalizeDeparture,
listActiveUsers,
logger,
...options,
});

const flushPromises = async () => {
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
await Promise.resolve();
};

beforeEach(() => {
jest.useFakeTimers();
hasActiveSockets = jest.fn().mockResolvedValue(false);
Expand Down Expand Up @@ -142,6 +152,83 @@ describe('reconnect grace', () => {
expect(finalizeDeparture).toHaveBeenCalledWith(departure);
});

it('limits reconciliation concurrency across rooms', async () => {
const departures = Array.from({ length: 5 }, (_, index) => ({
...departure,
roomId: `room-${index}`,
userId: 200 + index,
}));
let releaseFinalizations;
const finalizationsBlocked = new Promise((resolve) => {
releaseFinalizations = resolve;
});
let activeFinalizations = 0;
let peakFinalizations = 0;
listActiveUsers.mockResolvedValue(departures);
finalizeDeparture.mockImplementation(async () => {
activeFinalizations += 1;
peakFinalizations = Math.max(peakFinalizations, activeFinalizations);
await finalizationsBlocked;
activeFinalizations -= 1;
return true;
});
const manager = createManager(30000, { reconciliationConcurrency: 2 });

const reconciliation = manager.reconcile();
await flushPromises();

expect(finalizeDeparture).toHaveBeenCalledTimes(2);

releaseFinalizations();
await reconciliation;

expect(finalizeDeparture).toHaveBeenCalledTimes(5);
expect(peakFinalizations).toBe(2);
});

it('retries a failed disconnect cleanup after checking socket activity again', async () => {
finalizeDeparture
.mockRejectedValueOnce(new Error('database unavailable'))
.mockResolvedValueOnce(true);
const manager = createManager(30000, {
maxFinalizationRetries: 1,
finalizationRetryDelayMs: 5000,
});

manager.schedule(departure);
jest.advanceTimersByTime(30000);
await flushPromises();

expect(finalizeDeparture).toHaveBeenCalledTimes(1);
expect(hasActiveSockets).toHaveBeenCalledTimes(1);

jest.advanceTimersByTime(5000);
await flushPromises();

expect(finalizeDeparture).toHaveBeenCalledTimes(2);
expect(hasActiveSockets).toHaveBeenCalledTimes(2);
});

it('does not retry departure after the user reconnects', async () => {
hasActiveSockets
.mockResolvedValueOnce(false)
.mockResolvedValueOnce(true);
finalizeDeparture.mockRejectedValueOnce(new Error('database unavailable'));
const manager = createManager(30000, {
maxFinalizationRetries: 1,
finalizationRetryDelayMs: 5000,
});

manager.schedule(departure);
jest.advanceTimersByTime(30000);
await flushPromises();
jest.advanceTimersByTime(5000);
await flushPromises();

expect(hasActiveSockets).toHaveBeenCalledTimes(2);
expect(finalizeDeparture).toHaveBeenCalledTimes(1);
});

it('serializes cleanup for users in the same room', async () => {
let finishFirstDeparture;
let markFirstStarted;
Expand Down Expand Up @@ -184,4 +271,22 @@ describe('reconnect grace', () => {

expect(listActiveUsers).toHaveBeenCalledTimes(1);
});

it('repeats reconciliation after the configured interval', async () => {
const manager = createManager(30000, { reconciliationIntervalMs: 60000 });

manager.startReconciliation();
jest.advanceTimersByTime(30000);
await flushPromises();

expect(listActiveUsers).toHaveBeenCalledTimes(1);

jest.advanceTimersByTime(59999);
expect(listActiveUsers).toHaveBeenCalledTimes(1);

jest.advanceTimersByTime(1);
await flushPromises();

expect(listActiveUsers).toHaveBeenCalledTimes(2);
});
});
Loading