feat(ssr): opt-in loopback endpoint listing the SSR cache contents

.github/workflows/build.yml

@@ -83,6 +83,9 @@ jobs:
       - name: Run lint
         run: yarn run lint --quiet
 
+      - name: Run SSR cache stats tests
+        run: yarn run test:ssr-cache-stats
+
       - name: Check for circular dependencies
         run: yarn run check-circ-deps
 

config/config.example.yml

@@ -97,6 +97,16 @@ cache:
   serverSide:
     # Set to true to see all cache hits/misses/refreshes in your console logs. Useful for debugging SSR caching issues.
     debug: false
+    # Lets operators on this machine see what the caches below hold: GET /app/cache-stats (JSON).
+    # Each Node process reports its own cache. Off by default.
+    stats:
+      enabled: false
+      # 0 = answer on the UI port, but only to requests made directly from this host: connecting from 127.0.0.1 and
+      # carrying no proxy headers (Forwarded, Via, X-Forwarded-*, X-Real-IP). Everyone else is handled by the app like any
+      # unknown path. A same-host reverse proxy that adds none of those headers would pass this check, so use a port number there.
+      # A port number (e.g. 4001) = listen on 127.0.0.1:<port> instead; under pm2 cluster mode worker N uses <port> + N,
+      # which is the only way to pick a specific worker.
+      port: 0
     # When enabled (i.e. max > 0), known bots will be sent pages from a server side cache specific for bots.
     # (Keep in mind, bot detection cannot be guarranteed. It is possible some bots will bypass this cache.)
     botCache:

package.json

@@ -5,6 +5,7 @@
     "ng": "ng",
     "config:watch": "nodemon",
     "test:rest": "ts-node --project ./tsconfig.ts-node.json scripts/test-rest.ts",
+    "test:ssr-cache-stats": "node --test scripts/test-ssr-cache-stats.js",
     "start": "yarn run start:prod",
     "start:dev": "nodemon --exec \"cross-env NODE_ENV=development yarn run serve\"",
     "start:prod": "yarn run build:prod && cross-env NODE_ENV=production yarn run serve:ssr",

scripts/test-ssr-cache-stats.js

@@ -0,0 +1,120 @@
+const assert = require('node:assert/strict');
+const { once } = require('node:events');
+const { test } = require('node:test');
+const { setTimeout: sleep } = require('node:timers/promises');
+const LRU = require('lru-cache');
+
+process.env.TS_NODE_PROJECT = 'tsconfig.ts-node.json';
+require('ts-node/register/transpile-only');
+const {
+  snapshotCache, startSsrCacheStatsServer, isDirectLoopbackRequest
+} = require('../src/backend/ssr-cache-stats.ts');
+
+const statsOf = (caches) => ({
+  host: 'node-1', pid: 1, instance: 0, generatedAt: 'now', uptimeSeconds: 2,
+  memory: { rss: 1, heapUsed: 1 }, caches
+});
+
+test('snapshot reports entries, HTML byte sizes and TTL without changing LRU order', () => {
+  const cache = new LRU({ max: 3, ttl: 60000 });
+  cache.set('/small', { page: 'é', headers: { 'content-type': 'text/html' } });
+  cache.set('/large', { page: 'larger', headers: {} });
+  const before = [...cache.keys()];
+
+  const stats = snapshotCache('bot', cache, true);
+
+  assert.equal(stats.entryCount, 2);
+  assert.equal(stats.totalHtmlBytes, 8);
+  assert.deepEqual(stats.entries.map(e => e.key), ['/large', '/small']);
+  assert.equal(stats.maxEntries, 3);
+  assert.equal(stats.configuredTtlMs, 60000);
+  assert.equal(stats.entries[0].remainingTtlMs > 0, true);
+  assert.equal(stats.entries[0].stale, false);
+  assert.deepEqual([...cache.keys()], before);
+});
+
+test('snapshot counts expired entries even when the cache does not serve stale pages', async () => {
+  const cache = new LRU({ max: 2, ttl: 1, allowStale: false });
+  cache.set('/expired', { page: 'x' });
+  await sleep(5);
+  assert.equal(cache.size, 1);
+
+  const stats = snapshotCache('anonymous', cache, true);
+
+  assert.equal(stats.entryCount, 1);
+  assert.equal(stats.staleCount, 1);
+  assert.deepEqual(stats.entries[0], { key: '/expired', htmlBytes: 1, remainingTtlMs: 0, stale: true });
+  assert.equal(cache.size, 1);
+});
+
+test('snapshot of a missing cache and of a cache without TTL', () => {
+  assert.deepEqual(snapshotCache('bot', undefined, false), {
+    name: 'bot', enabled: false, maxEntries: 0, configuredTtlMs: 0, entryCount: 0, staleCount: 0, totalHtmlBytes: 0, entries: []
+  });
+  const cache = new LRU({ max: 1 });
+  cache.set('/forever', { page: 'x' });
+  assert.equal(snapshotCache('bot', cache, true).entries[0].remainingTtlMs, null);
+});
+
+test('stats server listens on loopback and answers only GET /app/cache-stats', async () => {
+  const payload = statsOf([]);
+  const server = startSsrCacheStatsServer(0, () => payload);
+  try {
+    await once(server, 'listening');
+    const address = server.address();
+    assert.equal(address.address, '127.0.0.1');
+    const base = `http://127.0.0.1:${address.port}`;
+
+    const json = await fetch(`${base}/app/cache-stats`);
+    assert.equal(json.status, 200);
+    assert.equal(json.headers.get('cache-control'), 'no-store');
+    assert.deepEqual(await json.json(), payload);
+
+    assert.equal((await fetch(`${base}/app/cache-stats?x=1`)).status, 200);
+    assert.equal((await fetch(`${base}/other`)).status, 404);
+    assert.equal((await fetch(`${base}/app/cache-stats/`)).status, 404);
+    assert.equal((await fetch(`${base}/app/cache-stats`, { method: 'HEAD' })).status, 404);
+    assert.equal((await fetch(`${base}/app/cache-stats`, { method: 'POST' })).status, 404);
+
+  } finally {
+    await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve()));
+  }
+});
+
+test('stats server answers 500 when the snapshot throws', async () => {
+  const server = startSsrCacheStatsServer(0, () => { throw new Error('boom'); });
+  try {
+    await once(server, 'listening');
+    const response = await fetch(`http://127.0.0.1:${server.address().port}/app/cache-stats`);
+    assert.equal(response.status, 500);
+    assert.equal(response.headers.get('cache-control'), 'no-store');
+  } finally {
+    await new Promise((resolve) => server.close(resolve));
+  }
+});
+
+test('direct loopback check accepts local sockets without proxy headers only', () => {
+  const request = (remoteAddress, headers = {}) => ({ socket: { remoteAddress }, headers });
+  for (const address of ['127.0.0.1', '::1', '::ffff:127.0.0.1']) {
+    assert.equal(isDirectLoopbackRequest(request(address)), true, address);
+  }
+  for (const address of ['10.0.0.5', '::ffff:10.0.0.5', '0.0.0.0', '', undefined]) {
+    assert.equal(isDirectLoopbackRequest(request(address)), false, String(address));
+  }
+  for (const header of ['forwarded', 'via', 'x-forwarded-for', 'x-forwarded-host', 'x-forwarded-proto', 'x-forwarded-port', 'x-forwarded-ssl', 'x-real-ip']) {
+    assert.equal(isDirectLoopbackRequest(request('127.0.0.1', { [header]: '' })), false, header);
+  }
+  assert.equal(isDirectLoopbackRequest(request('127.0.0.1', { 'user-agent': 'curl' })), true);
+});
+
+test('stats server reports a failed bind on its error event instead of throwing', async () => {
+  const blocker = startSsrCacheStatsServer(0, () => statsOf([]));
+  await once(blocker, 'listening');
+  try {
+    const clash = startSsrCacheStatsServer(blocker.address().port, () => statsOf([]));
+    const [error] = await once(clash, 'error');
+    assert.equal(error.code, 'EADDRINUSE');
+  } finally {
+    await new Promise((resolve) => blocker.close(resolve));
+  }
+});

server.ts

@@ -36,6 +36,7 @@ import { createHttpTerminator } from 'http-terminator';
 
 import { readFileSync } from 'fs';
 import { join } from 'path';
+import { hostname } from 'os';
 
 import { enableProdMode } from '@angular/core';
 
@@ -56,6 +57,15 @@ import { extendEnvironmentWithAppConfig } from './src/config/config.util';
 import { logStartupMessage } from './startup-message';
 import { TOKENITEM } from './src/app/core/auth/models/auth-token-info.model';
 import { SsrExcludePatterns } from './src/config/universal-config.interface';
+import {
+  createSsrCacheStatsHandler,
+  isDirectLoopbackRequest,
+  snapshotCache,
+  SSR_CACHE_STATS_PATH,
+  SsrCacheEntry,
+  SsrCacheStats,
+  startSsrCacheStatsServer
+} from './src/backend/ssr-cache-stats';
 
 
 /*
@@ -72,10 +82,10 @@ const cookieParser = require('cookie-parser');
 const appConfig: AppConfig = buildAppConfig(join(DIST_FOLDER, 'assets/config.json'));
 
 // cache of SSR pages for known bots, only enabled in production mode
-let botCache: LRU<string, any>;
+let botCache: LRU<string, SsrCacheEntry>;
 
 // cache of SSR pages for anonymous users. Disabled by default, and only available in production mode
-let anonymousCache: LRU<string, any>;
+let anonymousCache: LRU<string, SsrCacheEntry>;
 
 // extend environment with app config for server
 extendEnvironmentWithAppConfig(environment, appConfig);
@@ -229,6 +239,21 @@ export function app() {
    */
   server.get('/app/health', healthCheck);
 
+  /**
+   * SSR cache contents for operators on this host (see cache.serverSide.stats). With stats.port = 0 they are
+   * served here, but only to requests made directly from this host: loopback socket address (not req.ip, which
+   * honours X-Forwarded-For when `trust proxy` is on) and no proxy headers, so traffic relayed by a same-host
+   * reverse proxy is refused too. Everyone else falls through to the catch-all route, exactly as for an unknown path.
+   */
+  if (environment.cache.serverSide.stats.enabled && environment.cache.serverSide.stats.port === 0) {
+    if (environment.ui.useProxies) {
+      console.warn('SSR cache stats are served on the UI port behind a proxy; a same-host proxy adding no forwarding headers would expose them. Prefer cache.serverSide.stats.port > 0.');
+    }
+    const statsHandler = createSsrCacheStatsHandler(collectCacheStats);
+    server.get(SSR_CACHE_STATS_PATH, (req, res, next) =>
+      req.method === 'GET' && req.path === SSR_CACHE_STATS_PATH && isDirectLoopbackRequest(req) ? statsHandler(req, res) : next());
+  }
+
   /**
    * Default sending all incoming requests to ngApp() function, after first checking for a cached
    * copy of the page (see cacheCheck())
@@ -592,9 +617,66 @@ function run() {
       });
 }
 
+/** pm2 cluster mode numbers its workers 0..n-1 in NODE_APP_INSTANCE; a single process reports 0. */
+const CLUSTER_INSTANCE = Number(process.env.NODE_APP_INSTANCE) || 0;
+
+function collectCacheStats(): SsrCacheStats {
+  const memory = process.memoryUsage();
+  return {
+    host: hostname(),
+    pid: process.pid,
+    instance: CLUSTER_INSTANCE,
+    generatedAt: new Date().toISOString(),
+    uptimeSeconds: Math.round(process.uptime()),
+    memory: { rss: memory.rss, heapUsed: memory.heapUsed },
+    caches: [
+      snapshotCache('bot', botCache, !!(environment.production && botCacheEnabled())),
+      snapshotCache('anonymous', anonymousCache, !!(environment.production && anonymousCacheEnabled()))
+    ]
+  };
+}
+
+/**
+ * Serve the cache stats on a loopback-only port of their own. Every pm2 cluster worker gets basePort + its
+ * instance number, because a port shared by the workers would be answered by an arbitrary one.
+ * An invalid port, a clash with the UI port or a failed bind is logged and never stops the UI.
+ */
+function startCacheStatsServer(basePort: number) {
+  const port = basePort + CLUSTER_INSTANCE;
+  const notStarted = (e: unknown) => console.error(`SSR cache stats server not started on port ${port}: ${e instanceof Error ? e.message : String(e)}`);
+  if (port === environment.ui.port) {
+    notStarted('it is the UI port');
+    return;
+  }
+  let statsServer: ReturnType<typeof startSsrCacheStatsServer>;
+  try {
+    statsServer = startSsrCacheStatsServer(port, collectCacheStats);
+  } catch (e) {
+    notStarted(e);
+    return;
+  }
+  let listening = false;
+  statsServer.on('listening', () => {
+    listening = true;
+    console.log(`SSR cache stats listening at http://127.0.0.1:${port}${SSR_CACHE_STATS_PATH}`);
+  });
+  statsServer.on('error', (e: Error) => listening ? console.error(`SSR cache stats server on port ${port}: ${e.message}`) : notStarted(e));
+  process.on('SIGINT', () => statsServer.close());
+}
+
 function start() {
   logStartupMessage(environment);
 
+  const statsConfig = environment.cache.serverSide.stats;
+  if (statsConfig.enabled && statsConfig.port !== 0) {
+    // port 0 is served by the Express app, see app()
+    if (Number.isInteger(statsConfig.port) && statsConfig.port > 0) {
+      startCacheStatsServer(statsConfig.port);
+    } else {
+      console.error(`SSR cache stats server not started: invalid port ${JSON.stringify(statsConfig.port)}`);
+    }
+  }
+
   /*
   * If SSL is enabled
   * - Read credentials from configuration files