Reduce public forum catalog latency with SQL pagination and batched access checks

docs/public-forums.md

@@ -119,11 +119,85 @@ Use a disposable local database whose name ends in `_http_test`, apply Prisma
 migrations, then run:
 
 ```sh
-PUBLIC_FORUMS_TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:55439/public_forums_http_test \
+TZ=UTC PUBLIC_FORUMS_TEST_DATABASE_URL=postgresql://postgres@127.0.0.1:55439/public_forums_http_test \
   pnpm test --runInBand
 ```
 
 That suite truncates its dedicated local test database, rejects remote hosts and
 other database names, uses real SQL and policies, and substitutes JWT validation
 and email publishing so it cannot notify real users. The standalone unit suite
-runs without a database.
+runs without a database. Use `TZ=UTC` for the database suite so PostgreSQL
+timestamps without time zones are compared consistently by the import checks.
+
+## Catalog query behavior
+
+Public catalog reads batch category ancestry, inherited ACLs, and imported role
+memberships once per request. Topic lists apply visibility, direct topic roles,
+search, watches, and sorting in PostgreSQL before pagination. Only the selected
+page receives full post/participant summaries; totals remain available for an
+out-of-range page. Search treats `%` and `_` as literal substring characters.
+
+Category statistics are aggregated in SQL over authorized threads and their
+non-deleted posts. Each response contains at most five participant previews per
+category, selected by latest activity with deterministic ID tie-breaking. Counts
+include all distinct authorized participants. Latest activity and unread state
+exclude restricted or deleted descendants. No shared cache stores personalized
+ACLs, watches, or unread state, so permission and membership changes are visible
+on the next request.
+
+## Development latency remediation — 7 October 2026
+
+The public legacy import exposed full-catalog reads, per-category authorization
+queries, CPU saturation and a Node heap crash. The read-path changes above are
+implemented in source commit `a4691d4` on `codex/forum-catalog-latency`.
+
+The development ECS service `tc-forums-serverless/forums-api-v6` now has a desired
+count of **two tasks**, each retaining **1 vCPU / 2,048 MiB**. The existing deploy
+suite updates only the task definition and therefore preserves the service's
+desired count. There is no service autoscaling target overriding that count.
+Both tasks retain the existing subnet placement; this adds process redundancy,
+not multi-AZ redundancy.
+
+The dev `services-alb` HTTPS listener default action now returns HTTP **404** with
+`application/json` body `{"message":"Not found"}`. Its non-default route rules
+are unchanged. Previously unmatched requests reached the empty retired
+`tc-email-service-svc-tg` target group and generated HTTP 503. No monitoring or
+alarm configuration was changed.
+
+Deployment artifact:
+
+- Task definition: `forums-api-v6:27`.
+- Image tag: `forums-api-v6:catalog-latency-a4691d4`.
+- Image digest: `sha256:5b38d36f40f2287d9b04a7d57beaeeef2aa12d707cca7cf231a274e8af3b4a65`.
+- Runtime base: the existing `legacy-jive-catalog-4bee3af` image, pinned to
+  `sha256:7e108dd776dba91546a2cfb057010f3b5bbf8a2871fceb9317508dec2f75e4fb`.
+
+The release reuses the unchanged runtime/dependencies and copies the compiled
+JavaScript and source maps for `forums-access-policy.service`,
+`forums-topic-context.service`, `forums.module`, `public-forum-access.service`,
+`public-forums.service`, and `public-forums-query.service` into
+`/usr/src/app/dist/forums/`. Every copied artifact's SHA-256 was checked against
+the local validated build. No database migration is required by this change.
+The standard source Docker build includes the same implementation for subsequent
+releases. Task revision 24 remains available as the previous application version;
+an application rollback can preserve the two-task count and the default 404.
+
+Validation includes `pnpm lint`, `pnpm build`, 173 passing tests (four optional
+tests skipped), real local PostgreSQL/HTTP permission and pagination checks,
+and read-only dev verification against the immutable migration source. The
+source audit matched all 844 guest-visible and 854 member-visible legacy
+categories, the respective thread/post counts, and private-thread denial.
+
+The focused PR branch `codex/forum-catalog-latency-develop` carries the same
+catalog implementation onto `develop` without the earlier Jive migration tools
+or dependency changes. Its lint and build checks pass, and its UTC/local
+PostgreSQL test run passes 170 tests across 21 suites (three optional tests
+skipped); the additional Jive tests above belong to the original deployment
+branch.
+
+Initial live checks of the optimized read path measured topic pages at
+1.7–2.2 seconds versus 14.6 seconds before remediation, and categories at
+4.3–5.0 seconds versus 10.9 seconds. The category response still aggregates the
+full authorized tree and is approximately 700 KB; these are sampled timings,
+not a sustained-load benchmark. Verification artifacts and pre-change AWS
+snapshots are retained in `/tmp/dev-latency-triage` on the operator workstation.

src/forums/forums-access-policy.service.ts

@@ -19,7 +19,10 @@ import {
   ResourceAccessService,
 } from './resource-access.service';
 
-import { PublicForumAccessService } from './public-forum-access.service';
+import {
+  PublicForumAccess,
+  PublicForumAccessService,
+} from './public-forum-access.service';
 
 const CHALLENGE_CHILD_TOPIC_DENY_REASON =
   'Child-topic creation is only allowed for non-challenge effective contexts.';
@@ -71,6 +74,63 @@ export class ForumsAccessPolicyService {
     if (!this.isTopicTargetAvailable(context)) {
       return this.denyTopicDecisions('Topic not found.');
     }
+    const categoryAccess = await this.publicAccess.decide(
+      this.resolveEvaluationPrincipal(principal, options),
+      context.topic.id,
+    );
+    return this.evaluateTopicAccess(
+      principal,
+      context,
+      categoryAccess,
+      options,
+    );
+  }
+
+  /** Evaluates category actions in a batch while preserving the shared topic policy.
+   * @param principal Catalog reader. @param contexts Preloaded category contexts.
+   * @returns Topic decisions keyed by ID, including inherited ACLs and restrictions.
+   * @throws Database errors from ACL and challenge access lookups.
+   */
+  async decideForTopics(
+    principal: ForumsPrincipal,
+    contexts: ForumsTopicContext[],
+  ): Promise<Map<string, ForumsTopicAccessDecisions>> {
+    const access = await this.publicAccess.decideMany(
+      principal,
+      contexts.map((context) => context.topic.id),
+    );
+    return new Map(
+      await Promise.all(
+        contexts.map(
+          async (context) =>
+            [
+              context.topic.id,
+              await this.evaluateTopicAccess(
+                principal,
+                context,
+                access.get(context.topic.id)!,
+                {},
+              ),
+            ] as const,
+        ),
+      ),
+    );
+  }
+
+  /** Applies the shared topic action rules to preloaded public ACL facts.
+   * @param principal Actor. @param context Target context. @param categoryAccess Resolved category ACL.
+   * @param options Optional on-behalf identity. @returns Topic action decisions.
+   * @throws Database errors from challenge/resource policy adapters.
+   */
+  private async evaluateTopicAccess(
+    principal: ForumsPrincipal,
+    context: ForumsTopicContext,
+    categoryAccess: PublicForumAccess,
+    options: ForumsAccessEvaluationOptions,
+  ): Promise<ForumsTopicAccessDecisions> {
+    if (!this.isTopicTargetAvailable(context)) {
+      return this.denyTopicDecisions('Topic not found.');
+    }
 
     const evaluation = await this.evaluateRestrictions(
       principal,
@@ -81,13 +141,20 @@ export class ForumsAccessPolicyService {
       },
       options,
     );
-    const categoryAccess = await this.publicAccess.decide(this.resolveEvaluationPrincipal(principal, options), context.topic.id);
-    if (!categoryAccess.view) return this.denyTopicDecisions('Required public forum access is missing.');
-    if (!evaluation.visibility.allowed) return this.denyTopicDecisions(evaluation.visibility.reason ?? 'Insufficient forums access.');
+    if (!categoryAccess.view)
+      return this.denyTopicDecisions(
+        'Required public forum access is missing.',
+      );
+    if (!evaluation.visibility.allowed)
+      return this.denyTopicDecisions(
+        evaluation.visibility.reason ?? 'Insufficient forums access.',
+      );
     const authenticated = !!principal.memberId || principal.isMachine;
     const canParticipate = evaluation.visibility.allowed && authenticated;
     const canOwnContent =
-      evaluation.visibility.allowed && context.isTopicAuthor && (!categoryAccess.category || evaluation.isElevated);
+      evaluation.visibility.allowed &&
+      context.isTopicAuthor &&
+      (!categoryAccess.category || evaluation.isElevated);
     const canMutateContent =
       evaluation.isElevated || canOwnContent
         ? this.allow()
@@ -100,15 +167,29 @@ export class ForumsAccessPolicyService {
     return {
       canView: evaluation.visibility,
       canCreateTopLevelTopic: this.deny('Use create-topic evaluation.'),
-      canCreateChildTopic: canParticipate && categoryAccess.create ? this.allow() : this.deny('Topic creation is not allowed.'),
-      canCreatePost: canParticipate && categoryAccess.reply ? this.allow() : this.deny('Replying is not allowed.'),
+      canCreateChildTopic:
+        canParticipate && categoryAccess.create
+          ? this.allow()
+          : this.deny('Topic creation is not allowed.'),
+      canCreatePost:
+        canParticipate && categoryAccess.reply
+          ? this.allow()
+          : this.deny('Replying is not allowed.'),
       canUpdateTopic: canMutateContent,
       canDeleteTopic,
-      canAddWatch: canParticipate ? this.allow() : this.deny('Authenticated member required.'),
-      canRemoveWatch: canParticipate ? this.allow() : this.deny('Authenticated member required.'),
-      canMarkRead: canParticipate ? this.allow() : this.deny('Authenticated member required.'),
+      canAddWatch: canParticipate
+        ? this.allow()
+        : this.deny('Authenticated member required.'),
+      canRemoveWatch: canParticipate
+        ? this.allow()
+        : this.deny('Authenticated member required.'),
+      canMarkRead: canParticipate
+        ? this.allow()
+        : this.deny('Authenticated member required.'),
       canControlAnnouncement,
-      canReceiveNotification: canParticipate ? this.allow() : this.deny('Authenticated member required.'),
+      canReceiveNotification: canParticipate
+        ? this.allow()
+        : this.deny('Authenticated member required.'),
     };
   }
 
@@ -132,7 +213,10 @@ export class ForumsAccessPolicyService {
       options,
     );
 
-    if (!this.isTopicTargetAvailable(context) || !topicDecisions.canView.allowed) {
+    if (
+      !this.isTopicTargetAvailable(context) ||
+      !topicDecisions.canView.allowed
+    ) {
       return {
         ...topicDecisions,
         canUpdatePost: this.deny('Post not found.'),
@@ -199,9 +283,16 @@ export class ForumsAccessPolicyService {
       return this.denyTopicDecisions('Topic not found.');
     }
 
-    if (!principal.memberId && !principal.isMachine) return this.denyTopicDecisions('Authenticated member required.');
-    if (parentContext && !(await this.publicAccess.decide(principal, parentContext.topic.id)).create) {
-      return this.denyTopicDecisions('Topic creation is not allowed in this category.');
+    if (!principal.memberId && !principal.isMachine)
+      return this.denyTopicDecisions('Authenticated member required.');
+    if (
+      parentContext &&
+      !(await this.publicAccess.decide(principal, parentContext.topic.id))
+        .create
+    ) {
+      return this.denyTopicDecisions(
+        'Topic creation is not allowed in this category.',
+      );
     }
 
     const parentVisibility = parentContext
@@ -337,7 +428,15 @@ export class ForumsAccessPolicyService {
     target: ForumsRestrictionVisibilityTarget,
     options: ForumsAccessEvaluationOptions = {},
   ): Promise<ForumsAccessDecision> {
-    if (target.topicId && !(await this.publicAccess.decide(this.resolveEvaluationPrincipal(principal, options), target.topicId)).view) {
+    if (
+      target.topicId &&
+      !(
+        await this.publicAccess.decide(
+          this.resolveEvaluationPrincipal(principal, options),
+          target.topicId,
+        )
+      ).view
+    ) {
       return this.deny('Required public forum access is missing.');
     }
     return (await this.evaluateRestrictions(principal, target, options))
@@ -363,8 +462,16 @@ export class ForumsAccessPolicyService {
       options,
     );
 
-    if (!evaluationPrincipal.memberId && !evaluationPrincipal.isMachine && (input.challengeId || input.roleName)) {
-      return { visibility: this.deny('Authenticated member required.'), isElevated: false, isChallengeCopilot: false };
+    if (
+      !evaluationPrincipal.memberId &&
+      !evaluationPrincipal.isMachine &&
+      (input.challengeId || input.roleName)
+    ) {
+      return {
+        visibility: this.deny('Authenticated member required.'),
+        isElevated: false,
+        isChallengeCopilot: false,
+      };
     }
 
     if (

src/forums/forums-topic-context.service.ts

@@ -48,6 +48,56 @@ export class ForumsTopicContextService {
     return this.buildTopicContext(topic, principal);
   }
 
+  /** Loads catalog authorization contexts with two batched database reads.
+   * @param topicIds Category topic IDs to resolve. Missing topics are omitted.
+   * @param principal Reader whose ownership flags are computed.
+   * @returns Contexts keyed by topic ID, using the same inherited rules as individual reads.
+   * @throws Prisma errors when either read fails.
+   */
+  async loadTopicContexts(
+    topicIds: string[],
+    principal: ForumsPrincipal,
+  ): Promise<Map<string, ForumsTopicContext>> {
+    if (!topicIds.length) return new Map();
+    const closures = await this.db.topicClosure.findMany({
+      where: { descendantTopicId: { in: topicIds } },
+      select: { descendantTopicId: true, ancestorTopicId: true, depth: true },
+      orderBy: { depth: 'asc' },
+    });
+    const allIds = [
+      ...new Set([...topicIds, ...closures.map((row) => row.ancestorTopicId)]),
+    ];
+    const topics = await this.db.topic.findMany({
+      where: { id: { in: allIds } },
+    });
+    const byId = new Map(topics.map((topic) => [topic.id, topic]));
+    const ancestors = new Map<string, Topic[]>();
+    for (const closure of closures) {
+      const ancestor = byId.get(closure.ancestorTopicId);
+      if (!ancestor) continue;
+      const group = ancestors.get(closure.descendantTopicId) ?? [];
+      group.push(ancestor);
+      ancestors.set(closure.descendantTopicId, group);
+    }
+    return new Map(
+      topicIds.flatMap((id) => {
+        const topic = byId.get(id);
+        return topic
+          ? [
+              [
+                id,
+                this.contextFromAncestors(
+                  topic,
+                  ancestors.get(id) ?? [topic],
+                  principal,
+                ),
+              ] as const,
+            ]
+          : [];
+      }),
+    );
+  }
+
   /**
    * Loads context for a post target.
    *
@@ -103,6 +153,19 @@ export class ForumsTopicContextService {
     const ancestors = closureRows.length
       ? closureRows.map((closure) => closure.ancestorTopic)
       : [topic];
+    return this.contextFromAncestors(topic, ancestors, principal);
+  }
+
+  /** Resolves a previously loaded ancestor chain for both single and batch reads.
+   * @param topic Target topic. @param ancestors Target-first ancestor chain.
+   * @param principal Reader identity. @returns Effective restrictions and ownership.
+   * @throws Never; this is an in-memory projection.
+   */
+  private contextFromAncestors(
+    topic: Topic,
+    ancestors: Topic[],
+    principal: ForumsPrincipal,
+  ): ForumsTopicContext {
     const ancestorTopics = ancestors.filter(
       (ancestor) => ancestor.id !== topic.id,
     );

src/forums/forums.module.ts

@@ -17,6 +17,7 @@ import { ModerationController } from './moderation.controller';
 import { PostsController } from './posts.controller';
 import { ResourceAccessService } from './resource-access.service';
 import { PublicForumsController } from './public-forums.controller';
+import { PublicForumsQueryService } from './public-forums-query.service';
 import { PublicForumsService } from './public-forums.service';
 import { PublicForumAccessService } from './public-forum-access.service';
 import { TopicsController } from './topics.controller';
@@ -32,14 +33,20 @@ import { TopicsController } from './topics.controller';
  */
 @Module({
   imports: [DbModule],
-  controllers: [TopicsController, PostsController, ModerationController, PublicForumsController],
+  controllers: [
+    TopicsController,
+    PostsController,
+    ModerationController,
+    PublicForumsController,
+  ],
   providers: [
     ChallengeAccessService,
     ChallengeApiService,
     EventBusService,
     ForumsAccessPolicyService,
     PublicForumAccessService,
     PublicForumsService,
+    PublicForumsQueryService,
     ForumsCommandService,
     ForumsMemberDirectoryService,
     ForumsModerationService,