Fix sec issues #388–#395 and remove sec-base: diagnostics, CI gates, Bun pin, XBRL scaling, primary_doc, fixture fetches

.claude/CLAUDE.md

@@ -34,26 +34,38 @@ independently, and a comment must explain the code in front of it.
 
 ```bash
 bun install
-bun run build                # bundles the two binaries
+bun run build                # bundles the binary
 bun run typecheck            # bun check over src, scripts AND tests
 bun run test                 # vitest run
 bunx vitest run <file>       # what you normally want — the full suite is slow
 bun run lint                 # oxlint + tsgolint type-aware rules; CI runs this
 bun run format               # oxfmt write — run before pushing
 ```
 
-CI runs `format-check` → `lint` → `typecheck` → `build` → `test`, cheapest first.
+CI and `bun run release-checks` run one gate list, `GATES` in
+`scripts/releaseChecks.ts`: `format-check` → `lint` → `typecheck` → `build` →
+`prepack-check` → `smoke-bins` → `test`, cheapest first. Add a gate there, never
+as a workflow step of its own — `toolchainPins.test.ts` fails on a restated one.
+`smoke-bins` starts the built `sec` unconfigured and requires `--help` and
+`--version` to exit 0.
 
 **Tests are vitest with `pool: "forks"` and `isolate: true`.** Run them with
 `bunx vitest run`, never `bun test`: the Bun shim does not give each file a fresh
 module registry, so `globalServiceRegistry` leaks between files.
 
-**Node 24, Bun 1.4+.** `node:sqlite` backs the SQLite storage and is neither stable
+**Node 24, Bun 1.4.3.** `node:sqlite` backs the SQLite storage and is neither stable
 nor unflagged below those. A subprocess CLI test on an older Bun fails with what
 reads like a storage bug.
 
+**Bun is pinned once, in `packageManager` (`bun@1.4.3`)**, and CI installs exactly
+that (`setup-bun` reads it from `package.json`). `typecheck` is `bun check`, so a
+Bun release could otherwise turn an unchanged commit red. `engines.bun` is the
+same version as a floor and `@types/bun` is the same version exactly;
+`toolchainPins.test.ts` fails when they disagree, so bump all three in one
+commit. `bun-latest.yml` runs the gates on the newest Bun weekly, gating nothing.
+
 There is one tsconfig covering src, scripts and tests. It emits nothing — the
-binaries bundle — so there is no declaration build to keep tests out of.
+binary bundles — so there is no declaration build to keep tests out of.
 
 ## The CLI
 
@@ -106,12 +118,10 @@ that list. `TestingDI` binds every entry as in-memory; call
 ## Cross-cutting rules
 
 **`bootstrapSecRuntime` is the one path that brings up the runtime** — the SQLite
-binding, DI, models, providers, the started fetch queue. Both binaries call it. A
-second entrypoint booting another way drifts silently, with late and misleading
-failures (a task resolving no model, a fetch with no rate limiter).
-
-**`registerSecTasks` is a curated list**, not every class under `src/task/`. A task
-earns its place there by answering a question on its own.
+binding, DI, models, providers, the started fetch queue. `sec` calls it from its
+one `preAction` hook, and any new entrypoint must call it too: one booting another
+way drifts silently, with late and misleading failures (a task resolving no
+model, a fetch with no rate limiter).
 
 **`getDb()` is SQLite-only** and throws when `SEC_DB_TYPE` is not sqlite. Before
 that guard it would open a stray SQLite file under Postgres, and rows written
@@ -139,6 +149,18 @@ everything else as its primary document. 8-K is unconditional because its primar
 document is routinely four sentences pointing at the EX-99.1 that carries the news
 — for that form the exhibits *are* the filing, and only the `.txt` has them.
 
+**Diagnostics are buffered, not printed.** Anything a task wants to tell an
+operator — a CIK that failed, a skipped row, a closing summary, what a dry run
+would have done — goes through `reportDiagnostic` / `reportDryRun`
+(`src/util/diagnostics.ts`), never `console.*`. `runWorkflowCli` holds them until
+the run settles and prints them after the frame closes; outside a run they fall
+through to `console.warn`. A console write underneath the live Ink frame shifts
+the lines its erase is counting, so the tail of the message is what gets
+overwritten. Progress text goes through the task's `updateProgress`. The count is
+pinned per file in `CONSOLE_DIAGNOSTIC_BUDGET`, exact in both directions, over
+every tree but `src/cli/` (commands render their answer after the run);
+`consoleDiagnostics.test.ts` fails on a new write.
+
 **`FILING_CONVERTER_VERSION` is the only version knob left.** Bump it by hand after
 a parser change to re-select already-converted filings; never truncate, since a
 half-finished re-run then leaves the old rows readable.

.github/workflows/bun-latest.yml

@@ -0,0 +1,31 @@
+name: Bun latest
+
+# The pinned Bun is what gates trunk (`test.yml`). This runs the same gates on
+# the newest Bun once a week so a release that changes `bun check` or the test
+# runner shows up here, as a red run nobody is blocked by, before the pin is
+# bumped onto it.
+
+on:
+  schedule:
+    - cron: "17 6 * * 1"
+  workflow_dispatch:
+
+env:
+  CI: "true"
+  DO_NOT_TRACK: "1"
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v6
+      - uses: actions/setup-node@v6
+        with:
+          node-version: "24"
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+      - run: bun --version
+      - run: bun i
+      - run: bun run release-checks

.github/workflows/test.yml

@@ -25,15 +25,15 @@ jobs:
       - uses: actions/setup-node@v6
         with:
           node-version: "24"
+      # The Bun pinned by `packageManager`, not `latest`: `typecheck` is
+      # `bun check`, so a Bun release can change a run's result on a commit
+      # that did not change. `bun-latest.yml` watches for that drift.
       - uses: oven-sh/setup-bun@v2
         with:
-          bun-version: latest
+          bun-version-file: package.json
       - run: bun i
-      - run: bun run format-check
-      # oxlint + tsgolint. Needs no build of its own: the type-aware rules
-      # resolve `@workglow/*` through the published `dist/*.d.ts` that `bun i`
-      # already put in node_modules.
-      - run: bun run lint
-      - run: bun run typecheck
-      - run: bun run build
-      - run: bun run test
+      # The gate list `release` runs, from the one script both call
+      # (`scripts/releaseChecks.ts`); each gate is its own log group. lint needs
+      # no build of its own: the type-aware rules resolve `@workglow/*` through
+      # the published `dist/*.d.ts` that `bun i` already put in node_modules.
+      - run: bun run release-checks

README.md

@@ -62,14 +62,6 @@ answer says which one wrote it.
 
 The old names still work: `sync`, `bootstrap`, `query`, `init` are aliases.
 
-There is a second binary, `sec-base`, which is the generic Workglow CLI — `task`,
-`model`, `mcp`, `workflow`, `agent`, `web` — with this repo's tasks registered:
-
-```sh
-sec-base task list
-sec-base task run QueryFilings --input-json '{"cik":320193}'
-```
-
 ## What does the work
 
 Each step is one Workglow package doing its job, and the point of the example is
@@ -123,23 +115,28 @@ bun run lint          # oxlint + type-aware rules
 bun run format        # oxfmt
 ```
 
-Node 24 and Bun 1.4+, declared as `engines` so a lower Node fails at install
+Node 24 and Bun 1.4.3, declared as `engines` so a lower Node fails at install
 rather than later inside `node:sqlite`. `node:sqlite` is what backs the SQLite
-storage, and it is neither stable nor unflagged below those.
+storage, and it is neither stable nor unflagged below those. CI runs exactly
+Bun 1.4.3, read from `packageManager` in `package.json` — the one place the
+version is written, since `typecheck` is `bun check` and a new Bun can change
+its answer on an unchanged commit.
 
-### The package is two binaries, not a library
+### The package is a binary, not a library
 
 `package.json` carries `bin` and no `exports`, `main` or `types`. That is the
-shape on purpose: the re-founding retired the library surface, both binaries
-bundle their dependencies, and `import ... from "@workglow/sec"` is not something
+shape on purpose: the re-founding retired the library surface, the binary
+bundles its dependencies, and `import ... from "@workglow/sec"` is not something
 this package offers. A manifest with no import entry point normally reads as a
 field someone deleted by mistake, so it is worth stating here: the absence is
 the decision.
 
 ### Cutting a release
 
 `bun run release-checks` is the gate set (format, lint, typecheck, build, packed
-contents). One script runs it and then bumps:
+contents, a start of the built binary with no configuration, tests) — the same
+list CI runs on every push, from `scripts/releaseChecks.ts`. One script runs it
+and then bumps:
 
 ```sh
 bun run release

package.json

@@ -12,19 +12,18 @@
   "homepage": "https://workglow.dev",
   "description": "A worked example of the Workglow libraries: SEC EDGAR and Form ADV ingestion, filing-to-markdown conversion, and retrieval-grounded question answering over what filings say.",
   "scripts": {
-    "release-checks": "bun run format-check && bun run lint && bun run typecheck && bun run build && bun run test && bun run prepack-check",
+    "release-checks": "bun ./scripts/releaseChecks.ts",
     "release": "bun run release-checks && bunset --auto --push --commit --tag",
     "prepack-check": "bun ./scripts/checkPackedContents.ts",
+    "smoke-bins": "bun ./scripts/smokeBinaries.ts",
     "dev": "concurrently -c 'auto' -n 'sec:' 'bun:dev-*'",
     "dev-sec": "bun build --watch --target=bun --sourcemap=external --packages=external --outfile ./dist/sec.js ./src/sec.ts",
-    "dev-sec-base": "bun build --watch --target=bun --sourcemap=external --packages=external --outfile ./dist/libs-cli.js ./src/libs-cli.ts",
     "dev-hft-worker": "bun build --watch --target=bun --sourcemap=external --packages=external --outfile ./dist/hftWorker.js ./src/config/hftWorker.ts",
     "dev-types": "tsc --noEmit --watch --preserveWatchOutput",
     "build": "bun run build-clean && bun run build-js",
     "build-clean": "rm -fr dist/*",
     "build-js": "bun run build-sec",
-    "build-sec": "bun build --target=bun --packages=external --outfile ./dist/sec.js ./src/sec.ts && bun ./scripts/shebang-bin.ts ./dist/sec.js && bun run build-sec-base",
-    "build-sec-base": "bun build --target=bun --packages=external --outfile ./dist/libs-cli.js ./src/libs-cli.ts && bun ./scripts/shebang-bin.ts ./dist/libs-cli.js && bun run build-hft-worker",
+    "build-sec": "bun build --target=bun --packages=external --outfile ./dist/sec.js ./src/sec.ts && bun ./scripts/shebang-bin.ts ./dist/sec.js && bun run build-hft-worker",
     "build-hft-worker": "bun build --target=bun --packages=external --outfile ./dist/hftWorker.js ./src/config/hftWorker.ts",
     "typecheck": "bun check",
     "use-source": "bun ./scripts/bunsrc.ts source",
@@ -39,13 +38,13 @@
     "lint-fix": "oxlint --fix src scripts"
   },
   "bin": {
-    "sec": "./dist/sec.js",
-    "sec-base": "./dist/libs-cli.js"
+    "sec": "./dist/sec.js"
   },
   "files": [
     "dist",
     "!**/*.map"
   ],
+  "packageManager": "bun@1.4.3",
   "engines": {
     "node": ">=24",
     "bun": ">=1.4.3"

scripts/checkFixtures.ts

@@ -9,21 +9,32 @@
  * Audits the committed EDGAR golden fixtures against their pinned manifest.
  *
  * A CI check on committed bytes, not something a user runs — which is why it is
- * a script and not a command. It reaches EDGAR over a plain fetch and writes
- * nothing, so it needs no configured database.
+ * a script and not a command. It writes nothing and needs no configured
+ * database, but it reaches EDGAR the way the pipeline does: through the fetch
+ * queue, started here because no CLI boot brings it up. Its rate limiter is
+ * in-process, since there is no database to share one through.
  *
  *   bun run check-fixtures            # verify, exit non-zero on any mismatch
  *   bun run check-fixtures --write    # re-download what no longer matches
  */
+import { shutdownCliResources } from "../src/cli/shutdownCliResources";
+import { startSecJobQueue } from "../src/task/fetch/SecJobQueue";
 import { GoldenFixturesTask } from "../src/task/fixtures/GoldenFixturesTask";
 
 const write = process.argv.includes("--write");
 const force = process.argv.includes("--force");
 
-const result = await new GoldenFixturesTask().run({
-  mode: write ? "download" : "verify",
-  force,
-});
+await startSecJobQueue();
+let result;
+try {
+  result = await new GoldenFixturesTask().run({
+    mode: write ? "download" : "verify",
+    force,
+  });
+} finally {
+  // The queue's workers keep the process alive until they are stopped.
+  await shutdownCliResources();
+}
 
 for (const problem of result.problems) console.error(problem);
 console.log(`Done. ok=${result.ok} written=${result.written} failed=${result.failed}`);

scripts/checkPackedContents.ts

@@ -35,18 +35,21 @@ import { findSourceStubs } from "./sourceStubs";
 // not when src sneaks back in.
 const MAX_UNPACKED_BYTES = 8_000_000; // 8 MB
 const FORBIDDEN_SUFFIXES = [".test.ts", ".map"] as const;
-const FORBIDDEN_PREFIXES = ["src/sec/html/mock_data/"] as const;
+// Any `src/` path is a leak: `files` ships `dist` alone, so a source file in
+// the listing means `files` or `.npmignore` widened. The size ceiling only
+// catches a leak big enough to cross it.
+const FORBIDDEN_PREFIXES = ["src/"] as const;
 const FORBIDDEN_SUBSTRINGS = ["/mock_data/"] as const;
 
 /**
- * Files the package cannot run without. The two `bin` entries are obvious; the
+ * Files the package cannot run without. The `bin` entry is obvious; the
  * worker is not, and is the reason this list exists — it is spawned by URL at
  * runtime rather than imported, so nothing in the build or the type system
  * notices when it stops being emitted, and the provider that needs it only
  * warns. `sec index` then fails on a published install while every check here
  * stays green.
  */
-const REQUIRED_PATHS = ["dist/sec.js", "dist/libs-cli.js", "dist/hftWorker.js"] as const;
+const REQUIRED_PATHS = ["dist/sec.js", "dist/hftWorker.js"] as const;
 
 interface PackReport {
   readonly source: string;
@@ -214,7 +217,7 @@ async function main(): Promise<void> {
     const extra = forbidden.length > 10 ? `\n  ... and ${forbidden.length - 10} more` : "";
     violations.push(
       `${forbidden.length} forbidden file(s) would be published ` +
-        `(tests / mock_data are not part of the shipped package):\n  ${sample}${extra}`
+        `(source, tests and mock_data are not part of the shipped package):\n  ${sample}${extra}`
     );
   }
 

scripts/releaseChecks.ts

@@ -0,0 +1,72 @@
+#!/usr/bin/env bun
+/**
+ * @license
+ * Copyright 2026 Steven Roussey <sroussey@gmail.com>
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * The gate list, once. `bun run release-checks` and CI both run this script,
+ * so what a release is checked against and what every push is checked against
+ * cannot drift apart — they did: CI ran five of the six, and the one it left
+ * out (what the tarball holds) was then found only on the maintainer's machine
+ * at release time.
+ *
+ * Cheapest first, stopping at the first failure. Under GitHub Actions each gate
+ * is a collapsible log group and a failure is an annotation naming the gate,
+ * which is what separate workflow steps used to give.
+ *
+ * `bun ./scripts/releaseChecks.ts --list` prints the gate names.
+ */
+import { spawnSync } from "node:child_process";
+
+interface Gate {
+  readonly name: string;
+  readonly command: readonly string[];
+}
+
+export const GATES: readonly Gate[] = [
+  { name: "format-check", command: ["bun", "run", "format-check"] },
+  { name: "lint", command: ["bun", "run", "lint"] },
+  { name: "typecheck", command: ["bun", "run", "typecheck"] },
+  { name: "build", command: ["bun", "run", "build"] },
+  // Both read `dist`, so they follow `build`; both take seconds, so they go
+  // ahead of the suite rather than after it.
+  { name: "prepack-check", command: ["bun", "run", "prepack-check"] },
+  { name: "smoke-bins", command: ["bun", "run", "smoke-bins"] },
+  { name: "test", command: ["bun", "run", "test"] },
+];
+
+const inActions = process.env.GITHUB_ACTIONS === "true";
+
+function runGate(gate: Gate): boolean {
+  const started = performance.now();
+  if (inActions) console.log(`::group::${gate.name}`);
+  else console.log(`\n== ${gate.name} ==`);
+  const [command, ...args] = gate.command;
+  const result = spawnSync(command, args, { stdio: "inherit" });
+  if (inActions) console.log("::endgroup::");
+  const seconds = ((performance.now() - started) / 1000).toFixed(1);
+  const ok = result.status === 0;
+  if (ok) {
+    console.log(`ok ${gate.name} (${seconds}s)`);
+  } else {
+    const why = result.error?.message ?? `exit ${result.status ?? result.signal}`;
+    console.log(
+      inActions
+        ? `::error title=${gate.name}::${gate.name} failed (${why})`
+        : `FAILED ${gate.name} (${why})`
+    );
+  }
+  return ok;
+}
+
+if (import.meta.main) {
+  if (process.argv.includes("--list")) {
+    for (const gate of GATES) console.log(gate.name);
+  } else {
+    for (const gate of GATES) {
+      if (!runGate(gate)) process.exit(1);
+    }
+  }
+}