From 2756951ed90742f06bb2c41113bb4d626d4c2d0b Mon Sep 17 00:00:00 2001 From: Kurt Overmier Date: Fri, 16 Oct 2026 14:58:28 -0500 Subject: [PATCH 1/3] feat(adf): compile preserves retained sections in a fenced block (closes #303) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +- README.md | 4 +- docs/cli-reference.md | 44 +- packages/cli/README.md | 4 +- .../cli/src/__tests__/adf-compile.test.ts | 44 +- .../cli/src/__tests__/adf-retained.test.ts | 498 ++++++++++++++++++ .../__tests__/adf-vendor-consumers.test.ts | 18 +- .../__tests__/adf-vendor-doctor-score.test.ts | 8 +- .../compile-no-retained.main-67f3c28.json | 92 ++++ packages/cli/src/commands/adf-compile.ts | 105 ++-- packages/cli/src/commands/adf-retained.ts | 236 +++++++++ packages/cli/src/commands/adf-vendor.ts | 32 +- 12 files changed, 995 insertions(+), 96 deletions(-) create mode 100644 packages/cli/src/__tests__/adf-retained.test.ts create mode 100644 packages/cli/src/__tests__/fixtures/compile-no-retained.main-67f3c28.json create mode 100644 packages/cli/src/commands/adf-retained.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d97e636..31f71e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,9 @@ The format is based on Keep a Changelog and follows Semantic Versioning. ### Changed +- **Compiled files that carry your own retained sections change bytes on the next `adf compile --write`** (`#303`, `#318` part 5) — a compiled `CLAUDE.md`, `AGENTS.md`, `.cursorrules` or `GEMINI.md` with a hand-added `## Environment` (or another section `adf tidy` retains) no longer loses it. The next `--write` moves it into a `` … `` block at the end of the file, and every later `--write` copies that block as it stands. A compiled file with nothing to carry gets no block and compiles byte-identically to before. The block uses HTML markers in every target, `.cursorrules` included. All-caps headings the compiler could have produced (a module since removed from `.ai/`) and `## Module Index` are never carried. `adf compile` to stdout and `charter score` still use the plain render. +- **`adf compile --check` reads such a file as stale until one `--write`** (`#303`) — the expected bytes are now the render plus the file's own retained block, so an edit inside the block is never stale. A compiled file whose retained sections sit outside the block reads `[stale]` with `staleReason: "retained-unfenced"`, and the remedy (`compile --target --write`) keeps them. Each `--check` JSON row adds `staleReason` (`content`, `retained-unfenced`, or null). +- **`refused-retained` is retired from `adf compile --write`, and `retained-divergent` from `adf vendor restore`** (`#303`) — JSON consumers no longer see either value. A pointer converted with `--mode compiled` now carries the retained sections that differ from its template into the block, instead of refusing. It still refuses, exit 1 with nothing written and `VENDORS` unchanged, over content the output would not carry: `refused-bloated` (items `adf tidy` would extract; the first remedy is `adf tidy --source `, then `--force`, which discards them) and `refused-would-drop` (other lines of yours). `--mode compiled --force` over a hand-authored file carries its retained sections and discards the rest. `--force` never drops retained sections. Write JSON rows add `retainedSections` and `retainedSource`. `vendor restore` of a compiled file carries its retained sections into the pointer and lists them in `keptSections`, and the block is not counted as a `compiled-differs` difference. - **The pre-commit hook tidies only the staged vendor files and re-stages only the files tidy wrote** (`#320`, `#318` part 4) — re-run `charter hook install --pre-commit` to pick up staged-only tidy. An installed hook keeps its old behaviour until you do, and `charter doctor` reports it with an INFO `pre-commit hook` row (INFO, so `--ci` does not fail). Reinstalling overwrites a Charter-managed hook without `--force`, as before. The old hook ran a whole-tree `adf tidy` whenever any vendor file was staged, then ran `git add` on every staged vendor file and on all of `.ai/*.adf`. That swept unrelated `.ai/` edits into the commit and rewrote vendor files that were not part of it. The new hook passes each staged vendor file to `adf tidy` with `--source` and stages only the paths that both the dry run and the real run report. A staged vendor file that also has unstaged changes is skipped with a message, because tidy works on the working tree and staging its result would commit those hunks. If tidy would write into a module that has unstaged changes or is untracked, nothing is tidied for that commit and a message says why. A gitignored module (for example an `.ai/` kept out of git) is not treated as work in progress: tidy writes it, and the hook says it was left unstaged. Any file the hook cannot stage, and any file the real run wrote that the dry run did not plan, is named and left unstaged rather than aborting the hook. A staged lowercase `agents.md` is now tidied (`#302`), and a commit staging only compiled output still makes no CLI call. The tidy block itself never fails a commit, but the ADF evidence gate after it can, and it is unchanged. In the generated hook (no `verify:adf` script), vendor bloat that a skipped tidy leaves in the working tree still fails `charter doctor --adf-only --ci`. The skip message now says so and names the ways out: stage or stash the unstaged work; run `charter adf tidy --source ` and `git add` the result; or `git commit --no-verify`. When the gate then fails, one line says which skipped file still carries how many items. A new hook with an older local CLI that does not know `--print-paths` reports the files as clean and tidies nothing, so upgrade the CLI too. - **BREAKING: `charter adf compile --write` no longer converts pointer files without `--mode compiled`** (`#318`, part 2) — compile used to overwrite any Charter thin pointer with compiled output, silently changing its mode. Each target's mode now comes from the `VENDORS` record, or from its markers when there is no entry, and `--write` only writes targets that are compiled or absent. Under `--target all`, pointer and unmanaged targets are skipped (exit 0), and each `[skip]` line names the command that converts it. A single such `--target` is refused (exit 1). `--force` alone never changes a mode. `--mode compiled` records `compiled` before writing. If that cannot be recorded, nothing is written (`refused-record`). If the write then fails, the record is restored and the run continues. A converted file is reported as `Written CLAUDE.md (mode: pointer -> compiled)`. JSON output adds `skipped` and per-target `targets`. A compiled-only repo with no `VENDORS` section compiles as before. @@ -28,6 +31,7 @@ The format is based on Keep a Changelog and follows Semantic Versioning. ### Fixed +- **`adf compile --write` no longer deletes your `## Environment` section from a compiled file** (`#303`) — the `#295` guard only covered pointer files. Re-compiling a file that already carried the compile banner silently dropped a user-added `## Environment` (or protocol) section with exit 0, and git was the only way back. Those sections are now kept in the retained block. - **`charter bootstrap --force` no longer destroys a hand-authored `CLAUDE.md` before it can be migrated** (`#318`, part 3) — phase 3 overwrote every existing pointer-file name with the template under `--force`, before phase 4 (migrate) ran. Vendor files are not backed up (only `.ai/*.adf` go to `.ai/.backup/`), so phase 4 then found a pointer, migrated nothing, and the file's rules were gone with no `.pre-adf-migrate.bak`. The file is now skipped in phase 3, and with `--yes` phase 4 migrates it into `.ai/` and writes the backup (when it has no `VENDORS` entry). - **`charter adf compile --check` and `charter adf tidy` read each vendor file's mode** (`#318`, part 2; `#302`) — `--check` compared every target byte-for-byte against compiled output, so a thin pointer was reported `[stale]` forever. Each target is now checked by its mode. A compiled file is `fresh` or `stale` as before. A pointer is `pointer-ok` when `adf tidy` would extract nothing from it, and `pointer-bloated` (exit 1, remedy `charter adf tidy --source `) when it would. An unmanaged file is `skipped-unmanaged` (exit 0). A recorded `compiled` or `pointer` file that looks like another mode on disk is `mode-drift` (exit 1), with both fixes printed: `charter adf vendor set ` to keep the file, or the command that restores it. A missing file is `missing`, and a recorded pointer that is missing names `git checkout -- ` or `adf vendor set unmanaged`. No remedy suggests `compile --write` for a pointer or unmanaged file. JSON keeps `stale` (which now also lists bloated and drifted files), `current`, `missing`, `upToDate` and `nextActions`, and adds `skipped` and `targets`. `adf tidy` processes a file only when it is a pointer: no compile banner, at least one pointer marker, and not recorded otherwise. The banner still wins whatever the record says (`#296`). A file recorded `compiled` is skipped with status `compiled`. Its warning lists the content-preserving fix first (`adf vendor set pointer`, then `adf tidy --source `) and says how many items regenerating would discard; `--check` mode-drift does the same. An `unmanaged` file (recorded, or hand-authored with no marker) now has status `unmanaged` instead of `not-found`, and its warning appears when it is recorded or named with `--source`. A recorded pointer with no marker is skipped rather than guessed at. File rows gain `vendorFile`, `mode`, `source` and `skipReason`. A `--source` path that is not a repo-root vendor name keeps the old marker logic. Tidy now finds files through the shared discovery helper, so `AGENTS.md` and `agents.md` that are one file on a case-insensitive filesystem are walked once, not twice. With no `SYNC` entries, `charter adf sync` now points to `charter adf vendor status` in text and in JSON `nextActions`, rather than ending at "Wrote empty lock file". Record entry problems are still reported only by `adf vendor status`. `doctor`, `score`, `bootstrap` and both pre-commit hooks are unchanged and are later parts of `#318`. - **`charter doctor` now names the fix when `core.adf` or `state.adf` is missing from `DEFAULT_LOAD`** (`#241`) — the `adf required wiring` warning said only `Missing from DEFAULT_LOAD: core.adf`. It now says to move the module from `ON_DEMAND` to `DEFAULT_LOAD` in `.ai/manifest.adf` when it is registered there, or to add it (or run `charter adf create --load default`) when it is not registered at all. The rule itself is unchanged and is filename-only — `core.adf` and `state.adf` are required in `DEFAULT_LOAD`, and module content is not consulted — but it now lives in one shared helper that `doctor` and `bootstrap`'s orphan registration both call, so `adf migrate` can adopt the same classification when it starts registering the modules it creates (`#239`). @@ -42,7 +46,7 @@ The format is based on Keep a Changelog and follows Semantic Versioning. - **`charter --help` now reaches the command's own docs** (`#240`) — the top-level dispatcher matched `--help`/`-h` anywhere in the raw arguments, so `charter adf migrate --help` printed the global help and the ADF subcommand reference was only reachable by running `charter adf` bare. `adf`, `hook`, `score` and `telemetry` now render their own help. The remaining commands have no `--help` handler and deliberately keep the root-help fallback, so a stray `--help` can never execute one for real. - **`adf populate` no longer invents an ESM extension rule for bundler projects** (`#294`) — the constraint "Use .js extensions for all ESM imports" was emitted whenever any `package.json` carried `"type": "module"`, which is true of every Vite, esbuild and modern Workers project. Under `moduleResolution: "bundler"` extensionless relative imports are the correct convention, so bootstrap was writing a wrong rule into `CONSTRAINTS [load-bearing]` and inviting agents to "fix" a correct codebase. The rule is now gated on TypeScript's `moduleResolution` — emitted for `node16`/`nodenext`, withheld for `bundler`, `node` and anything unrecognised — and a single bundler package vetoes it for the repository. Ambiguity (an unparseable JSONC `tsconfig.json`, a solution-style config, an `extends` chain this resolver does not follow) withholds the rule rather than guessing. - **`adf compile --write` no longer refuses the files `bootstrap` just generated** (`#295`) — the overwrite guard keyed on the compile banner alone, so a clean `bootstrap` followed by the documented `adf compile --target all --write` produced a wall of refusals and trained users to reach for `--force`. The guard now recognises Charter thin pointers as well as compiler output. Hand-authored files carrying no Charter marker are still refused, and a file converted from pointer to compiled output is reported as `(replaced charter pointer stub)` so the transition is visible. -- **`adf compile --write` no longer discards retained sections** (`#295`) — it refuses, rather than silently overwriting, a pointer file carrying your own content under `## Environment` or another retained heading. Compiled output has no such section and `.ai/` does not hold that content, so the previous behavior lost it with git as the only recovery. Pass `--force` to discard deliberately. +- **`adf compile --write` no longer discards retained sections** (`#295`) — it refuses, rather than silently overwriting, a pointer file carrying your own content under `## Environment` or another retained heading. Compiled output has no such section and `.ai/` does not hold that content, so the previous behavior lost it with git as the only recovery. Pass `--force` to discard deliberately. Since `#303`, the file is written with that content carried into a retained block instead (see Changed). - **`charter score` no longer counts directory-tree filenames as broken references** (`#293`) — a fenced tree block expresses the path prefix structurally, through indentation, so `random.ts` and `simulation/random.ts` were reported broken although `src/simulation/random.ts` exists. Grounding now falls back to matching a reference against the repository's real files before classifying it as broken. Two conditions must both hold: the basename is unique, and every path segment the author actually wrote agrees with the real file's trailing segments. The second is checked against the authored text rather than the resolved path, so a reference written inside `.ai/` still resolves, while a stale `src/utils/helper.ts` pointing at a file since moved to `packages/core/lib/helper.ts` stays broken — documentation rot is precisely what this check exists to surface. An ambiguous basename such as `index.ts` under several directories, and a filename that exists nowhere, also stay broken. On this repository the broken count falls from 79 to 48 and Grounding rises from 13/20 to 15/20, while genuinely absent files (`Cargo.toml`, `wrangler.toml`, `schema.sql`, `src/worker.ts`) are all still reported. ### Changed diff --git a/README.md b/README.md index a58483b..dbba0ed 100644 --- a/README.md +++ b/README.md @@ -255,7 +255,7 @@ For live session continuity snapshots, use `charter context-refresh` to produce | I want to compile task-specific agent context | `charter adf bundle --task "Fix the login flow"` | | I want to enforce metric/file-size ceilings | `charter adf evidence --auto-measure --ci` | | I want to migrate existing agent docs | `charter adf migrate --dry-run` | -| I want to regenerate CLAUDE.md / AGENTS.md / .cursorrules / GEMINI.md from `.ai/` | `charter adf compile --target all --write` (add `--mode compiled` to convert pointer files) | +| I want to regenerate CLAUDE.md / AGENTS.md / .cursorrules / GEMINI.md from `.ai/` | `charter adf compile --target all --write` (add `--mode compiled` to convert pointer files; your `## Environment` notes are kept in a retained block at the end) | | I want a letter-grade AI-readiness audit | `charter score` | | I want to validate governance in CI | `charter validate --ci --format json` | | I want to check pattern drift | `charter drift --ci --format json` | @@ -288,7 +288,7 @@ charter adf init # Scaffold .ai/ directory charter adf bundle --task "..." # Merge context for a task charter adf evidence --auto-measure # Validate metric constraints charter adf migrate # Migrate existing configs -charter adf compile --target all --write # Re-render compiled-mode CLAUDE.md, AGENTS.md, .cursorrules, GEMINI.md +charter adf compile --target all --write # Re-render compiled-mode CLAUDE.md, AGENTS.md, .cursorrules, GEMINI.md (keeps your ## Environment notes) charter adf compile --target all --write --mode compiled # Switch pointer files to compiled output charter adf vendor status # Show each vendor file's mode (pointer, compiled, unmanaged) charter adf vendor restore CLAUDE.md # Rewrite one pointer from its template (refuses if content would be lost) diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 02b19fc..5e88c92 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -542,29 +542,51 @@ Default with no `--write` or `--check` prints to stdout, single target only. | pointer | skipped (its `[skip]` line names the command), exit 0 | refused, exit 1 | record set to compiled, then written as `Written CLAUDE.md (mode: pointer -> compiled)` | | unmanaged | skipped, exit 0 | refused, exit 1 | as pointer, but an existing hand-authored file also needs `--force` | -When every target is skipped, compile says that nothing was written and how to switch. JSON output keeps `written`, `refused` and `nextActions`, and adds `skipped` and `targets` (one row per target: `state`, `source`, `status`, `recordedMode`, `previousMode`, `remedy`, `writeError`). +When every target is skipped, compile says that nothing was written and how to switch. JSON output keeps `written`, `refused` and `nextActions`, and adds `skipped` and `targets` (one row per target: `state`, `source`, `status`, `recordedMode`, `previousMode`, `remedy`, `writeError`, `retainedSections`, `retainedSource`). `previousMode` (`pointer` or `unmanaged`) is how a JSON consumer tells a conversion from a re-compile. + +**Retained sections.** Compiled output keeps your own `## Environment` section and the other sections `adf tidy` retains (`## Session ...` other than `## Session Start`, and headings containing Protocol, Receipt, Provenance or Registration). They live in a block at the very end of the file, after the module index: + +```markdown + + + +## Environment +- Use WSL for builds + +``` + +`--write` carries them from wherever they are: + +- An existing block is copied as it stands, so an edit inside it survives every re-compile, and a section you delete from it stays deleted. Delete the whole block (both markers) to stop carrying anything. +- A compiled file with retained sections outside the block (for example a hand-added `## Environment`) gets them moved into the block. An all-caps heading the compiler could have produced (`## PROTOCOL`, a section since removed from `.ai/`) is never carried, and neither is `## Module Index`. +- A pointer converted with `--mode compiled` carries the retained sections that differ from its template. +- A hand-authored file converted with `--mode compiled --force` carries its retained sections; the rest is discarded. + +A file with nothing to carry gets no block, so its output is byte-identical to the plain render. The block uses the HTML markers in every target, `.cursorrules` included. Sections keep their source order, and the output is LF. A carried section whose heading matches a compiled one (say `## CONTEXT`) is kept, with a warning on stderr. The written line names what was kept: `Written CLAUDE.md (kept ## Environment)`. `adf compile` without `--write` prints the plain render, with no block. `adf tidy`, `doctor` and `adf migrate` ignore the block, because it is a Charter sentinel. + +Converting a pointer refuses (exit 1, nothing written, `VENDORS` unchanged) when content the output would not carry would be lost: + +- `refused-bloated`: the pointer carries items `adf tidy` would extract. The first remedy is `charter adf tidy --source ` (moves them into `.ai/`, then re-run). The second is `--mode compiled --force`, which discards N item(s). +- `refused-would-drop`: other lines of yours (an extra note, an edited title, edited `## Session Start` text) would be lost. Charter pointer-template lines and the module index do not count. Move them under `## Environment` or into a `.ai/` module, or pass `--force`, which discards N line(s). + +`--force` skips these refusals but never drops retained sections. `refused-retained` is retired: a pointer with notes under a retained heading is now written, with the notes carried. `--check` per target: | Status | When | Exit | |---|---|---| -| `fresh` / `stale` | compiled: bytes equal / differ (remedy: `compile --target --write`) | 0 / 1 | +| `fresh` / `stale` | compiled: the file equals / differs from what `--write` would write, which is the render plus the file's own retained block (remedy: `compile --target --write`). `staleReason` is `retained-unfenced` when retained sections sit outside the block (the remedy keeps them), else `content` | 0 / 1 | | `missing` | no file on disk (a recorded pointer suggests `adf vendor restore ` or `adf vendor set unmanaged`) | 1 | | `pointer-ok` | pointer with nothing `adf tidy` would extract | 0 | | `pointer-bloated` | pointer carrying content that belongs in `.ai/` (remedy: `adf tidy --source `) | 1 | | `mode-drift` | recorded `compiled` or `pointer`, but the file looks like another mode. Both fixes are printed, the content-preserving one first; regenerating says how many items it would discard | 1 | | `skipped-unmanaged` | unmanaged: not checked | 0 | -No `--check` remedy suggests `compile --write` for a pointer or unmanaged file. JSON output keeps `stale` (stale, bloated and drifted files), `current`, `missing`, `upToDate` and `nextActions`, and adds `skipped` and `targets`. - -**Overwrite guard.** With `--write` and no `--force`, compile writes an existing file only when it recognises it as Charter-owned — either compiler output (carrying the compile banner) or a Charter thin pointer. It refuses in two cases: - -- the file is hand-authored and carries no Charter marker -- the file is a pointer carrying your own content under a retained heading (`## Environment` and the other headings `adf tidy` retains) that the pointer template does not have +No `--check` remedy suggests `compile --write` for a pointer or unmanaged file. An edit inside the retained block never makes a file stale. A compiled file with a hand-added `## Environment` outside the block reads `stale` (`retained-unfenced`) until one `--write` moves it into the block. The comparison is still byte for byte, so a CRLF checkout of a compiled file reads stale. JSON output keeps `stale` (stale, bloated and drifted files), `current`, `missing`, `upToDate` and `nextActions`, and adds `skipped` and `targets` (each row has `staleReason`, null unless stale). -The second case matters because compiled output has no `## Environment` section and `.ai/` does not hold that content, so overwriting would lose it with no way back except git. Move such content into a `.ai/` module first, or pass `--force` to discard it. +**Overwrite guard.** With `--write` and no `--force`, compile writes an existing file only when it recognises it as Charter-owned — either compiler output (carrying the compile banner) or a Charter thin pointer. A hand-authored file with no Charter marker is refused (`refused-hand-authored`). A pointer is refused only over content the output would not carry (`refused-bloated`, `refused-would-drop`; see retained sections above). -Both refusals happen before the record is touched, so a refused target never changes `VENDORS`. +These refusals happen before the record is touched, so a refused target never changes `VENDORS`. ### charter adf tidy @@ -660,7 +682,7 @@ npx charter adf vendor restore CLAUDE.md # rewrite one pointer from i - `status`: read-only. A file with a valid entry shows its recorded mode, plus `(recorded; file looks compiled)` when the markers on disk disagree. A file without one shows the mode inferred from its markers, the way `adf tidy` classifies it: the compile banner means `compiled`, a pointer marker means `pointer`, and anything else is `unmanaged`. Malformed, duplicate (the last entry wins), unknown-file and not-allowed entries are reported and ignored. A missing manifest exits 0 with a warning. - `set `: records or replaces one entry, editing only that line. Only `CLAUDE.md`, `AGENTS.md`, `.cursorrules` and `GEMINI.md` may be `compiled`. `agents.md` and `copilot-instructions.md` have no compile target, so `compiled` is refused with exit 1. An unknown file or mode, or a missing manifest, exits 2. The file does not have to exist yet. - `init`: records the inferred mode of every vendor file on disk that has no entry. It never changes an existing entry, and a second run writes nothing. -- `restore `: rewrites one file as its thin pointer template (`CLAUDE.md` gets the hybrid template with the module index filled in, as bootstrap writes it). It runs only when the file's mode is `pointer` (recorded or inferred), or the file is absent with no entry, and it refuses (exit 1, file untouched) whenever content would be lost: `mode-not-pointer` (record the intent first with `vendor set pointer`), `hand-authored` (run `adf migrate --source `), `pointer-bloated` (run `adf tidy --source `), `retained-divergent` (a compiled file carrying your own `## Environment` or protocol section), and `compiled-differs` (a compiled file that differs from what `.ai/` compiles to now, checked in memory; keep it with `vendor set compiled`). For a pointer, every non-blank line must survive: a line that is not from a Charter pointer template (current or earlier) or the generated module index, and that the new pointer would not carry, is refused as `hand-authored` with the first few lines named. Retained sections of a pointer are carried over verbatim and listed. It is the fix for a recorded pointer that now holds compiled output. A file with no entry gets `pointer` recorded before the file is written; if that cannot be recorded, nothing is written (`record-write`), and if the file write fails, the manifest is put back byte for byte (`write-failed`, with `writeError`). A result identical to the file on disk is `unchanged` and writes nothing. `--dry-run` reports `planned`. An unknown file name exits 2. JSON output: `file`, `status` (`restored`, `planned`, `unchanged`, `refused`), `refusal`, `keptSections`, `remedy`, `writeError`; text lines go to stderr under `--format json`. +- `restore `: rewrites one file as its thin pointer template (`CLAUDE.md` gets the hybrid template with the module index filled in, as bootstrap writes it). It runs only when the file's mode is `pointer` (recorded or inferred), or the file is absent with no entry, and it refuses (exit 1, file untouched) whenever content would be lost: `mode-not-pointer` (record the intent first with `vendor set pointer`), `hand-authored` (run `adf migrate --source `), `pointer-bloated` (run `adf tidy --source `), and `compiled-differs` (a compiled file that differs from what `.ai/` compiles to now, checked in memory; keep it with `vendor set compiled`). A compiled file's retained block, and retained sections outside one, are not a difference: they are carried into the pointer as plain sections and listed in `keptSections`. The `retained-divergent` refusal is retired. For a pointer, every non-blank line must survive: a line that is not from a Charter pointer template (current or earlier) or the generated module index, and that the new pointer would not carry, is refused as `hand-authored` with the first few lines named. Retained sections of a pointer are carried over verbatim and listed. It is the fix for a recorded pointer that now holds compiled output. A file with no entry gets `pointer` recorded before the file is written; if that cannot be recorded, nothing is written (`record-write`), and if the file write fails, the manifest is put back byte for byte (`write-failed`, with `writeError`). A result identical to the file on disk is `unchanged` and writes nothing. `--dry-run` reports `planned`. An unknown file name exits 2. JSON output: `file`, `status` (`restored`, `planned`, `unchanged`, `refused`), `refusal`, `keptSections`, `remedy`, `writeError`; text lines go to stderr under `--format json`. One physical file has one entry. When two vendor names are the same file (`AGENTS.md` and `agents.md` on a case-insensitive filesystem, or a hard link), every writer records it under the spelling already recorded, or `AGENTS.md` when neither is. `status` shows it once. `set` names the spelling it used. - `--dry-run`: with `set`, `init` or `restore`, report the change without writing diff --git a/packages/cli/README.md b/packages/cli/README.md index 9656fe5..efc95ce 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -77,7 +77,7 @@ Requires Node >= 18. ```bash charter score # read-only AI-readiness grade + prioritized fixes charter bootstrap # detect stack, create .ai/, migrate existing agent rules -charter adf compile --target all --write # re-render compiled-mode vendor files from .ai/ +charter adf compile --target all --write # re-render compiled-mode vendor files from .ai/ (keeps your ## Environment notes) charter adf compile --target all --write --mode compiled # switch pointer files to compiled output charter adf vendor status # show each vendor file's mode (pointer, compiled, unmanaged) charter adf vendor restore CLAUDE.md # rewrite one pointer from its template (refuses if content would be lost) @@ -337,7 +337,7 @@ charter adf vendor restore [--dry-run] [--ai-dir ] - `evidence`: Validate all metric ceilings in the merged document and produce a structured pass/fail evidence report. `--auto-measure` counts lines in files referenced by the manifest METRICS section. `--context` or `--context-file` inject external metric overrides that take precedence over auto-measured and document values. In `--ci` mode, exits 1 on constraint failures (warnings don't fail). Also reports stale-baseline warnings (baseline vs current delta + recommended ceiling) when baseline values drift significantly. The governance workflow template runs this automatically on PRs when `.ai/manifest.adf` is present. - `metrics recalibrate`: Re-measure current LOC from manifest metric sources, propose and apply new ceilings using configurable headroom, and append rationale records to `BUDGET_RATIONALES`. Requires explicit rationale (`--reason`) unless `--auto-rationale` is used. - `migrate`: Scan existing agent config files (CLAUDE.md, .cursorrules, agents.md, GEMINI.md, copilot-instructions.md), classify content using the ADX-002 decision tree, and migrate into ADF modules. `--dry-run` previews the migration plan without writing files. `--source ` targets a single file. `--no-backup` skips `.pre-adf-migrate.bak` creation. `--merge-strategy` controls deduplication: `dedupe` (default, skip items already in ADF), `append` (always add), or `replace`. Environment-specific rules (WSL, PATH, credential helpers) are retained in the thin pointer. Each module that receives content is registered in `manifest.adf`: `core.adf` and `state.adf` under `DEFAULT_LOAD`, the rest under `ON_DEMAND` with triggers inferred from the migrated headings. Registration is append-only, so existing entries are never rewritten. The `CLAUDE.md` module index is refreshed to match, and the plan shows a `REGISTER` line per module. -- `vendor`: Show or record whether each vendor file (`CLAUDE.md`, `AGENTS.md`, `.cursorrules`, `GEMINI.md`, `agents.md`, `copilot-instructions.md`) is a thin `pointer`, `compiled` output, or `unmanaged`. The record is a `VENDORS` list in `manifest.adf` (` - CLAUDE.md: pointer`). `status` is read-only and shows each file's recorded mode, or the mode inferred from its markers when none is recorded. `set` records one file's mode; only the four compile targets may be `compiled`. `init` records the inferred mode of every vendor file on disk that has no entry yet, and never changes an existing one. `bootstrap`, `adf init --emit-pointers`, `adf migrate` and `adf compile --write` also record the mode of each file they write, without changing an existing entry (an explicit `adf migrate --source ` changes `unmanaged` to `pointer`). `restore ` rewrites one pointer from its template, keeping retained sections such as `## Environment`, and refuses whenever content would be lost (hand-authored content, bloat, or your own sections in a compiled file). `adf compile`, `adf tidy`, `doctor`, `score`, `bootstrap` and `adf migrate` act on the record: for example, `bootstrap --force` never replaces a compiled or hand-authored vendor file, and bare `adf migrate` skips a file recorded `unmanaged` (`#318`). +- `vendor`: Show or record whether each vendor file (`CLAUDE.md`, `AGENTS.md`, `.cursorrules`, `GEMINI.md`, `agents.md`, `copilot-instructions.md`) is a thin `pointer`, `compiled` output, or `unmanaged`. The record is a `VENDORS` list in `manifest.adf` (` - CLAUDE.md: pointer`). `status` is read-only and shows each file's recorded mode, or the mode inferred from its markers when none is recorded. `set` records one file's mode; only the four compile targets may be `compiled`. `init` records the inferred mode of every vendor file on disk that has no entry yet, and never changes an existing one. `bootstrap`, `adf init --emit-pointers`, `adf migrate` and `adf compile --write` also record the mode of each file they write, without changing an existing entry (an explicit `adf migrate --source ` changes `unmanaged` to `pointer`). `restore ` rewrites one pointer from its template, keeping retained sections such as `## Environment`, and refuses whenever content would be lost (hand-authored content, bloat, or a compiled file that differs from what `.ai/` compiles to); a compiled file's retained sections are carried into the pointer. `adf compile --write` keeps your `## Environment` and other retained sections in a `charter:retained` block at the end of each compiled file, converts a pointer with notes without refusing, and refuses only over content it would not carry (`refused-bloated`, `refused-would-drop`) (`#303`). `adf compile`, `adf tidy`, `doctor`, `score`, `bootstrap` and `adf migrate` act on the record: for example, `bootstrap --force` never replaces a compiled or hand-authored vendor file, and bare `adf migrate` skips a file recorded `unmanaged` (`#318`). ### `charter blast` diff --git a/packages/cli/src/__tests__/adf-compile.test.ts b/packages/cli/src/__tests__/adf-compile.test.ts index 66a30c2..65c630b 100644 --- a/packages/cli/src/__tests__/adf-compile.test.ts +++ b/packages/cli/src/__tests__/adf-compile.test.ts @@ -448,45 +448,47 @@ describe('adf compile — --write overwrite guard', () => { expect(fs.readFileSync(path.join(tmp, 'CLAUDE.md'), 'utf-8')).toContain(COMPILE_BANNER_MARKER); }); - it('refuses a pointer carrying user notes under ## Environment', async () => { + it('carries a pointer\'s ## Environment notes into the retained block (#303)', async () => { const tmp = makeTmp(); // The shape `charter adf tidy` produces and the pre-commit hook manufactures: - // a thin pointer with retained Environment rules. The compiler emits no - // Environment section and .ai/ holds none of it, so overwriting destroys it. + // a thin pointer with retained Environment rules. .ai/ holds none of it, so + // the compiled file keeps it in its retained block (it used to be refused, #295). const withNotes = POINTER_CLAUDE_MD_HYBRID.replace( '', '- In WSL, use the Windows credential helper if HTTPS push fails.\n- Keep core.hooksPath pointed at .githooks.', ); scaffoldAi(tmp, { 'CLAUDE.md': withNotes }); - const spy = vi.spyOn(console, 'error').mockImplementation(() => {}); - const exitCode = await adfCommand(TEXT_OPTIONS, ['compile', '--target', 'claude', '--write', '--mode', 'compiled']); - const warning = spy.mock.calls.map(c => String(c[0])).join('\n'); - spy.mockRestore(); + const logs = await captureLog(() => + adfCommand(TEXT_OPTIONS, ['compile', '--target', 'claude', '--write', '--mode', 'compiled']), + ); - expect(exitCode).toBe(1); - expect(fs.readFileSync(path.join(tmp, 'CLAUDE.md'), 'utf-8')).toBe(withNotes); - // The message must name the cause, not the generic hand-authored wording. - expect(warning).toContain('## Environment'); - expect(warning).toContain('--force'); + const content = fs.readFileSync(path.join(tmp, 'CLAUDE.md'), 'utf-8'); + expect(content).toContain(COMPILE_BANNER_MARKER); + expect(content.endsWith( + '\n' + + '\n\n' + + '## Environment\n- In WSL, use the Windows credential helper if HTTPS push fails.\n- Keep core.hooksPath pointed at .githooks.\n' + + '\n', + )).toBe(true); + expect(logs.join('\n')).toContain('[ok] Written CLAUDE.md (mode: pointer -> compiled; kept ## Environment)'); }); - it('refuses a pointer carrying a retained operational protocol section', async () => { + it('carries a retained operational protocol section', async () => { const tmp = makeTmp(); const withProtocol = `${POINTER_AGENTS_MD}\n## Release Protocol\n\n- Tag before publishing.\n`; scaffoldAi(tmp, { 'AGENTS.md': withProtocol }); - const spy = vi.spyOn(console, 'error').mockImplementation(() => {}); const exitCode = await adfCommand(TEXT_OPTIONS, ['compile', '--target', 'agents', '--write', '--mode', 'compiled']); - const warning = spy.mock.calls.map(c => String(c[0])).join('\n'); - spy.mockRestore(); - expect(exitCode).toBe(1); - expect(fs.readFileSync(path.join(tmp, 'AGENTS.md'), 'utf-8')).toBe(withProtocol); - expect(warning).toContain('## Release Protocol'); + expect(exitCode).toBe(0); + const content = fs.readFileSync(path.join(tmp, 'AGENTS.md'), 'utf-8'); + expect(content).toContain(COMPILE_BANNER_MARKER); + expect(content).toContain(''); + expect(content).toContain('## Release Protocol\n\n- Tag before publishing.\n\n'); }); - it('--force overwrites a pointer carrying user notes', async () => { + it('--force still carries a pointer\'s retained notes: it skips guards, never preservation', async () => { const tmp = makeTmp(); const withNotes = POINTER_CLAUDE_MD_HYBRID.replace( '', @@ -501,7 +503,7 @@ describe('adf compile — --write overwrite guard', () => { expect(exitCode).toBe(0); const content = fs.readFileSync(path.join(tmp, 'CLAUDE.md'), 'utf-8'); expect(content).toContain(COMPILE_BANNER_MARKER); - expect(content).not.toContain('Some hand-written environment rule'); + expect(content).toContain('## Environment\n- Some hand-written environment rule.\n'); }); it('flags the pointer-to-compiled switch in the written line', async () => { diff --git a/packages/cli/src/__tests__/adf-retained.test.ts b/packages/cli/src/__tests__/adf-retained.test.ts new file mode 100644 index 0000000..7f07ae8 --- /dev/null +++ b/packages/cli/src/__tests__/adf-retained.test.ts @@ -0,0 +1,498 @@ +/** + * `adf compile` preserves retained sections in a fenced block (#303, #318 PR 5 of 6). + * + * Pins composeCompiled (the one function --write, --check and restore share), + * the compile --write / --check changes, restore's carry-over, and INV-P5-1..10. + */ + +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { COMPILE_BANNER_MARKER, stripCharterSentinels } from '@stackbilt/adf'; +import type { CLIOptions } from '../index'; +import { adfCommand, POINTER_AGENTS_MD, POINTER_CLAUDE_MD, POINTER_CLAUDE_MD_HYBRID } from '../commands/adf'; +import { + composeCompiled, + compiledMatchesFresh, + linesConvertWouldDrop, + GENERATED_HEADING_RE, + RETAINED_END, + RETAINED_NOTE, + RETAINED_START, +} from '../commands/adf-retained'; + +const TEXT: CLIOptions = { configPath: '.charter', format: 'text', ciMode: false, yes: false }; +const JSON_OPTS: CLIOptions = { ...TEXT, format: 'json' }; + +interface Fixture { + inputs: Record; + scenarios: Record; args: string[]; files: Record }>; +} +/** Generated by main @ 67f3c28 (the CLI before #303): see the scenario notes in each entry. */ +const MAIN_FIXTURE = JSON.parse( + fs.readFileSync(path.join(__dirname, 'fixtures', 'compile-no-retained.main-67f3c28.json'), 'utf-8'), +) as Fixture; + +const BASE_MANIFEST = MAIN_FIXTURE.inputs['.ai/manifest.adf']; +const TARGET_FILES = ['CLAUDE.md', 'AGENTS.md', '.cursorrules', 'GEMINI.md']; + +const originalCwd = process.cwd(); +let tmp = ''; +let out: string[] = []; +let err: string[] = []; + +beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'charter-retained-')); + process.chdir(tmp); + out = []; + err = []; + vi.spyOn(console, 'log').mockImplementation((...a: unknown[]) => { out.push(a.map(String).join(' ')); }); + vi.spyOn(console, 'error').mockImplementation((...a: unknown[]) => { err.push(a.map(String).join(' ')); }); + vi.spyOn(console, 'warn').mockImplementation((...a: unknown[]) => { err.push(a.map(String).join(' ')); }); + vi.spyOn(process.stdout, 'write').mockImplementation((chunk: string | Uint8Array) => { out.push(String(chunk)); return true; }); +}); + +afterEach(() => { + vi.restoreAllMocks(); + process.chdir(originalCwd); + fs.rmSync(tmp, { recursive: true, force: true }); +}); + +function scaffold(manifest = BASE_MANIFEST): void { + for (const [file, content] of Object.entries(MAIN_FIXTURE.inputs)) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, file === '.ai/manifest.adf' ? manifest : content); + } +} + +function vendors(...entries: string[]): string { + return `${BASE_MANIFEST}\n\u{1F3F7} VENDORS:\n${entries.map(e => ` - ${e}\n`).join('')}`; +} + +const read = (file: string): string => fs.readFileSync(file, 'utf-8'); + +async function run(args: string[], options: CLIOptions = TEXT): Promise<{ code: number; out: string; err: string }> { + out = []; + err = []; + const code = await adfCommand(options, args); + return { code, out: out.join('\n'), err: err.join('\n') }; +} + +async function runJson>(args: string[]): Promise<{ code: number; json: T; err: string }> { + const r = await run([...args, '--format', 'json'], JSON_OPTS); + return { code: r.code, json: JSON.parse(r.out) as T, err: r.err }; +} + +async function fresh(target: string): Promise { + return (await run(['compile', '--target', target])).out; +} + +interface WriteRow { file: string; status: string; previousMode: string | null; recordedMode: string | null; remedy: string | null; retainedSections: string[]; retainedSource: string } +interface WriteJson { written: string[]; refused: string[]; targets: WriteRow[]; nextActions: string[] } +interface CheckRow { file: string; status: string; staleReason: string | null; remedy: string | null } +interface CheckJson { upToDate: boolean; targets: CheckRow[]; nextActions: string[] } + +const block = (body: string): string => `${RETAINED_START}\n${RETAINED_NOTE}\n\n${body}\n${RETAINED_END}\n`; +const ENV = '## Environment\n- Use WSL for builds'; + +// A synthetic render, shaped as compileAdf emits it. +const FRESH = [ + ``, '', '# CLAUDE.md', '', '## CONTEXT', '', '- a', '', + '## RULES _(load-bearing)_', '', '- b', '', + '', '## Additional instructions', '', '- `.ai/x.adf`', '', '', +].join('\n'); + +// ============================================================================ +// 1. composeCompiled +// ============================================================================ + +describe('composeCompiled', () => { + it('returns the fresh render byte for byte when there is nothing to carry (P3)', () => { + expect(composeCompiled(FRESH, null, 'CLAUDE.md')).toMatchObject({ output: FRESH, retainedSource: 'none', retainedSections: [] }); + const edited = FRESH.replace('- a', '- a hand edit'); + expect(composeCompiled(FRESH, edited, 'CLAUDE.md')).toMatchObject({ output: FRESH, retainedSource: 'none' }); + }); + + it('moves a legacy user section into a block at the very end; the banner stays line 1 (P1, W1)', () => { + const legacy = `${FRESH}\n${ENV}\n`; + const c = composeCompiled(FRESH, legacy, 'CLAUDE.md'); + expect(c).toMatchObject({ retainedSource: 'legacy', retainedSections: ['## Environment'], unfenced: ['## Environment'] }); + expect(c.output).toBe(`${FRESH.trimEnd()}\n\n${block(ENV)}`); + expect(c.output.split('\n')[0]).toBe(``); + }); + + it('is idempotent for every source (R1, INV-P5-3)', () => { + const inputs: Record = { + legacy: `${FRESH}\n${ENV}\n`, + block: `${FRESH}\n${block(ENV)}`, + pointer: `${POINTER_AGENTS_MD}\n## Release Protocol\n\n- Tag first.\n`, + unmanaged: '# Mine\n\nHouse rules.\n\n## Environment\n- WSL\n', + }; + for (const [name, input] of Object.entries(inputs)) { + const file = name === 'pointer' ? 'AGENTS.md' : 'CLAUDE.md'; + const once = composeCompiled(FRESH, input, file); + expect(once.retainedSource, name).toBe(name); + expect(composeCompiled(FRESH, once.output, file).output, name).toBe(once.output); + } + }); + + it('copies an existing block verbatim, so an edit inside it changes nothing (C1, INV-P5-5)', () => { + const edited = `${FRESH}\n${RETAINED_START}\n\nmy own preface line\n${ENV}\n- second note\n\n\n${RETAINED_END}\n`; + const c = composeCompiled(FRESH, edited, 'CLAUDE.md'); + expect(c.output).toBe(edited); + expect(c.retainedSections).toEqual(['## Environment']); + }); + + it('keeps a section deleted inside the block deleted; deleting the whole block means nothing is carried (R2)', () => { + const two = composeCompiled(FRESH, `${FRESH}\n${ENV}\n\n## Release Protocol\n- tag\n`, 'CLAUDE.md').output; + const oneLeft = two.replace('## Release Protocol\n- tag\n', ''); + expect(composeCompiled(FRESH, oneLeft, 'CLAUDE.md').output).toBe(oneLeft); + expect(composeCompiled(FRESH, oneLeft, 'CLAUDE.md').output).not.toContain('Release Protocol'); + const noBlock = oneLeft.slice(0, oneLeft.indexOf(RETAINED_START)); + expect(composeCompiled(FRESH, noBlock, 'CLAUDE.md').output).toBe(FRESH); + }); + + it('keeps several sections in source order, and turns CRLF input into stable LF output (P2)', () => { + const legacy = `${FRESH}\n## Session Protocol\n- one\n\n${ENV}\n`.replace(/\n/g, '\r\n'); + const c = composeCompiled(FRESH, legacy, 'CLAUDE.md'); + expect(c.retainedSections).toEqual(['## Session Protocol', '## Environment']); + expect(c.output).not.toContain('\r'); + expect(c.output).toBe(`${FRESH.trimEnd()}\n\n${block(`## Session Protocol\n- one\n\n${ENV}`)}`); + expect(composeCompiled(FRESH, c.output, 'CLAUDE.md').output).toBe(c.output); + }); + + it('keeps a block section whose heading matches a compiled one, and reports the collision (R4)', () => { + const withRules = `${FRESH}\n${block('## RULES\n- my rule')}`; + const c = composeCompiled(FRESH, withRules, 'CLAUDE.md'); + expect(c.output).toBe(withRules); + expect(c.collisions).toEqual(['## RULES']); + }); + + it('never resurrects a generated heading from a legacy file (INV-P5-4, #296 class)', () => { + expect(GENERATED_HEADING_RE.test('## PROTOCOL')).toBe(true); + expect(GENERATED_HEADING_RE.test('## RELEASE_PROTOCOL _(advisory)_')).toBe(true); + // A module since removed from .ai/ left these behind in the compiled file. + const stale = `${FRESH}\n## PROTOCOL\n\n- old rule\n\n## RELEASE_PROTOCOL _(advisory)_\n\n- old\n\n## SESSION\n\n- old\n`; + const c = composeCompiled(FRESH, stale, 'CLAUDE.md'); + expect(c.output).toBe(FRESH); + expect(c.retainedSource).toBe('none'); + }); + + it('never carries ## Module Index or ## Session Start', () => { + const legacy = `${FRESH}\n## Module Index\n| x |\n\n## Session Start\nCall it.\n`; + expect(composeCompiled(FRESH, legacy, 'CLAUDE.md').output).toBe(FRESH); + }); + + it('falls back to legacy when an end marker has no start, dropping the stray marker (R3)', () => { + const stray = `${FRESH}\n${ENV}\n${RETAINED_END}\n`; + const c = composeCompiled(FRESH, stray, 'CLAUDE.md'); + expect(c.retainedSource).toBe('legacy'); + expect(c.output).toBe(`${FRESH.trimEnd()}\n\n${block(ENV)}`); + }); + + it('runs a start marker with no end to EOF, then closes it, stably (R3)', () => { + const open = `${FRESH}\n${RETAINED_START}\n${ENV}\n`; + const c = composeCompiled(FRESH, open, 'CLAUDE.md'); + expect(c).toMatchObject({ retainedSource: 'block', retainedSections: ['## Environment'] }); + expect(c.output).toBe(`${FRESH.trimEnd()}\n\n${RETAINED_START}\n${ENV}\n${RETAINED_END}\n`); + expect(composeCompiled(FRESH, c.output, 'CLAUDE.md').output).toBe(c.output); + // The stripper agrees: an unmatched start strips to EOF. + expect(stripCharterSentinels(open)).not.toContain('Use WSL'); + }); + + it('takes retained sections that sit outside an existing block into it', () => { + const both = `${FRESH}\n## Session Protocol\n- new\n\n${block(ENV)}`; + const c = composeCompiled(FRESH, both, 'CLAUDE.md'); + expect(c).toMatchObject({ retainedSource: 'block', unfenced: ['## Session Protocol'], retainedSections: ['## Session Protocol', '## Environment'] }); + expect(c.output).toBe(`${FRESH.trimEnd()}\n\n${block(`## Session Protocol\n- new\n\n${ENV}`)}`); + }); + + it('does not carry a retained-looking line the render itself emits (a .ai/ text value)', () => { + const withEmbedded = FRESH.replace('- a', '- a\n## Environment\n- from .ai/'); + expect(composeCompiled(withEmbedded, withEmbedded, 'CLAUDE.md').output).toBe(withEmbedded); + }); + + it('ends a legacy section at the module index fence instead of swallowing it', () => { + const inserted = FRESH.replace('', `${ENV}\n\n`); + const c = composeCompiled(FRESH, inserted, 'CLAUDE.md'); + expect(c.output).toBe(`${FRESH.trimEnd()}\n\n${block(ENV)}`); + }); + + it('carries only the divergent retained sections of a pointer (source pointer)', () => { + const pristine = composeCompiled(FRESH, POINTER_CLAUDE_MD_HYBRID, 'CLAUDE.md'); + expect(pristine.output).toBe(FRESH); + const notes = POINTER_CLAUDE_MD_HYBRID.replace('', '- WSL'); + expect(composeCompiled(FRESH, notes, 'CLAUDE.md')).toMatchObject({ retainedSource: 'pointer', retainedSections: ['## Environment'], body: '## Environment\n- WSL' }); + }); + + it('linesConvertWouldDrop ignores template lines and the module index, and names the rest', () => { + const notes = POINTER_CLAUDE_MD.replace('', '- WSL'); + expect(linesConvertWouldDrop(notes, composeCompiled(FRESH, notes, 'CLAUDE.md').output)).toEqual([]); + const stray = POINTER_CLAUDE_MD.replace('# CLAUDE.md', '# CLAUDE.md\n\n> my own note'); + expect(linesConvertWouldDrop(stray, composeCompiled(FRESH, stray, 'CLAUDE.md').output)).toEqual(['> my own note']); + }); + + it('compiledMatchesFresh ignores the block and legacy sections, but not other edits (K2)', () => { + expect(compiledMatchesFresh(FRESH, `${FRESH}\n${block(ENV)}`, 'CLAUDE.md')).toBe(true); + expect(compiledMatchesFresh(FRESH, `${FRESH}\n${ENV}\n`, 'CLAUDE.md')).toBe(true); + expect(compiledMatchesFresh(FRESH, FRESH.replace('- b', '- edited'), 'CLAUDE.md')).toBe(false); + }); + + it('the block is a charter sentinel: tidy, doctor and migrate strip it (K5)', () => { + expect(stripCharterSentinels(`${FRESH}\n${block(ENV)}`)).not.toContain('Use WSL'); + }); +}); + +// ============================================================================ +// 2. compile --write and --check (#303 hole) +// ============================================================================ + +describe('compile keeps a compiled file\'s user sections (#303)', () => { + it('--check reads stale (retained-unfenced) until one --write, which keeps them; then fresh and idempotent', async () => { + scaffold(); + expect((await run(['compile', '--target', 'claude', '--write'])).code).toBe(0); + const render = read('CLAUDE.md'); + fs.writeFileSync('CLAUDE.md', `${render}\n${ENV}\n`); + + const textCheck = await run(['compile', '--target', 'claude', '--check']); + expect(textCheck.code).toBe(1); + expect(textCheck.out).toContain(' [stale] CLAUDE.md — your ## Environment section(s) sit outside the retained block.\n Run: charter adf compile --target claude --write (keeps them)'); + const check = await runJson(['compile', '--target', 'claude', '--check']); + expect(check.json.targets[0]).toMatchObject({ status: 'stale', staleReason: 'retained-unfenced', remedy: 'charter adf compile --target claude --write' }); + + const write = await runJson(['compile', '--target', 'claude', '--write']); + expect(write.code).toBe(0); + expect(write.json.targets[0]).toMatchObject({ status: 'written', retainedSections: ['## Environment'], retainedSource: 'legacy' }); + const kept = read('CLAUDE.md'); + expect(kept).toBe(`${render.trimEnd()}\n\n${block(ENV)}`); + + const text = await run(['compile', '--target', 'claude', '--write']); + expect(text.out).toContain(' [ok] Written CLAUDE.md (kept ## Environment)'); + expect(read('CLAUDE.md')).toBe(kept); + const again = await runJson(['compile', '--target', 'claude', '--check']); + expect(again.json.targets[0]).toMatchObject({ status: 'fresh', staleReason: null }); + + // Edit inside the block: still fresh, and kept by the next write. + const edited = kept.replace('- Use WSL for builds', '- Use WSL for builds\n- And pnpm 9'); + fs.writeFileSync('CLAUDE.md', edited); + expect((await run(['compile', '--target', 'claude', '--check'])).code).toBe(0); + await run(['compile', '--target', 'claude', '--write']); + expect(read('CLAUDE.md')).toBe(edited); + + // Delete the whole block: it stays deleted. + fs.writeFileSync('CLAUDE.md', edited.slice(0, edited.indexOf(RETAINED_START))); + await run(['compile', '--target', 'claude', '--write']); + expect(read('CLAUDE.md')).toBe(render); + }); + + it('a content change still reads stale with staleReason content', async () => { + scaffold(); + await run(['compile', '--target', 'claude', '--write']); + fs.writeFileSync('CLAUDE.md', `${read('CLAUDE.md').replace('TypeScript', 'JavaScript')}\n${block(ENV)}`); + const check = await runJson(['compile', '--target', 'claude', '--check']); + expect(check.json.targets[0]).toMatchObject({ status: 'stale', staleReason: 'content' }); + }); + + it('two --write runs are byte-identical for every target, with and without a block, LF and CRLF (INV-P5-3)', async () => { + scaffold(); + await run(['compile', '--target', 'all', '--write']); + const plain = Object.fromEntries(TARGET_FILES.map(f => [f, read(f)])); + await run(['compile', '--target', 'all', '--write']); + for (const f of TARGET_FILES) expect(read(f), f).toBe(plain[f]); + + for (const f of TARGET_FILES) fs.writeFileSync(f, `${plain[f]}\n${ENV}\n`.replace(/\n/g, '\r\n')); + await run(['compile', '--target', 'all', '--write']); + const kept = Object.fromEntries(TARGET_FILES.map(f => [f, read(f)])); + await run(['compile', '--target', 'all', '--write']); + for (const f of TARGET_FILES) { + expect(read(f), f).toBe(kept[f]); + expect(kept[f], f).toBe(`${plain[f].trimEnd()}\n\n${block(ENV)}`); + } + // .cursorrules uses the HTML markers too (CP-3). + expect(kept['.cursorrules'].startsWith('# GENERATED')).toBe(true); + expect(kept['.cursorrules']).toContain(RETAINED_START); + expect((await run(['compile', '--target', 'all', '--check'])).code).toBe(0); + }); + + it('warns on a heading collision and keeps both (R4)', async () => { + scaffold(); + await run(['compile', '--target', 'claude', '--write']); + const withCollision = `${read('CLAUDE.md')}\n${block('## CONTEXT\n- my context')}`; + fs.writeFileSync('CLAUDE.md', withCollision); + const r = await run(['compile', '--target', 'claude', '--write']); + expect(r.code).toBe(0); + expect(r.err).toContain('CLAUDE.md: your retained section ## CONTEXT has the same heading as a compiled section; both are kept.'); + expect(read('CLAUDE.md')).toBe(withCollision); + const json = await run(['compile', '--target', 'claude', '--write', '--format', 'json'], JSON_OPTS); + expect(() => JSON.parse(json.out)).not.toThrow(); + }); + + it('stdout mode prints the pure render even when the file has a block (INV-P5-7)', async () => { + scaffold(); + await run(['compile', '--target', 'claude', '--write']); + const render = read('CLAUDE.md'); + fs.writeFileSync('CLAUDE.md', `${render}\n${ENV}\n`); + await run(['compile', '--target', 'claude', '--write']); + expect(await fresh('claude')).toBe(render); + }); +}); + +// ============================================================================ +// 3. Converting a pointer or unmanaged file (W2, W3, W4) +// ============================================================================ + +describe('compile --write --mode compiled carries retained sections', () => { + const notes = POINTER_CLAUDE_MD_HYBRID.replace('', '- Use WSL for builds'); + + it('writes a pointer with notes and carries them, with no refusal', async () => { + scaffold(); + fs.writeFileSync('CLAUDE.md', notes); + const r = await runJson(['compile', '--target', 'claude', '--write', '--mode', 'compiled']); + expect(r.code).toBe(0); + expect(r.json.targets[0]).toMatchObject({ status: 'written', previousMode: 'pointer', retainedSections: ['## Environment'], retainedSource: 'pointer' }); + expect(read('CLAUDE.md')).toBe(`${(await fresh('claude')).trimEnd()}\n\n${block(ENV)}`); + }); + + it('refuses a bloated pointer, tidy remedy first, the record unchanged; --force discards the bloat', async () => { + scaffold(); + const bloated = `${notes}\n## Testing\n\n- Always run pnpm test before pushing to main.\n- Never skip the integration suite in CI.\n`; + fs.writeFileSync('CLAUDE.md', bloated); + const text = await run(['compile', '--target', 'claude', '--write', '--mode', 'compiled']); + expect(text.code).toBe(1); + expect(text.err).toMatch(/Refused to compile CLAUDE\.md: it carries (\d+) item\(s\) beyond the pointer that \.ai\/ does not hold yet\.\n {9}Move them into \.ai\/ first: charter adf tidy --source CLAUDE\.md\n {9}Or discard them: charter adf compile --target claude --write --mode compiled --force {2}\(discards \1 item\(s\)\)/); + const r = await runJson(['compile', '--target', 'claude', '--write', '--mode', 'compiled']); + expect(r.json.targets[0]).toMatchObject({ status: 'refused-bloated', remedy: 'charter adf tidy --source CLAUDE.md' }); + expect(r.json.nextActions).toEqual(['charter adf tidy --source CLAUDE.md', 'charter adf compile --target claude --write --mode compiled --force']); + expect(read('CLAUDE.md')).toBe(bloated); + expect(read('.ai/manifest.adf')).toBe(BASE_MANIFEST); + + const forced = await runJson(['compile', '--target', 'claude', '--write', '--mode', 'compiled', '--force']); + expect(forced.json.targets[0]).toMatchObject({ status: 'written', retainedSections: ['## Environment'] }); + expect(read('CLAUDE.md')).not.toContain('Always run pnpm test'); + expect(read('CLAUDE.md')).toContain('- Use WSL for builds'); + }); + + it('refuses a pointer whose other lines would be lost (refused-would-drop); --force discards them', async () => { + scaffold(); + const stray = notes.replace('# CLAUDE.md\n', '# CLAUDE.md\n\n> Ask Kim before touching billing.\n'); + fs.writeFileSync('CLAUDE.md', stray); + const text = await run(['compile', '--target', 'claude', '--write', '--mode', 'compiled']); + expect(text.code).toBe(1); + expect(text.err).toContain("Refused to compile CLAUDE.md: 1 line(s) of yours would be lost ('> Ask Kim before touching billing.')."); + expect(text.err).toContain('--force (discards 1 line(s))'); + const r = await runJson(['compile', '--target', 'claude', '--write', '--mode', 'compiled']); + expect(r.json.targets[0]).toMatchObject({ status: 'refused-would-drop', remedy: 'charter adf compile --target claude --write --mode compiled --force' }); + expect(read('CLAUDE.md')).toBe(stray); + expect(read('.ai/manifest.adf')).toBe(BASE_MANIFEST); + + const forced = await runJson(['compile', '--target', 'claude', '--write', '--mode', 'compiled', '--force']); + expect(forced.code).toBe(0); + expect(read('CLAUDE.md')).not.toContain('Ask Kim'); + expect(read('CLAUDE.md')).toContain('- Use WSL for builds'); + }); + + it('refuses a pointer with bloat or stray lines even when it has no retained sections to carry', async () => { + scaffold(); + const bloated = `${POINTER_AGENTS_MD}\n## Testing\n\n- Always run pnpm test before pushing to main.\n`; + fs.writeFileSync('AGENTS.md', bloated); + const r = await runJson(['compile', '--target', 'agents', '--write', '--mode', 'compiled']); + expect(r.code).toBe(1); + expect(r.json.targets[0]).toMatchObject({ status: 'refused-bloated', remedy: 'charter adf tidy --source AGENTS.md' }); + expect(read('AGENTS.md')).toBe(bloated); + + const stray = POINTER_AGENTS_MD.replace('# AGENTS.md\n', '# AGENTS.md\n\n> Ask Kim first.\n'); + fs.writeFileSync('AGENTS.md', stray); + const s2 = await runJson(['compile', '--target', 'agents', '--write', '--mode', 'compiled']); + expect(s2.json.targets[0]).toMatchObject({ status: 'refused-would-drop' }); + expect(read('AGENTS.md')).toBe(stray); + expect(read('.ai/manifest.adf')).toBe(BASE_MANIFEST); + }); + + it('carries retained sections from a hand-authored file under --force (CP-4, W3)', async () => { + scaffold(); + fs.writeFileSync('CLAUDE.md', `# Mine\n\nHouse rules.\n\n${ENV}\n\n## Deploy Protocol\n- staging first\n`); + const r = await runJson(['compile', '--target', 'claude', '--write', '--mode', 'compiled', '--force']); + expect(r.code).toBe(0); + expect(r.json.targets[0]).toMatchObject({ status: 'written', previousMode: 'unmanaged', retainedSource: 'unmanaged', retainedSections: ['## Environment', '## Deploy Protocol'] }); + expect(read('CLAUDE.md')).not.toContain('House rules'); + expect(read('CLAUDE.md')).toContain(block(`${ENV}\n\n## Deploy Protocol\n- staging first`)); + }); + + it('never emits refused-retained, and every JSON stdout parses (INV-P5-8)', async () => { + scaffold(); + fs.writeFileSync('CLAUDE.md', notes); + fs.writeFileSync('AGENTS.md', `${POINTER_AGENTS_MD}\n## Release Protocol\n\n- Tag first.\n`); + const r = await runJson(['compile', '--target', 'all', '--write', '--mode', 'compiled']); + expect(r.json.targets.map(t => t.status)).not.toContain('refused-retained'); + expect(r.json.targets.every(t => Array.isArray(t.retainedSections) && typeof t.retainedSource === 'string')).toBe(true); + expect(r.code).toBe(0); + }); +}); + +// ============================================================================ +// 4. vendor restore (K2, K3) +// ============================================================================ + +interface RestoreJson { status: string; refusal: string | null; keptSections: string[] } + +describe('adf vendor restore of a compiled file with a retained block', () => { + it('restores the pointer and carries the block sections into it', async () => { + scaffold(vendors('CLAUDE.md: pointer')); + const withBlock = `${(await fresh('claude')).trimEnd()}\n\n${block(`${ENV}\n\n## Deploy Protocol\n- staging first`)}`; + fs.writeFileSync('CLAUDE.md', withBlock); + const r = await run(['vendor', 'restore', 'CLAUDE.md', '--format', 'json'], JSON_OPTS); + const json = JSON.parse(r.out) as RestoreJson; + expect(json).toMatchObject({ status: 'restored', refusal: null, keptSections: ['## Environment', '## Deploy Protocol'] }); + const pointer = read('CLAUDE.md'); + expect(pointer).not.toContain(COMPILE_BANNER_MARKER); + expect(pointer).not.toContain(RETAINED_START); + expect(pointer.endsWith(`${ENV}\n\n## Deploy Protocol\n- staging first\n`)).toBe(true); + }); + + it('is compiled-differs only when something besides the block differs', async () => { + scaffold(vendors('CLAUDE.md: pointer')); + const withBlock = `${(await fresh('claude')).trimEnd()}\n\n${block(ENV)}`; + fs.writeFileSync('CLAUDE.md', withBlock.replace('TypeScript', 'JavaScript')); + const r = await run(['vendor', 'restore', 'CLAUDE.md', '--format', 'json'], JSON_OPTS); + expect(JSON.parse(r.out)).toMatchObject({ status: 'refused', refusal: 'compiled-differs' }); + }); +}); + +// ============================================================================ +// 5. Regression and structure (INV-P5-1, -6, -9, -10) +// ============================================================================ + +describe('regression and structure', () => { + it('a repo with no retained content compiles byte-identically to main @ 67f3c28 (INV-P5-1)', async () => { + for (const [name, scenario] of Object.entries(MAIN_FIXTURE.scenarios)) { + process.chdir(originalCwd); + fs.rmSync(tmp, { recursive: true, force: true }); + fs.mkdirSync(tmp); + process.chdir(tmp); + scaffold(); + for (const [file, content] of Object.entries(scenario.setup)) fs.writeFileSync(file, content); + await run(scenario.args); + for (const [file, content] of Object.entries(scenario.files)) expect(read(file), `${name}: ${file}`).toBe(content); + } + }); + + it('write, check and restore compose through composeCompiled; score keeps the pure render (INV-P5-6, K4)', () => { + const src = (f: string): string => fs.readFileSync(path.join(__dirname, '..', 'commands', f), 'utf-8'); + expect(src('adf-compile.ts').match(/composeCompiled\(/g)?.length).toBe(2); + expect(src('adf-vendor.ts')).toMatch(/compiledMatchesFresh\(|composeCompiled\(/); + expect(src('score.ts')).not.toContain('composeCompiled'); + expect(src('score.ts')).not.toContain('adf-retained'); + }); + + it('adf-retained.ts never imports adf.ts, and works when the CLI entry loads first (INV-P5-10, #355)', async () => { + const src = fs.readFileSync(path.join(__dirname, '..', 'commands', 'adf-retained.ts'), 'utf-8'); + expect(src).not.toMatch(/from '\.\/adf'/); + vi.resetModules(); + await import('../index'); + const mod = await import('../commands/adf-retained'); + const notes = `${POINTER_AGENTS_MD}\n## Release Protocol\n\n- Tag first.\n`; + expect(mod.composeCompiled(FRESH, notes, 'AGENTS.md')).toMatchObject({ retainedSource: 'pointer', retainedSections: ['## Release Protocol'] }); + expect(mod.composeCompiled(FRESH, POINTER_AGENTS_MD, 'AGENTS.md').output).toBe(FRESH); + }); +}); diff --git a/packages/cli/src/__tests__/adf-vendor-consumers.test.ts b/packages/cli/src/__tests__/adf-vendor-consumers.test.ts index 56230e6..7868fc1 100644 --- a/packages/cli/src/__tests__/adf-vendor-consumers.test.ts +++ b/packages/cli/src/__tests__/adf-vendor-consumers.test.ts @@ -110,9 +110,9 @@ async function compiledOutput(target: string): Promise { return r.out; } -interface CheckRow { target: string; file: string; mode: string | null; source: string | null; status: string; extractable: number | null; remedy: string | null } +interface CheckRow { target: string; file: string; mode: string | null; source: string | null; status: string; extractable: number | null; remedy: string | null; staleReason?: string | null } interface CheckJson { stale: string[]; current: string[]; missing: string[]; skipped: string[]; upToDate: boolean; targets: CheckRow[]; nextActions: string[] } -interface WriteRow { target: string; file: string; state: string; source: string | null; status: string; recordedMode: string | null; previousMode: string | null; remedy: string | null } +interface WriteRow { target: string; file: string; state: string; source: string | null; status: string; recordedMode: string | null; previousMode: string | null; remedy: string | null; retainedSections?: string[]; retainedSource?: string } interface WriteJson { written: string[]; refused: string[]; skipped: string[]; targets: WriteRow[]; nextActions: string[] } interface TidyJson { files: Array<{ file: string; vendorFile: string | null; mode: string | null; source: string | null; status: string; skipReason: string | null; itemsExtracted: number }>; totalExtracted: number; warnings: string[] } @@ -142,7 +142,7 @@ describe('compile --check is mode-aware', () => { const byFile = Object.fromEntries(r.json.targets.map(t => [t.file, t])); expect(byFile['CLAUDE.md']).toMatchObject({ status: 'pointer-bloated', mode: 'pointer', source: 'inferred', remedy: 'charter adf tidy --source CLAUDE.md' }); expect(byFile['CLAUDE.md'].extractable).toBeGreaterThan(0); - expect(byFile['AGENTS.md']).toEqual({ target: 'agents', file: 'AGENTS.md', mode: 'pointer', source: 'inferred', status: 'pointer-ok', extractable: null, remedy: null }); + expect(byFile['AGENTS.md']).toEqual({ target: 'agents', file: 'AGENTS.md', mode: 'pointer', source: 'inferred', status: 'pointer-ok', extractable: null, remedy: null, staleReason: null }); expect(byFile['GEMINI.md']).toMatchObject({ status: 'mode-drift', mode: 'compiled', source: 'recorded', remedy: 'charter adf vendor set GEMINI.md pointer' }); expect(byFile['.cursorrules']).toMatchObject({ status: 'missing', mode: 'compiled', remedy: 'charter adf compile --target cursor --write' }); // Legacy keys keep their names and types (INV-P2-9). @@ -380,15 +380,17 @@ describe('compile --write decision table', () => { expect(manifestText()).toBe(vendors('CLAUDE.md: compiled')); }); - it('--mode compiled keeps the retained-section guard for pointers (#295) and records nothing on refusal', async () => { + it('--mode compiled carries a pointer\'s retained sections instead of refusing (#303), and records the switch', async () => { scaffold(); const withProtocol = `${POINTER_AGENTS_MD}\n## Release Protocol\n\n- Tag before publishing.\n`; fs.writeFileSync('AGENTS.md', withProtocol); const r = await runJson(['compile', '--target', 'agents', '--write', '--mode', 'compiled']); - expect(r.code).toBe(1); - expect(r.json.targets[0].status).toBe('refused-retained'); - expect(manifestText()).toBe(BASE_MANIFEST); - expect(read('AGENTS.md')).toBe(withProtocol); + expect(r.code).toBe(0); + expect(r.json.targets[0]).toMatchObject({ + status: 'written', previousMode: 'pointer', retainedSections: ['## Release Protocol'], retainedSource: 'pointer', + }); + expect(manifestText()).toBe(vendors('AGENTS.md: compiled')); + expect(read('AGENTS.md')).toContain('## Release Protocol\n\n- Tag before publishing.\n\n'); }); it('--mode compiled over a compiled or absent target writes as usual', async () => { diff --git a/packages/cli/src/__tests__/adf-vendor-doctor-score.test.ts b/packages/cli/src/__tests__/adf-vendor-doctor-score.test.ts index 9daafd2..14d5bcc 100644 --- a/packages/cli/src/__tests__/adf-vendor-doctor-score.test.ts +++ b/packages/cli/src/__tests__/adf-vendor-doctor-score.test.ts @@ -732,14 +732,14 @@ describe('adf vendor restore', () => { expect(read('CLAUDE.md')).toBe(BLOATED_CLAUDE); }); - it('refuses a compiled file carrying divergent retained sections (retained-divergent)', async () => { + it('carries a compiled file\'s own retained sections into the pointer (#303; retained-divergent retired)', async () => { scaffold(vendors('CLAUDE.md: pointer')); const compiled = `${await compiledOutput('claude')}\n## Environment\n- Use WSL for builds\n`; fs.writeFileSync('CLAUDE.md', compiled); const r = await restore('CLAUDE.md'); - expect(r.json).toMatchObject({ status: 'refused', refusal: 'retained-divergent' }); - expect(r.err).toContain(' [warn] CLAUDE.md carries your content under ## Environment, which the pointer template would drop. Move it into a .ai/ module first.'); - expect(read('CLAUDE.md')).toBe(compiled); + expect(r.json).toMatchObject({ status: 'restored', refusal: null, keptSections: ['## Environment'] }); + expect(read('CLAUDE.md')).toBe(restoredPointer('CLAUDE.md', null, '.ai', '## Environment\n- Use WSL for builds')); + expect(read('CLAUDE.md').endsWith('## Environment\n- Use WSL for builds\n')).toBe(true); }); it('refuses when the record cannot be written, and writes no file (record-write)', async () => { diff --git a/packages/cli/src/__tests__/fixtures/compile-no-retained.main-67f3c28.json b/packages/cli/src/__tests__/fixtures/compile-no-retained.main-67f3c28.json new file mode 100644 index 0000000..ecfe023 --- /dev/null +++ b/packages/cli/src/__tests__/fixtures/compile-no-retained.main-67f3c28.json @@ -0,0 +1,92 @@ +{ + "_note": "Generated by main @ 67f3c28 (before #303) with scratchpad gen-fixture.cjs. Each scenario: setup files written over the inputs, then `charter adf `, then the files on disk.", + "inputs": { + ".ai/manifest.adf": "ADF: 0.1\n\n🎯 ROLE: Test project\n\n📦 DEFAULT_LOAD:\n - core.adf\n - state.adf\n\n📂 ON_DEMAND:\n - frontend.adf (Triggers on: React, CSS)\n", + ".ai/core.adf": "ADF: 0.1\n\n📋 CONTEXT:\n - TypeScript monorepo\n\n⚠️ CONSTRAINTS [load-bearing]:\n - No secrets in commits\n\n📝 NOTES:\n Multi-line text value.\n Second line of it.\n", + ".ai/state.adf": "ADF: 0.1\n\n🧠 STATE:\n CURRENT: Active development\n", + ".ai/frontend.adf": "ADF: 0.1\n\n📋 CONTEXT:\n - React\n" + }, + "scenarios": { + "absent": { + "setup": {}, + "args": [ + "compile", + "--target", + "all", + "--write" + ], + "files": { + "CLAUDE.md": "\n\n# CLAUDE.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n", + "AGENTS.md": "\n\n# AGENTS.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n", + ".cursorrules": "# GENERATED by charter adf compile — DO NOT EDIT. Source of truth: .ai/ modules. Re-run: charter adf compile --target all --write\n\n# .cursorrules\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n# agents-modules\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n# /agents-modules\n", + "GEMINI.md": "\n\n# GEMINI.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n", + ".ai/manifest.adf": "ADF: 0.1\n\n🎯 ROLE: Test project\n\n📦 DEFAULT_LOAD:\n - core.adf\n - state.adf\n\n📂 ON_DEMAND:\n - frontend.adf (Triggers on: React, CSS)\n\n🏷 VENDORS:\n - CLAUDE.md: compiled\n - AGENTS.md: compiled\n - .cursorrules: compiled\n - GEMINI.md: compiled\n" + } + }, + "recompile-hand-edited": { + "setup": { + "CLAUDE.md": "\n\n# CLAUDE.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n- a stray hand edit\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n\n## PROTOCOL\n\n- a rule from a module since removed\n", + "AGENTS.md": "\n\n# AGENTS.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n- a stray hand edit\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n\n## PROTOCOL\n\n- a rule from a module since removed\n", + ".cursorrules": "# GENERATED by charter adf compile — DO NOT EDIT. Source of truth: .ai/ modules. Re-run: charter adf compile --target all --write\n\n# .cursorrules\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n- a stray hand edit\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n# agents-modules\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n# /agents-modules\n\n## PROTOCOL\n\n- a rule from a module since removed\n", + "GEMINI.md": "\n\n# GEMINI.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n- a stray hand edit\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n\n## PROTOCOL\n\n- a rule from a module since removed\n" + }, + "args": [ + "compile", + "--target", + "all", + "--write" + ], + "files": { + "CLAUDE.md": "\n\n# CLAUDE.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n", + "AGENTS.md": "\n\n# AGENTS.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n", + ".cursorrules": "# GENERATED by charter adf compile — DO NOT EDIT. Source of truth: .ai/ modules. Re-run: charter adf compile --target all --write\n\n# .cursorrules\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n# agents-modules\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n# /agents-modules\n", + "GEMINI.md": "\n\n# GEMINI.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n", + ".ai/manifest.adf": "ADF: 0.1\n\n🎯 ROLE: Test project\n\n📦 DEFAULT_LOAD:\n - core.adf\n - state.adf\n\n📂 ON_DEMAND:\n - frontend.adf (Triggers on: React, CSS)\n\n🏷 VENDORS:\n - CLAUDE.md: compiled\n - AGENTS.md: compiled\n - .cursorrules: compiled\n - GEMINI.md: compiled\n" + } + }, + "recompile-crlf": { + "setup": { + "CLAUDE.md": "\r\n\r\n# CLAUDE.md\r\n\r\n> Test project\r\n\r\n## CONTEXT\r\n\r\n- TypeScript monorepo\r\n\r\n## CONSTRAINTS _(load-bearing)_\r\n\r\n- No secrets in commits\r\n\r\n## NOTES\r\n\r\nMulti-line text value.\r\nSecond line of it.\r\n\r\n## STATE\r\n\r\n- **CURRENT**: Active development\r\n\r\n\r\n## Additional instructions\r\n\r\nRead the matching file before working in these areas:\r\n\r\n- `.ai/frontend.adf` — Triggers: React, CSS\r\n\r\n", + "AGENTS.md": "\r\n\r\n# AGENTS.md\r\n\r\n> Test project\r\n\r\n## CONTEXT\r\n\r\n- TypeScript monorepo\r\n\r\n## CONSTRAINTS _(load-bearing)_\r\n\r\n- No secrets in commits\r\n\r\n## NOTES\r\n\r\nMulti-line text value.\r\nSecond line of it.\r\n\r\n## STATE\r\n\r\n- **CURRENT**: Active development\r\n\r\n\r\n## Additional instructions\r\n\r\nRead the matching file before working in these areas:\r\n\r\n- `.ai/frontend.adf` — Triggers: React, CSS\r\n\r\n", + ".cursorrules": "# GENERATED by charter adf compile — DO NOT EDIT. Source of truth: .ai/ modules. Re-run: charter adf compile --target all --write\r\n\r\n# .cursorrules\r\n\r\n> Test project\r\n\r\n## CONTEXT\r\n\r\n- TypeScript monorepo\r\n\r\n## CONSTRAINTS _(load-bearing)_\r\n\r\n- No secrets in commits\r\n\r\n## NOTES\r\n\r\nMulti-line text value.\r\nSecond line of it.\r\n\r\n## STATE\r\n\r\n- **CURRENT**: Active development\r\n\r\n# agents-modules\r\n## Additional instructions\r\n\r\nRead the matching file before working in these areas:\r\n\r\n- `.ai/frontend.adf` — Triggers: React, CSS\r\n# /agents-modules\r\n", + "GEMINI.md": "\r\n\r\n# GEMINI.md\r\n\r\n> Test project\r\n\r\n## CONTEXT\r\n\r\n- TypeScript monorepo\r\n\r\n## CONSTRAINTS _(load-bearing)_\r\n\r\n- No secrets in commits\r\n\r\n## NOTES\r\n\r\nMulti-line text value.\r\nSecond line of it.\r\n\r\n## STATE\r\n\r\n- **CURRENT**: Active development\r\n\r\n\r\n## Additional instructions\r\n\r\nRead the matching file before working in these areas:\r\n\r\n- `.ai/frontend.adf` — Triggers: React, CSS\r\n\r\n" + }, + "args": [ + "compile", + "--target", + "all", + "--write" + ], + "files": { + "CLAUDE.md": "\n\n# CLAUDE.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n", + "AGENTS.md": "\n\n# AGENTS.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n", + ".cursorrules": "# GENERATED by charter adf compile — DO NOT EDIT. Source of truth: .ai/ modules. Re-run: charter adf compile --target all --write\n\n# .cursorrules\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n# agents-modules\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n# /agents-modules\n", + "GEMINI.md": "\n\n# GEMINI.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n", + ".ai/manifest.adf": "ADF: 0.1\n\n🎯 ROLE: Test project\n\n📦 DEFAULT_LOAD:\n - core.adf\n - state.adf\n\n📂 ON_DEMAND:\n - frontend.adf (Triggers on: React, CSS)\n\n🏷 VENDORS:\n - CLAUDE.md: compiled\n - AGENTS.md: compiled\n - .cursorrules: compiled\n - GEMINI.md: compiled\n" + } + }, + "pointer-clean": { + "setup": { + "CLAUDE.md": "# CLAUDE.md\n\n> **DO NOT add rules, constraints, or context to this file.**\n> This file is auto-managed by Charter. All project rules live in `.ai/`.\n> New rules should be added to the appropriate `.ai/*.adf` module.\n> See `.ai/manifest.adf` for the module routing manifest.\n\n## Session Start\n\nCall `charter_context` (MCP tool) before any other action. It returns the live constraint surface — open work, recent decisions, and metric ceilings — so you act within current project state rather than reconstructing it from scratch.\n\nIf `charter serve` is not running, read `.ai/context.snapshot.json` directly or run `charter context-refresh` to generate it.\n\n## Module Index\n\n\n\n## Environment\n\n", + "AGENTS.md": "# AGENTS.md\n\n> **DO NOT add instructions to this file.**\n> All agent instructions are managed in `.ai/` by Charter.\n> See `.ai/manifest.adf` for the module routing manifest.\n", + ".cursorrules": "DO NOT modify this file. All project rules are managed in .ai/ by Charter.\nSee .ai/manifest.adf for the module routing manifest.\n", + "GEMINI.md": "# GEMINI.md\n\n> **DO NOT add rules or context to this file.**\n> All project rules are managed in `.ai/` by Charter.\n> See `.ai/manifest.adf` for the module routing manifest.\n" + }, + "args": [ + "compile", + "--target", + "all", + "--write", + "--mode", + "compiled" + ], + "files": { + "CLAUDE.md": "\n\n# CLAUDE.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n", + "AGENTS.md": "\n\n# AGENTS.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n", + ".cursorrules": "# GENERATED by charter adf compile — DO NOT EDIT. Source of truth: .ai/ modules. Re-run: charter adf compile --target all --write\n\n# .cursorrules\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n# agents-modules\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n# /agents-modules\n", + "GEMINI.md": "\n\n# GEMINI.md\n\n> Test project\n\n## CONTEXT\n\n- TypeScript monorepo\n\n## CONSTRAINTS _(load-bearing)_\n\n- No secrets in commits\n\n## NOTES\n\nMulti-line text value.\nSecond line of it.\n\n## STATE\n\n- **CURRENT**: Active development\n\n\n## Additional instructions\n\nRead the matching file before working in these areas:\n\n- `.ai/frontend.adf` — Triggers: React, CSS\n\n", + ".ai/manifest.adf": "ADF: 0.1\n\n🎯 ROLE: Test project\n\n📦 DEFAULT_LOAD:\n - core.adf\n - state.adf\n\n📂 ON_DEMAND:\n - frontend.adf (Triggers on: React, CSS)\n\n🏷 VENDORS:\n - CLAUDE.md: compiled\n - AGENTS.md: compiled\n - .cursorrules: compiled\n - GEMINI.md: compiled\n" + } + } + } +} diff --git a/packages/cli/src/commands/adf-compile.ts b/packages/cli/src/commands/adf-compile.ts index 0d1d6d2..8b14556 100644 --- a/packages/cli/src/commands/adf-compile.ts +++ b/packages/cli/src/commands/adf-compile.ts @@ -16,6 +16,10 @@ * markers (#318). --write compiles only targets in compiled mode (or absent); * converting a pointer or unmanaged file needs --mode compiled, which records the * switch first. --check verifies each target by its mode. + * + * Compiled output keeps the user's retained sections (## Environment and the other + * headings adf tidy retains) in a block at the end of the file (#303); see + * adf-retained.ts. --write, --check and vendor restore all compose through it. */ import * as fs from 'node:fs'; @@ -25,7 +29,6 @@ import { compileAdf, COMPILE_TARGETS, TARGET_FILENAMES, - COMPILE_BANNER_MARKER, stripCharterSentinels, } from '@stackbilt/adf'; import type { CompileTarget } from '@stackbilt/adf'; @@ -41,8 +44,10 @@ import { POINTER_GEMINI_MD, } from './adf'; import { countExtractable, readRetainedSections } from './adf-tidy'; -import { readVendorRecord, recordedSpelling, recordVendorModes, resolveVendorMode } from './adf-vendor'; +import { inferVendorMode, readVendorRecord, recordedSpelling, recordVendorModes, resolveVendorMode } from './adf-vendor'; import type { ModeSource, VendorFile, VendorMode } from './adf-vendor'; +import { composeCompiled, linesConvertWouldDrop } from './adf-retained'; +import type { RetainedSource } from './adf-retained'; // ============================================================================ // Zod validation — at the CLI boundary per Zod-core-out architecture @@ -65,6 +70,12 @@ export const MSG = { writeFailedNotReverted: ' Its mode record still says compiled; restore it with: charter adf vendor set {file} {previous}', writtenSwitched: ' [ok] Written {file} (mode: {previous} -> compiled)', writtenOk: ' [ok] Written {file}', + writtenKept: ' [ok] Written {file} (kept {headings})', + writtenSwitchedKept: ' [ok] Written {file} (mode: {previous} -> compiled; kept {headings})', + refusedBloated: ' [warn] Refused to compile {file}: it carries {n} item(s) beyond the pointer that .ai/ does not hold yet.\n Move them into .ai/ first: charter adf tidy --source {file}\n Or discard them: charter adf compile --target {target} --write --mode compiled --force (discards {n} item(s))', + refusedWouldDrop: " [warn] Refused to compile {file}: {n} line(s) of yours would be lost ({lines}).\n Move them under ## Environment or into a .ai/ module, or discard them: charter adf compile --target {target} --write --mode compiled --force (discards {n} line(s))", + collision: '{file}: your retained section {heading} has the same heading as a compiled section; both are kept.', + checkStaleUnfenced: ' [stale] {file} — your {headings} section(s) sit outside the retained block.\n Run: charter adf compile --target {target} --write (keeps them)', nothingCompiled: ' No target is in compiled mode, so nothing was written. Pointer files stay pointers.\n To switch them: charter adf compile --target all --write --mode compiled. To see modes: charter adf vendor status', checkFresh: ' [ok] {file} is up-to-date', checkPointerOk: ' [ok] {file} is a thin pointer to {aiDir}/ ({source})', @@ -217,7 +228,7 @@ function runStdoutMode( } // ============================================================================ -// Retained-section protection (#295) +// Retained-section detection (#295, #303) // ============================================================================ /** @@ -269,9 +280,8 @@ function retainedSectionMap(content: string): Map { * invite them to use, or an operational `## ... Protocol` block that `adf tidy` * deliberately keeps in the vendor file (#198). * - * Compiled output carries none of these sections, so writing over them would - * destroy content no `.ai/` module holds. The guard refuses instead; `--force` - * remains the way through. + * No `.ai/` module holds these sections, so converting a pointer to compiled + * output carries them into its retained block (adf-retained.ts, #303). */ export function divergentRetainedHeadings(existing: string, filename: string): string[] { const templates = pointerTemplatesFor(filename); @@ -302,8 +312,10 @@ type WriteStatus = | 'skipped-mode' | 'refused-mode-change' | 'refused-hand-authored' - | 'refused-retained' + | 'refused-bloated' + | 'refused-would-drop' | 'refused-record'; +// 'refused-retained' is retired (#303): retained sections are carried, not refused over. interface WriteTargetOutcome { target: CompileTarget; @@ -316,6 +328,9 @@ interface WriteTargetOutcome { remedy: string | null; /** Set when the file write itself failed (status refused-record). */ writeError: string | null; + /** Retained headings carried into the written file, in order (#303). */ + retainedSections: string[]; + retainedSource: RetainedSource; } function runWriteMode( @@ -332,12 +347,15 @@ function runWriteMode( const rows: WriteTargetOutcome[] = []; const addIfAbsent: Array<{ file: VendorFile; mode: VendorMode }> = []; const modeSuffix = flags.modeCompiled ? ' --mode compiled' : ''; + /** A refusal's second remedy (the discarding one), listed after its first in nextActions. */ + const extraActions = new Map(); for (const target of targets) { const filename = TARGET_FILENAMES[target]; const { state, source } = targetState(aiDir, filename); const row: WriteTargetOutcome = { target, file: filename, state, source, status: 'written', recordedMode: null, previousMode: null, remedy: null, writeError: null, + retainedSections: [], retainedSource: 'none', }; rows.push(row); @@ -387,30 +405,38 @@ function runWriteMode( } continue; } - // A pointer stub the user has written notes into is not disposable: the - // compiler emits no Environment or Protocol section, and `adf tidy` routes - // none of it into .ai/, so overwriting would destroy it outright. - if (!existing.includes(COMPILE_BANNER_MARKER)) { - const divergent = divergentRetainedHeadings(existing, filename); - if (divergent.length > 0) { - row.status = 'refused-retained'; - row.remedy = `charter adf compile --target ${target} --write${modeSuffix} --force`; - if (!json) { - console.error( - ` [warn] Refused to overwrite ${filename} — it is a charter pointer carrying your own\n` + - ` content under ${divergent.join(', ')}. Compiled output has no such section,\n` + - ` and .ai/ does not hold this content, so it would be lost.\n` + - ` Move it into a .ai/ module first, or pass --force to discard it.`, - ); - } - continue; - } - } } // Render first: a compile error then leaves both record and file untouched. const result = compileAdf({ target, aiDir, displayAiDir, readFile }); + // Retained sections (#303) are carried into the output, never deleted, even + // under --force (W4). Converting a pointer refuses only over content the + // output would NOT carry: bloat tidy can fold into .ai/, or other stray lines. + const existing = fs.existsSync(filename) ? fs.readFileSync(filename, 'utf-8') : null; + const composed = composeCompiled(result.output, existing, filename); + // Decided by the file's markers, not by whether anything is carried: a + // pointer with no retained sections can still hold bloat or stray lines. + if (existing !== null && inferVendorMode(filename as VendorFile, existing).inferredMode === 'pointer' && !flags.force) { + const forceRemedy = `charter adf compile --target ${target} --write --mode compiled --force`; + const n = countExtractable(filename, aiDir); + if (n > 0) { + row.status = 'refused-bloated'; + row.remedy = `charter adf tidy --source ${filename}`; + extraActions.set(filename, forceRemedy); + if (!json) console.error(fill(MSG.refusedBloated, { file: filename, n, target })); + continue; + } + const lost = linesConvertWouldDrop(existing, composed.output); + if (lost.length > 0) { + const shown = lost.slice(0, 3).map(l => `'${l}'`).join(', ') + (lost.length > 3 ? `, and ${lost.length - 3} more` : ''); + row.status = 'refused-would-drop'; + row.remedy = forceRemedy; + if (!json) console.error(fill(MSG.refusedWouldDrop, { file: filename, n: lost.length, lines: shown, target })); + continue; + } + } + // --mode compiled: record the mode BEFORE writing, so a half-done switch can // never leave record and file disagreeing. A skipped upsert writes nothing. if (flags.modeCompiled && state !== 'absent-unrecorded') { @@ -433,7 +459,7 @@ function runWriteMode( // A failed write is reported per target, and the run goes on. A switch that // was already recorded is reverted, so record and file still agree. try { - fs.writeFileSync(filename, result.output, 'utf-8'); + fs.writeFileSync(filename, composed.output, 'utf-8'); } catch (err: unknown) { row.status = 'refused-record'; row.writeError = err instanceof Error ? err.message : String(err); @@ -457,10 +483,15 @@ function runWriteMode( } if (!upserted) addIfAbsent.push({ file: filename as VendorFile, mode: 'compiled' }); if (state === 'pointer' || state === 'unmanaged') row.previousMode = state; + row.retainedSections = composed.retainedSections; + row.retainedSource = composed.retainedSource; + for (const heading of composed.collisions) warn(fill(MSG.collision, { file: filename, heading })); if (!json) { + const headings = composed.retainedSections.join(', '); + const values = { file: filename, previous: row.previousMode ?? '', headings }; console.log(row.previousMode - ? fill(MSG.writtenSwitched, { file: filename, previous: row.previousMode }) - : fill(MSG.writtenOk, { file: filename })); + ? fill(headings ? MSG.writtenSwitchedKept : MSG.writtenSwitched, values) + : fill(headings ? MSG.writtenKept : MSG.writtenOk, values)); } } @@ -483,7 +514,7 @@ function runWriteMode( if (!json && skipped.length === rows.length) console.log(MSG.nothingCompiled); if (json) { - const remedies = dedupe([...refused, ...skipped].map(r => r.remedy).filter((r): r is string => r !== null)); + const remedies = dedupe([...refused, ...skipped].flatMap(r => [r.remedy, extraActions.get(r.file) ?? null]).filter((r): r is string => r !== null)); console.log(JSON.stringify({ written: rows.filter(r => r.status === 'written').map(r => r.file), refused: refused.map(r => r.file), @@ -518,6 +549,8 @@ interface CheckTargetResult { status: CheckStatus; extractable: number | null; remedy: string | null; + /** Set iff status is stale: 'retained-unfenced' when retained sections sit outside the retained block, else 'content'. */ + staleReason: 'content' | 'retained-unfenced' | null; } /** @@ -548,7 +581,7 @@ function runCheckMode( const file = TARGET_FILENAMES[target]; const ts = targetState(aiDir, file); const mode = ts.state === 'absent-unrecorded' ? null : ts.state; - const row: CheckTargetResult = { target, file, mode, source: ts.source, status: 'fresh', extractable: null, remedy: null }; + const row: CheckTargetResult = { target, file, mode, source: ts.source, status: 'fresh', extractable: null, remedy: null, staleReason: null }; rows.push(row); const source = ts.source ?? 'inferred'; const compileRemedy = `charter adf compile --target ${target} --write`; @@ -582,14 +615,20 @@ function runCheckMode( file, recorded: ts.state, inferred: ts.inferredMode, keep: keep.join(', then '), restore: `${restore}${discards}`, })); } else if (ts.state === 'compiled') { + // Expected bytes are what --write would write: the fresh render plus the + // file's own retained block, so an edit inside the block is never stale (#303). const onDisk = fs.readFileSync(file, 'utf-8'); const result = compileAdf({ target, aiDir, displayAiDir, readFile }); - if (onDisk === result.output) { + const composed = composeCompiled(result.output, onDisk, file); + if (onDisk === composed.output) { print(fill(MSG.checkFresh, { file })); } else { row.status = 'stale'; row.remedy = compileRemedy; - print(fill(MSG.checkStale, { file, aiDir: displayAiDir, target })); + row.staleReason = composed.unfenced.length > 0 ? 'retained-unfenced' : 'content'; + print(row.staleReason === 'retained-unfenced' + ? fill(MSG.checkStaleUnfenced, { file, headings: composed.unfenced.join(', '), target }) + : fill(MSG.checkStale, { file, aiDir: displayAiDir, target })); } } else if (ts.state === 'pointer') { // A pointer is checked for being intact, never against compiled output. diff --git a/packages/cli/src/commands/adf-retained.ts b/packages/cli/src/commands/adf-retained.ts new file mode 100644 index 0000000..1592aff --- /dev/null +++ b/packages/cli/src/commands/adf-retained.ts @@ -0,0 +1,236 @@ +/** + * Retained sections in compiled output (#303). + * + * `adf compile --write` used to delete a user's `## Environment` (or protocol) + * section from a compiled file, and refused to convert a pointer carrying one. + * Compiled output now keeps those sections in a fenced block at the end of the + * file: + * + * + * + * + * ## Environment + * - ... + * + * + * composeCompiled is the ONE function that says what "freshly compiled" means + * for an existing file. compile --write, compile --check and vendor restore all + * use it, so they cannot disagree. score deliberately does not: it asks whether + * .ai/ alone is substantive, so it keeps the pure compileAdf output. + * + * Nothing here imports adf.ts: pointer templates and markers are reached at + * call time through adf-compile and adf-vendor, so the #355 circular-import + * class cannot recur here. + */ + +import { readRetainedSections } from './adf-tidy'; +import { inferVendorMode, linesRestoreWouldDrop } from './adf-vendor'; +import type { VendorFile } from './adf-vendor'; +import { divergentRetainedHeadings } from './adf-compile'; + +export const RETAINED_START = ''; +export const RETAINED_END = ''; +/** First line inside a new block. The reader drops it. */ +export const RETAINED_NOTE = ''; + +/** + * A heading the renderer could have produced: all caps, with an optional weight + * suffix. Never treated as user content in a compiled file without a block, so + * a section since removed from .ai/ cannot be resurrected (#296 class). + */ +export const GENERATED_HEADING_RE = /^## [A-Z][A-Z0-9_]*( _\((load-bearing|advisory)\)_)?$/; + +/** The on-demand module index fence lines the compiler emits (html and hash forms). */ +const MODULE_INDEX_FENCE_LINES = new Set(['', '', '# agents-modules', '# /agents-modules']); + +/** + * Where retained content was read from: + * block the existing file has a retained block + * legacy a compiled file (banner) with retained sections outside any block + * pointer a pointer being converted: the retained sections that differ from its template + * unmanaged a hand-authored file being converted under --force: every retained section + * none no file, or nothing qualifies + */ +export type RetainedSource = 'block' | 'legacy' | 'pointer' | 'unmanaged' | 'none'; + +export interface ComposedCompiled { + /** What the file should hold: the fresh render, plus the retained block when there is content. */ + output: string; + /** Headings carried, in order. */ + retainedSections: string[]; + retainedSource: RetainedSource; + /** The carried text (no markers, no note). '' when nothing is carried. */ + body: string; + /** Headings of retained sections that sat outside a block (they move into it). */ + unfenced: string[]; + /** Carried headings that equal a heading the fresh render emits (both are kept). */ + collisions: string[]; + /** The existing file without its retained block and carried sections ('' when no file). */ + generated: string; +} + +interface Section { + heading: string; + start: number; + /** Exclusive. */ + end: number; +} + +const isH2 = (line: string): boolean => line.trim().startsWith('## '); + +/** tidy's own retained-heading rule (isRetainedHeading, minus `## Session Start`), reused unchanged. */ +const isRetained = (heading: string): boolean => readRetainedSections(heading) !== ''; + +/** H2 sections of `lines`. A section runs to the next H2, module index fence, retained marker, or the end. */ +function sectionsOf(lines: string[], from = 0, to = lines.length): Section[] { + const out: Section[] = []; + let current: Section | null = null; + for (let i = from; i < to; i++) { + const t = lines[i].trim(); + const boundary = isH2(lines[i]) || MODULE_INDEX_FENCE_LINES.has(t) || t === RETAINED_START || t === RETAINED_END; + if (boundary && current) { + current.end = i; + out.push(current); + current = null; + } + if (isH2(lines[i])) current = { heading: t, start: i, end: to }; + } + if (current) out.push(current); + return out; +} + +function trimBlankEdges(lines: string[]): string[] { + let a = 0; + let b = lines.length; + while (a < b && lines[a].trim() === '') a++; + while (b > a && lines[b - 1].trim() === '') b--; + return lines.slice(a, b); +} + +/** Join sections verbatim, with a blank line between two that would otherwise touch. */ +function joinSections(parts: string[][]): string[] { + const out: string[] = []; + for (const part of parts) { + const lines = trimBlankEdges(part); + if (lines.length === 0) continue; + if (out.length > 0) out.push(''); + out.push(...lines); + } + return out; +} + +const withoutWeight = (heading: string): string => heading.replace(/ _\((load-bearing|advisory)\)_$/, ''); + +/** + * User sections in compiled text with no block (or outside one): retained headings + * that the renderer could not have produced, other than `## Module Index`, and + * whose text the fresh render does not already hold (a .ai/ text value may embed + * a `## ` line; carrying it would duplicate it on every run). + */ +function legacySections(lines: string[], from: number, to: number, fresh: string): Section[] { + return sectionsOf(lines, from, to).filter(s => { + if (!isRetained(s.heading) || GENERATED_HEADING_RE.test(s.heading) || s.heading === '## Module Index') return false; + const text = trimBlankEdges(lines.slice(s.start, s.end)).join('\n'); + return !fresh.includes(text); + }); +} + +const sectionLines = (lines: string[], s: Section): string[] => + lines.slice(s.start, s.end).filter(l => l.trim() !== RETAINED_END && l.trim() !== RETAINED_NOTE); + +function newBlock(body: string[]): string { + return [RETAINED_START, RETAINED_NOTE, '', ...body, RETAINED_END].join('\n') + '\n'; +} + +/** + * Compose compiled output for `file` from the fresh render and the file on disk. + * Pure. With nothing to carry, the output is `fresh` byte for byte (P3). + */ +export function composeCompiled(fresh: string, existing: string | null, file: string): ComposedCompiled { + const none: ComposedCompiled = { + output: fresh, retainedSections: [], retainedSource: 'none', body: '', unfenced: [], collisions: [], generated: existing ?? '', + }; + if (existing === null) return none; + + const lines = existing.replace(/\r\n/g, '\n').split('\n'); + const removed = new Set(); + const take = (s: Section): string[] => { + for (let i = s.start; i < s.end; i++) removed.add(i); + return sectionLines(lines, s); + }; + + let source: RetainedSource; + let body: string[]; + let unfenced: string[] = []; + let block: string | null = null; + + const start = lines.findIndex(l => l.trim() === RETAINED_START); + if (start >= 0) { + // A start with no end runs to EOF, matching stripCharterSentinels. + let end = lines.findIndex((l, i) => i > start && l.trim() === RETAINED_END); + const closed = end >= 0; + if (!closed) end = lines.length; + for (let i = start; i < Math.min(end + 1, lines.length); i++) removed.add(i); + const inner = lines.slice(start + 1, end); + const blockBody = trimBlankEdges(inner.filter(l => l.trim() !== RETAINED_NOTE)); + const before = legacySections(lines, 0, start, fresh); + const after = closed ? legacySections(lines, end + 1, lines.length, fresh) : []; + unfenced = [...before, ...after].map(s => s.heading); + const beforeLines = before.map(take); + const afterLines = after.map(take); + body = joinSections([...beforeLines, blockBody, ...afterLines]); + source = 'block'; + // The block is copied from disk as it stands, so an edit inside it can never + // make --check stale (C1). It is rebuilt only to take in unfenced sections. + if (unfenced.length === 0) { + const raw = lines.slice(start, closed ? end + 1 : end); + while (!closed && raw.length > 1 && raw[raw.length - 1].trim() === '') raw.pop(); + block = (closed ? raw : [...raw, RETAINED_END]).join('\n') + '\n'; + } + } else { + const mode = inferVendorMode(file as VendorFile, lines.join('\n')).inferredMode; + if (mode === 'compiled') { + const found = legacySections(lines, 0, lines.length, fresh); + unfenced = found.map(s => s.heading); + body = joinSections(found.map(take)); + source = 'legacy'; + } else if (mode === 'pointer') { + const divergent = new Set(divergentRetainedHeadings(lines.join('\n'), file).filter(h => h !== '## Module Index')); + body = joinSections(sectionsOf(lines).filter(s => divergent.has(s.heading)).map(take)); + source = 'pointer'; + } else { + body = joinSections(sectionsOf(lines).filter(s => isRetained(s.heading) && s.heading !== '## Module Index').map(take)); + source = 'unmanaged'; + } + } + + const generated = lines.filter((_, i) => !removed.has(i)).join('\n'); + if (body.length === 0 && block === null) return { ...none, generated }; + + const retainedSections = body.filter(isH2).map(l => l.trim()); + const freshHeadings = new Set(fresh.split('\n').filter(isH2).map(l => withoutWeight(l.trim()))); + const collisions = retainedSections.filter(h => freshHeadings.has(withoutWeight(h))); + const output = fresh.trimEnd() + '\n\n' + (block ?? newBlock(body)); + return { + output, retainedSections, retainedSource: source, body: body.join('\n'), unfenced, collisions, generated, + }; +} + +/** + * Non-blank lines of `existing` that the composed compiled output would not keep, + * ignoring Charter pointer-template lines and the generated module index + * (linesRestoreWouldDrop semantics). + */ +export function linesConvertWouldDrop(existing: string, composed: string): string[] { + return linesRestoreWouldDrop(existing, composed); +} + +/** + * True when `current` (a compiled file) holds nothing beyond `fresh` but its + * retained content: the block, or retained sections outside one. Compared with + * CRLF and trailing whitespace normalised, as restore always has. + */ +export function compiledMatchesFresh(fresh: string, current: string, file: string): boolean { + const norm = (t: string): string => t.replace(/\r\n/g, '\n').trimEnd(); + return norm(composeCompiled(fresh, current, file).generated) === norm(fresh); +} diff --git a/packages/cli/src/commands/adf-vendor.ts b/packages/cli/src/commands/adf-vendor.ts index 77be5eb..4c920a5 100644 --- a/packages/cli/src/commands/adf-vendor.ts +++ b/packages/cli/src/commands/adf-vendor.ts @@ -36,6 +36,7 @@ import { } from './adf'; import { fileIdentity, generateModuleIndex } from './adf-migrate'; import { divergentRetainedHeadings } from './adf-compile'; +import { composeCompiled, compiledMatchesFresh } from './adf-retained'; import { countExtractable, readRetainedSections } from './adf-tidy'; import { insertManifestBullet, lastSectionIndex, replaceManifestBullet, sectionHeaders } from './adf-manifest-register'; @@ -151,7 +152,6 @@ export const MSG = { restoreModeNotPointer: ' [warn] {file} is {mode} ({source}); restore writes a pointer. Record that intent first: charter adf vendor set {file} pointer', restoreHandAuthored: ' [warn] {file} has hand-authored content that .ai/ does not hold. Move it first: charter adf migrate --source {file}', restoreBloated: ' [warn] {file} carries {n} item(s) beyond the pointer. Fold them into .ai/ first: charter adf tidy --source {file}', - restoreRetained: ' [warn] {file} carries your content under {headings}, which the pointer template would drop. Move it into a .ai/ module first.', restoreLost: ' [warn] {file} has {n} line(s) the pointer template would drop: {lines}. Move them into a .ai/ module (or under ## Environment), or delete them, first.', restoreCompiledDiffers: ' [warn] {file} differs from what .ai/ compiles to (a hand edit, or .ai/ changed since), and restoring the pointer would lose that difference. Check it with: charter adf compile --target {target} --check. To keep the file as it is: charter adf vendor set {file} compiled', restoreRecord: ' [warn] {file} was not restored: its mode could not be recorded as pointer in {manifestPath}. Fix the manifest or run: charter adf vendor status', @@ -754,7 +754,6 @@ export type RestoreRefusal = | 'mode-not-pointer' | 'hand-authored' | 'pointer-bloated' - | 'retained-divergent' | 'compiled-differs' | 'record-write' | 'write-failed'; @@ -800,11 +799,13 @@ function withoutModuleIndexSection(block: string): string { * sections re-appended verbatim (as tidy's restorePointer does) and, for * CLAUDE.md, the module index filled in (as bootstrap's updateModuleIndex does). * A clean bootstrap pointer therefore comes back byte-equal. Pure: never writes. + * `carried` is a compiled file's retained text (composeCompiled's body), appended + * as plain sections in place of reading `current` (#303). */ -export function restoredPointer(file: VendorFile, current: string | null, aiDir: string): string { +export function restoredPointer(file: VendorFile, current: string | null, aiDir: string, carried = ''): string { let pointer = restoreTemplate(file); const hasIndex = pointer.includes(MODULE_INDEX_START); - let retained = current === null ? '' : readRetainedSections(current); + let retained = carried ? `\n\n${carried}\n` : current === null ? '' : readRetainedSections(current); if (hasIndex) retained = withoutModuleIndexSection(retained); if (retained) pointer = pointer.replace(/\n## Environment[\s\S]*$/, '').trimEnd() + retained; const index = hasIndex ? generateModuleIndex(aiDir) : ''; @@ -917,13 +918,13 @@ export function restoreVendorFile(aiDir: string, file: VendorFile, options: Rest const n = countExtractable(file, aiDir); if (n > 0) return refuse('pointer-bloated', `charter adf tidy --source ${file}`, fill(MSG.restoreBloated, { file, n })); } + let carried = ''; + let carriedSections: string[] = []; if (ev.inferredMode === 'compiled' && current !== null) { - const divergent = divergentRetainedHeadings(current, file); - if (divergent.length > 0) { - return refuse('retained-divergent', null, fill(MSG.restoreRetained, { file, headings: divergent.join(', ') })); - } // Compiled output that .ai/ no longer reproduces holds something .ai/ does not // (a hand edit, or rules since removed from .ai/). Compiled in memory; nothing is written. + // Its retained block (or retained sections outside one) is the user's: it is + // carried into the pointer, so it never counts as a difference (#303). const target = (Object.keys(TARGET_FILENAMES) as CompileTarget[]).find(t => TARGET_FILENAMES[t] === file); let fresh: string | null = null; try { @@ -931,20 +932,23 @@ export function restoreVendorFile(aiDir: string, file: VendorFile, options: Rest } catch { fresh = null; } - const norm = (t: string): string => t.replace(/\r\n/g, '\n').trimEnd(); - if (fresh === null || norm(fresh) !== norm(current)) { + if (fresh === null || !compiledMatchesFresh(fresh, current, file)) { return refuse('compiled-differs', `charter adf vendor set ${file} compiled`, fill(MSG.restoreCompiledDiffers, { file, target: target ?? 'all' })); } + const composed = composeCompiled(fresh, current, file); + carried = composed.body; + carriedSections = composed.retainedSections; } - // Only a pointer's retained sections are the user's; a compiled file's come from .ai/. - // The template brings its own Module Index, so that heading is never "kept". + // A pointer's retained sections that differ from its template are the user's; + // a compiled file's are the ones composeCompiled carries. The template brings + // its own Module Index, so that heading is never "kept". const keptSections = ev.inferredMode === 'pointer' && current !== null ? divergentRetainedHeadings(current, file).filter(h => h !== '## Module Index') - : []; + : carriedSections; const kept = keptSections.length > 0 ? ` (kept ${keptSections.join(', ')})` : ''; - const next = restoredPointer(file, ev.inferredMode === 'pointer' ? current : null, aiDir); + const next = restoredPointer(file, ev.inferredMode === 'pointer' ? current : null, aiDir, carried); // Any line of the user's that the new pointer would not carry is a loss: refuse. if (ev.inferredMode === 'pointer' && current !== null) { const lost = linesRestoreWouldDrop(current, next); From f18afd8561060e644a51815d2b2043d0bbd38fc0 Mon Sep 17 00:00:00 2001 From: Kurt Overmier Date: Fri, 16 Oct 2026 15:42:32 -0500 Subject: [PATCH 2/3] fix(adf): retained carry ignores rendered .ai/ text and code fences (refs #303) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 +- docs/cli-reference.md | 15 +- .../cli/src/__tests__/adf-retained.test.ts | 164 +++++++++- packages/cli/src/commands/adf-compile.ts | 12 +- packages/cli/src/commands/adf-retained.ts | 295 +++++++++++++++--- packages/cli/src/commands/adf-vendor.ts | 15 +- 6 files changed, 452 insertions(+), 51 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31f71e6..3a9ddc3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ The format is based on Keep a Changelog and follows Semantic Versioning. - **Compiled files that carry your own retained sections change bytes on the next `adf compile --write`** (`#303`, `#318` part 5) — a compiled `CLAUDE.md`, `AGENTS.md`, `.cursorrules` or `GEMINI.md` with a hand-added `## Environment` (or another section `adf tidy` retains) no longer loses it. The next `--write` moves it into a `` … `` block at the end of the file, and every later `--write` copies that block as it stands. A compiled file with nothing to carry gets no block and compiles byte-identically to before. The block uses HTML markers in every target, `.cursorrules` included. All-caps headings the compiler could have produced (a module since removed from `.ai/`) and `## Module Index` are never carried. `adf compile` to stdout and `charter score` still use the plain render. - **`adf compile --check` reads such a file as stale until one `--write`** (`#303`) — the expected bytes are now the render plus the file's own retained block, so an edit inside the block is never stale. A compiled file whose retained sections sit outside the block reads `[stale]` with `staleReason: "retained-unfenced"`, and the remedy (`compile --target --write`) keeps them. Each `--check` JSON row adds `staleReason` (`content`, `retained-unfenced`, or null). -- **`refused-retained` is retired from `adf compile --write`, and `retained-divergent` from `adf vendor restore`** (`#303`) — JSON consumers no longer see either value. A pointer converted with `--mode compiled` now carries the retained sections that differ from its template into the block, instead of refusing. It still refuses, exit 1 with nothing written and `VENDORS` unchanged, over content the output would not carry: `refused-bloated` (items `adf tidy` would extract; the first remedy is `adf tidy --source `, then `--force`, which discards them) and `refused-would-drop` (other lines of yours). `--mode compiled --force` over a hand-authored file carries its retained sections and discards the rest. `--force` never drops retained sections. Write JSON rows add `retainedSections` and `retainedSource`. `vendor restore` of a compiled file carries its retained sections into the pointer and lists them in `keptSections`, and the block is not counted as a `compiled-differs` difference. +- **`refused-retained` is retired from `adf compile --write`, and `retained-divergent` from `adf vendor restore`** (`#303`) — JSON consumers no longer see either value. A pointer converted with `--mode compiled` now carries the retained sections that differ from its template into the block, instead of refusing. It still refuses, exit 1 with nothing written and `VENDORS` unchanged, over content the output would not carry: `refused-bloated` (items `adf tidy` would extract; the first remedy is `adf tidy --source `, then `--force`, which discards them) and `refused-would-drop` (other lines of yours). `--mode compiled --force` over a hand-authored file carries its retained sections and discards the rest. `--force` never drops retained sections. Write JSON rows add `retainedSections`, `retainedSource` and `retainedSkipped`. That last one lists retained-looking sections that are really rendered `.ai/` text left behind by an edit or removal in `.ai/`; they are not carried, and each one is named on stderr. Parsing is code-fence aware, and marker lines match as loosely as `stripCharterSentinels`. `vendor restore` of a compiled file carries its retained sections into the pointer and lists them in `keptSections`, and the block is not counted as a `compiled-differs` difference. A block holding a non-retained heading, or text before its first heading, is refused as `would-drop` (new value) with `vendor set compiled` as the first remedy, because a pointer cannot keep it. - **The pre-commit hook tidies only the staged vendor files and re-stages only the files tidy wrote** (`#320`, `#318` part 4) — re-run `charter hook install --pre-commit` to pick up staged-only tidy. An installed hook keeps its old behaviour until you do, and `charter doctor` reports it with an INFO `pre-commit hook` row (INFO, so `--ci` does not fail). Reinstalling overwrites a Charter-managed hook without `--force`, as before. The old hook ran a whole-tree `adf tidy` whenever any vendor file was staged, then ran `git add` on every staged vendor file and on all of `.ai/*.adf`. That swept unrelated `.ai/` edits into the commit and rewrote vendor files that were not part of it. The new hook passes each staged vendor file to `adf tidy` with `--source` and stages only the paths that both the dry run and the real run report. A staged vendor file that also has unstaged changes is skipped with a message, because tidy works on the working tree and staging its result would commit those hunks. If tidy would write into a module that has unstaged changes or is untracked, nothing is tidied for that commit and a message says why. A gitignored module (for example an `.ai/` kept out of git) is not treated as work in progress: tidy writes it, and the hook says it was left unstaged. Any file the hook cannot stage, and any file the real run wrote that the dry run did not plan, is named and left unstaged rather than aborting the hook. A staged lowercase `agents.md` is now tidied (`#302`), and a commit staging only compiled output still makes no CLI call. The tidy block itself never fails a commit, but the ADF evidence gate after it can, and it is unchanged. In the generated hook (no `verify:adf` script), vendor bloat that a skipped tidy leaves in the working tree still fails `charter doctor --adf-only --ci`. The skip message now says so and names the ways out: stage or stash the unstaged work; run `charter adf tidy --source ` and `git add` the result; or `git commit --no-verify`. When the gate then fails, one line says which skipped file still carries how many items. A new hook with an older local CLI that does not know `--print-paths` reports the files as clean and tidies nothing, so upgrade the CLI too. - **BREAKING: `charter adf compile --write` no longer converts pointer files without `--mode compiled`** (`#318`, part 2) — compile used to overwrite any Charter thin pointer with compiled output, silently changing its mode. Each target's mode now comes from the `VENDORS` record, or from its markers when there is no entry, and `--write` only writes targets that are compiled or absent. Under `--target all`, pointer and unmanaged targets are skipped (exit 0), and each `[skip]` line names the command that converts it. A single such `--target` is refused (exit 1). `--force` alone never changes a mode. `--mode compiled` records `compiled` before writing. If that cannot be recorded, nothing is written (`refused-record`). If the write then fails, the record is restored and the run continues. A converted file is reported as `Written CLAUDE.md (mode: pointer -> compiled)`. JSON output adds `skipped` and per-target `targets`. A compiled-only repo with no `VENDORS` section compiles as before. diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 5e88c92..4406af4 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -542,7 +542,7 @@ Default with no `--write` or `--check` prints to stdout, single target only. | pointer | skipped (its `[skip]` line names the command), exit 0 | refused, exit 1 | record set to compiled, then written as `Written CLAUDE.md (mode: pointer -> compiled)` | | unmanaged | skipped, exit 0 | refused, exit 1 | as pointer, but an existing hand-authored file also needs `--force` | -When every target is skipped, compile says that nothing was written and how to switch. JSON output keeps `written`, `refused` and `nextActions`, and adds `skipped` and `targets` (one row per target: `state`, `source`, `status`, `recordedMode`, `previousMode`, `remedy`, `writeError`, `retainedSections`, `retainedSource`). `previousMode` (`pointer` or `unmanaged`) is how a JSON consumer tells a conversion from a re-compile. +When every target is skipped, compile says that nothing was written and how to switch. JSON output keeps `written`, `refused` and `nextActions`, and adds `skipped` and `targets` (one row per target: `state`, `source`, `status`, `recordedMode`, `previousMode`, `remedy`, `writeError`, `retainedSections`, `retainedSource`, `retainedSkipped`). `previousMode` (`pointer` or `unmanaged`) is how a JSON consumer tells a conversion from a re-compile. **Retained sections.** Compiled output keeps your own `## Environment` section and the other sections `adf tidy` retains (`## Session ...` other than `## Session Start`, and headings containing Protocol, Receipt, Provenance or Registration). They live in a block at the very end of the file, after the module index: @@ -564,13 +564,22 @@ When every target is skipped, compile says that nothing was written and how to s A file with nothing to carry gets no block, so its output is byte-identical to the plain render. The block uses the HTML markers in every target, `.cursorrules` included. Sections keep their source order, and the output is LF. A carried section whose heading matches a compiled one (say `## CONTEXT`) is kept, with a warning on stderr. The written line names what was kept: `Written CLAUDE.md (kept ## Environment)`. `adf compile` without `--write` prints the plain render, with no block. `adf tidy`, `doctor` and `adf migrate` ignore the block, because it is a Charter sentinel. -Converting a pointer refuses (exit 1, nothing written, `VENDORS` unchanged) when content the output would not carry would be lost: +A retained-looking section that is really rendered `.ai/` text is not carried. This covers a `## Session notes` or `## Environment` line inside a `.ai/` text value, left in the compiled file after you edited or removed that value. Two signs mark such a section. It sits inside a generated section's span (from a `## KEY` heading to the next generated heading or the module index) that no longer matches the current render. Or its heading appears in a current `.ai/` module. Compile prints `: was not carried: it looks like compiler output from .ai/. If it is yours, move it into the retained block.` and lists the heading in the row's `retainedSkipped`. A section appended after the last rendered section is still carried when that section matches the current render. + +Parsing follows Markdown. A `## ` line, a module index fence or a retained marker inside a ```` ``` ```` or `~~~` code fence is text, and an indented ` ## x` is code, not a heading. Marker lines match with the same loose whitespace that `stripCharterSentinels` accepts (`` works). A marker line that the render itself contains, quoted in a `.ai/` text value, never starts the block. + +Converting a pointer refuses (exit 1, nothing written, `VENDORS` unchanged) when content the output would not carry would be lost. This also applies to a file recorded `compiled` that still looks like a pointer, under plain `--write`: - `refused-bloated`: the pointer carries items `adf tidy` would extract. The first remedy is `charter adf tidy --source ` (moves them into `.ai/`, then re-run). The second is `--mode compiled --force`, which discards N item(s). - `refused-would-drop`: other lines of yours (an extra note, an edited title, edited `## Session Start` text) would be lost. Charter pointer-template lines and the module index do not count. Move them under `## Environment` or into a `.ai/` module, or pass `--force`, which discards N line(s). `--force` skips these refusals but never drops retained sections. `refused-retained` is retired: a pointer with notes under a retained heading is now written, with the notes carried. +Limitations: +- Outside the block, only retained-heading sections are kept. A non-retained section after the block, or the non-retained content of a second `charter:retained` block, is dropped, as before. +- Edits to a compiled file's own `## Session Start`, `## Module Index` or preamble are not kept. The banner says DO NOT EDIT, and that is by design. +- `--check` still compares bytes, so a CRLF checkout of a compiled file reads stale. + `--check` per target: | Status | When | Exit | @@ -682,7 +691,7 @@ npx charter adf vendor restore CLAUDE.md # rewrite one pointer from i - `status`: read-only. A file with a valid entry shows its recorded mode, plus `(recorded; file looks compiled)` when the markers on disk disagree. A file without one shows the mode inferred from its markers, the way `adf tidy` classifies it: the compile banner means `compiled`, a pointer marker means `pointer`, and anything else is `unmanaged`. Malformed, duplicate (the last entry wins), unknown-file and not-allowed entries are reported and ignored. A missing manifest exits 0 with a warning. - `set `: records or replaces one entry, editing only that line. Only `CLAUDE.md`, `AGENTS.md`, `.cursorrules` and `GEMINI.md` may be `compiled`. `agents.md` and `copilot-instructions.md` have no compile target, so `compiled` is refused with exit 1. An unknown file or mode, or a missing manifest, exits 2. The file does not have to exist yet. - `init`: records the inferred mode of every vendor file on disk that has no entry. It never changes an existing entry, and a second run writes nothing. -- `restore `: rewrites one file as its thin pointer template (`CLAUDE.md` gets the hybrid template with the module index filled in, as bootstrap writes it). It runs only when the file's mode is `pointer` (recorded or inferred), or the file is absent with no entry, and it refuses (exit 1, file untouched) whenever content would be lost: `mode-not-pointer` (record the intent first with `vendor set pointer`), `hand-authored` (run `adf migrate --source `), `pointer-bloated` (run `adf tidy --source `), and `compiled-differs` (a compiled file that differs from what `.ai/` compiles to now, checked in memory; keep it with `vendor set compiled`). A compiled file's retained block, and retained sections outside one, are not a difference: they are carried into the pointer as plain sections and listed in `keptSections`. The `retained-divergent` refusal is retired. For a pointer, every non-blank line must survive: a line that is not from a Charter pointer template (current or earlier) or the generated module index, and that the new pointer would not carry, is refused as `hand-authored` with the first few lines named. Retained sections of a pointer are carried over verbatim and listed. It is the fix for a recorded pointer that now holds compiled output. A file with no entry gets `pointer` recorded before the file is written; if that cannot be recorded, nothing is written (`record-write`), and if the file write fails, the manifest is put back byte for byte (`write-failed`, with `writeError`). A result identical to the file on disk is `unchanged` and writes nothing. `--dry-run` reports `planned`. An unknown file name exits 2. JSON output: `file`, `status` (`restored`, `planned`, `unchanged`, `refused`), `refusal`, `keptSections`, `remedy`, `writeError`; text lines go to stderr under `--format json`. +- `restore `: rewrites one file as its thin pointer template (`CLAUDE.md` gets the hybrid template with the module index filled in, as bootstrap writes it). It runs only when the file's mode is `pointer` (recorded or inferred), or the file is absent with no entry, and it refuses (exit 1, file untouched) whenever content would be lost: `mode-not-pointer` (record the intent first with `vendor set pointer`), `hand-authored` (run `adf migrate --source `), `pointer-bloated` (run `adf tidy --source `), and `compiled-differs` (a compiled file that differs from what `.ai/` compiles to now, checked in memory; keep it with `vendor set compiled`). A compiled file's retained block, and retained sections outside one, are not a difference: they are carried into the pointer as plain sections and listed in `keptSections`. The `retained-divergent` refusal is retired. A block holding anything a pointer cannot keep as a retained section is refused as `would-drop`, naming it, and the file is left as it is. That covers a non-retained heading such as `## My Notes`, and text before the first heading (`adf tidy` would move it out of the pointer). The first remedy is `vendor set compiled`, which keeps the file compiled. Otherwise, move that content under `## Environment` or into a `.ai/` module first. For a pointer, every non-blank line must survive: a line that is not from a Charter pointer template (current or earlier) or the generated module index, and that the new pointer would not carry, is refused as `hand-authored` with the first few lines named. Retained sections of a pointer are carried over verbatim and listed. It is the fix for a recorded pointer that now holds compiled output. A file with no entry gets `pointer` recorded before the file is written; if that cannot be recorded, nothing is written (`record-write`), and if the file write fails, the manifest is put back byte for byte (`write-failed`, with `writeError`). A result identical to the file on disk is `unchanged` and writes nothing. `--dry-run` reports `planned`. An unknown file name exits 2. JSON output: `file`, `status` (`restored`, `planned`, `unchanged`, `refused`), `refusal`, `keptSections`, `remedy`, `writeError`; text lines go to stderr under `--format json`. One physical file has one entry. When two vendor names are the same file (`AGENTS.md` and `agents.md` on a case-insensitive filesystem, or a hard link), every writer records it under the spelling already recorded, or `AGENTS.md` when neither is. `status` shows it once. `set` names the spelling it used. - `--dry-run`: with `set`, `init` or `restore`, report the change without writing diff --git a/packages/cli/src/__tests__/adf-retained.test.ts b/packages/cli/src/__tests__/adf-retained.test.ts index 7f07ae8..cdcadd7 100644 --- a/packages/cli/src/__tests__/adf-retained.test.ts +++ b/packages/cli/src/__tests__/adf-retained.test.ts @@ -88,7 +88,7 @@ async function fresh(target: string): Promise { return (await run(['compile', '--target', target])).out; } -interface WriteRow { file: string; status: string; previousMode: string | null; recordedMode: string | null; remedy: string | null; retainedSections: string[]; retainedSource: string } +interface WriteRow { file: string; status: string; previousMode: string | null; recordedMode: string | null; remedy: string | null; retainedSections: string[]; retainedSource: string; retainedSkipped: string[] } interface WriteJson { written: string[]; refused: string[]; targets: WriteRow[]; nextActions: string[] } interface CheckRow { file: string; status: string; staleReason: string | null; remedy: string | null } interface CheckJson { upToDate: boolean; targets: CheckRow[]; nextActions: string[] } @@ -496,3 +496,165 @@ describe('regression and structure', () => { expect(mod.composeCompiled(FRESH, POINTER_AGENTS_MD, 'AGENTS.md').output).toBe(FRESH); }); }); + +// ============================================================================ +// 6. Review round 1: rendered .ai/ text, markers in .ai/ text, code fences +// ============================================================================ + +const NOTES_CORE = (body: string): string => + `ADF: 0.1\n\n\u{1F4CB} CONTEXT:\n - TypeScript monorepo\n\n\u{1F4DD} NOTES:\n${body.split('\n').map(l => ` ${l}`).join('\n')}\n`; +const TARGET_OF_FILE: Record = { 'CLAUDE.md': 'claude', 'AGENTS.md': 'agents', '.cursorrules': 'cursor', 'GEMINI.md': 'gemini' }; +type SkipJson = WriteJson; + +describe('rendered .ai/ text is never carried as the user\'s (review item 1)', () => { + const cases: Array<[string, string, string | null]> = [ + ['edited', 'Intro line\n## Session notes\n- keep tabs', 'Intro line\n## Session notes\n- keep spaces'], + ['edited (## Environment)', '## Environment\nUse WSL.', '## Environment\nUse Docker.'], + ['removed', 'Intro line\n## Session notes\n- keep tabs', null], + ]; + for (const [name, before, after] of cases) { + it(`does not carry the old rendered copy when the .ai/ value is ${name}, and reports it`, async () => { + scaffold(); + fs.writeFileSync('.ai/core.adf', NOTES_CORE(before)); + await run(['compile', '--target', 'all', '--write']); + fs.writeFileSync('.ai/core.adf', after === null ? 'ADF: 0.1\n\n\u{1F4CB} CONTEXT:\n - TypeScript monorepo\n' : NOTES_CORE(after)); + const heading = before.split('\n').find(l => l.startsWith('## ')) as string; + const r = await runJson(['compile', '--target', 'all', '--write']); + expect(r.code).toBe(0); + for (const t of r.json.targets) { + expect(t.retainedSkipped, t.file).toEqual([heading]); + expect(t.retainedSource, t.file).toBe('none'); + } + expect(r.err).toContain(`CLAUDE.md: ${heading} was not carried: it looks like compiler output from .ai/. If it is yours, move it into the retained block.`); + for (const f of TARGET_FILES) { + expect(read(f), f).not.toContain(RETAINED_START); + expect(read(f), f).toBe(await fresh(TARGET_OF_FILE[f])); + } + const again = await runJson(['compile', '--target', 'all', '--write']); + expect(again.json.targets.every(t => t.retainedSkipped.length === 0)).toBe(true); + }); + } + + it('does not carry a legacy section whose heading is in a current .ai/ module (rule b)', () => { + const c = composeCompiled(FRESH, `${FRESH}\n## Session notes\n- old\n`, 'CLAUDE.md', { aiLines: new Set(['## Session notes']) }); + expect(c).toMatchObject({ output: FRESH, skipped: ['## Session notes'], retainedSource: 'none' }); + }); + + it('still carries a genuine ## Environment appended after the last rendered section (no module index)', async () => { + scaffold(BASE_MANIFEST.replace(/\n\u{1F4C2} ON_DEMAND:\n.*\n/u, '\n')); + fs.writeFileSync('.ai/core.adf', NOTES_CORE('Intro line\n## Session notes\n- keep tabs')); + await run(['compile', '--target', 'claude', '--write']); + const render = read('CLAUDE.md'); + expect(render).not.toContain('agents-modules'); + fs.writeFileSync('CLAUDE.md', `${render}\n${ENV}\n`); + const r = await runJson(['compile', '--target', 'claude', '--write']); + expect(r.json.targets[0]).toMatchObject({ retainedSections: ['## Environment'], retainedSource: 'legacy', retainedSkipped: [] }); + expect(read('CLAUDE.md')).toBe(`${render.trimEnd()}\n\n${block(ENV)}`); + }); +}); + +describe('a marker line inside rendered .ai/ text (review item 2)', () => { + it('is never taken as the block start: three --write runs are byte-identical and --check is fresh', async () => { + scaffold(); + fs.writeFileSync('.ai/core.adf', NOTES_CORE(`Quoted marker:\n${RETAINED_START}\nend of quote`)); + await run(['compile', '--target', 'all', '--write']); + for (const f of TARGET_FILES) fs.writeFileSync(f, `${read(f)}\n${ENV}\n`); + await run(['compile', '--target', 'all', '--write']); + const first = Object.fromEntries(TARGET_FILES.map(f => [f, read(f)])); + for (let i = 0; i < 3; i++) { + await run(['compile', '--target', 'all', '--write']); + for (const f of TARGET_FILES) expect(read(f), `${f} run ${i}`).toBe(first[f]); + } + for (const f of TARGET_FILES) expect(first[f].endsWith(block(ENV)), f).toBe(true); + expect((await run(['compile', '--target', 'all', '--check'])).code).toBe(0); + }); + + it('still finds the real block when the rendered part is stale and the render holds a marker', async () => { + scaffold(); + fs.writeFileSync('.ai/core.adf', NOTES_CORE(`Quoted marker:\n${RETAINED_START}\nend of quote`)); + await run(['compile', '--target', 'claude', '--write']); + fs.writeFileSync('CLAUDE.md', `${read('CLAUDE.md')}\n${ENV}\n`); + await run(['compile', '--target', 'claude', '--write']); + fs.writeFileSync('.ai/core.adf', NOTES_CORE(`Quoted marker, edited:\n${RETAINED_START}\nend of quote`)); + await run(['compile', '--target', 'claude', '--write']); + const once = read('CLAUDE.md'); + expect(once).toBe(`${(await fresh('claude')).trimEnd()}\n\n${block(ENV)}`); + await run(['compile', '--target', 'claude', '--write']); + expect(read('CLAUDE.md')).toBe(once); + }); + + it('ignores a marker the render holds when there is no block yet', async () => { + scaffold(); + fs.writeFileSync('.ai/core.adf', NOTES_CORE(`${RETAINED_START}\n## Environment\nfrom .ai`)); + await run(['compile', '--target', 'claude', '--write']); + const render = read('CLAUDE.md'); + for (let i = 0; i < 3; i++) await run(['compile', '--target', 'claude', '--write']); + expect(read('CLAUDE.md')).toBe(render); + }); +}); + +describe('code fences inside retained sections (review item 3)', () => { + const FENCED = '## Environment\n\n```sh\n## install deps\npnpm i\n\n```\n\n~~~\n## also code\n~~~\n ## indented code\n- after fence'; + + it('keeps a legacy section whole across ``` and ~~~ fences and indented code', () => { + const c = composeCompiled(FRESH, `${FRESH}\n${FENCED}\n`, 'CLAUDE.md'); + expect(c.retainedSections).toEqual(['## Environment']); + expect(c.output).toBe(`${FRESH.trimEnd()}\n\n${block(FENCED)}`); + expect(composeCompiled(FRESH, c.output, 'CLAUDE.md').output).toBe(c.output); + }); + + it('reports only real headings from a block, and an end marker inside a fence is text', () => { + const withFencedEnd = `${FRESH}\n${RETAINED_START}\n${FENCED}\n\`\`\`\n${RETAINED_END}\n\`\`\`\n- still mine\n${RETAINED_END}\n`; + const c = composeCompiled(FRESH, withFencedEnd, 'CLAUDE.md'); + expect(c.retainedSections).toEqual(['## Environment']); + expect(c.output).toBe(withFencedEnd); + expect(c.body).toContain('- still mine'); + }); + + it('carries fenced content from a hand-authored file under --force', async () => { + scaffold(); + fs.writeFileSync('CLAUDE.md', `# Mine\n\nHouse rules.\n\n${FENCED}\n`); + const r = await runJson(['compile', '--target', 'claude', '--write', '--mode', 'compiled', '--force']); + expect(r.json.targets[0]).toMatchObject({ retainedSections: ['## Environment'], retainedSource: 'unmanaged' }); + expect(read('CLAUDE.md')).toContain(block(FENCED)); + }); + + it('restore keeps the fenced content and reports no fake heading', async () => { + scaffold(vendors('CLAUDE.md: pointer')); + fs.writeFileSync('CLAUDE.md', `${(await fresh('claude')).trimEnd()}\n\n${block(FENCED)}`); + const r = await run(['vendor', 'restore', 'CLAUDE.md', '--format', 'json'], JSON_OPTS); + expect(JSON.parse(r.out)).toMatchObject({ status: 'restored', keptSections: ['## Environment'] }); + expect(read('CLAUDE.md').endsWith(`${FENCED}\n`)).toBe(true); + }); +}); + +describe('marker matching agrees with stripCharterSentinels (review note 3)', () => { + it('composeCompiled sees a block exactly when the stripper strips it', () => { + const variants = [ + RETAINED_START, '', '', ` ${RETAINED_START} `, + ' x', '', '', + ]; + for (const v of variants) { + const file = `${FRESH}\n${v}\n## My Notes\n- kept\n${RETAINED_END}\n`; + const stripped = !stripCharterSentinels(file).includes('- kept'); + const c = composeCompiled(FRESH, file, 'CLAUDE.md'); + expect(c.retainedSource === 'block', v).toBe(stripped); + } + }); +}); + +describe('restore of a block holding non-retained content (review note 1)', () => { + const cases = [['a non-retained heading', `${ENV}\n\n## My Notes\n- x`, '## My Notes'], ['text before the first heading', `loose line\n\n${ENV}`, '(text before the first heading)']] as const; + for (const [name, inner, named] of cases) { + it(`refuses (would-drop) for ${name}, naming it, and writes nothing`, async () => { + scaffold(vendors('CLAUDE.md: pointer')); + const file = `${(await fresh('claude')).trimEnd()}\n\n${block(inner)}`; + fs.writeFileSync('CLAUDE.md', file); + const r = await run(['vendor', 'restore', 'CLAUDE.md', '--format', 'json'], JSON_OPTS); + expect(JSON.parse(r.out)).toMatchObject({ status: 'refused', refusal: 'would-drop', remedy: 'charter adf vendor set CLAUDE.md compiled' }); + expect(r.err).toContain(named); + expect(r.err).toContain('Keep the file as it is: charter adf vendor set CLAUDE.md compiled'); + expect(read('CLAUDE.md')).toBe(file); + }); + } +}); diff --git a/packages/cli/src/commands/adf-compile.ts b/packages/cli/src/commands/adf-compile.ts index 8b14556..2c4aab7 100644 --- a/packages/cli/src/commands/adf-compile.ts +++ b/packages/cli/src/commands/adf-compile.ts @@ -46,7 +46,7 @@ import { import { countExtractable, readRetainedSections } from './adf-tidy'; import { inferVendorMode, readVendorRecord, recordedSpelling, recordVendorModes, resolveVendorMode } from './adf-vendor'; import type { ModeSource, VendorFile, VendorMode } from './adf-vendor'; -import { composeCompiled, linesConvertWouldDrop } from './adf-retained'; +import { composeCompiled, linesConvertWouldDrop, readAiModuleLines, RETAINED_MSG } from './adf-retained'; import type { RetainedSource } from './adf-retained'; // ============================================================================ @@ -331,6 +331,8 @@ interface WriteTargetOutcome { /** Retained headings carried into the written file, in order (#303). */ retainedSections: string[]; retainedSource: RetainedSource; + /** Legacy retained-heading sections not carried because they look like rendered .ai/ text. */ + retainedSkipped: string[]; } function runWriteMode( @@ -355,7 +357,7 @@ function runWriteMode( const { state, source } = targetState(aiDir, filename); const row: WriteTargetOutcome = { target, file: filename, state, source, status: 'written', recordedMode: null, previousMode: null, remedy: null, writeError: null, - retainedSections: [], retainedSource: 'none', + retainedSections: [], retainedSource: 'none', retainedSkipped: [], }; rows.push(row); @@ -414,7 +416,7 @@ function runWriteMode( // under --force (W4). Converting a pointer refuses only over content the // output would NOT carry: bloat tidy can fold into .ai/, or other stray lines. const existing = fs.existsSync(filename) ? fs.readFileSync(filename, 'utf-8') : null; - const composed = composeCompiled(result.output, existing, filename); + const composed = composeCompiled(result.output, existing, filename, { aiLines: readAiModuleLines(aiDir) }); // Decided by the file's markers, not by whether anything is carried: a // pointer with no retained sections can still hold bloat or stray lines. if (existing !== null && inferVendorMode(filename as VendorFile, existing).inferredMode === 'pointer' && !flags.force) { @@ -485,6 +487,8 @@ function runWriteMode( if (state === 'pointer' || state === 'unmanaged') row.previousMode = state; row.retainedSections = composed.retainedSections; row.retainedSource = composed.retainedSource; + row.retainedSkipped = composed.skipped; + for (const heading of composed.skipped) warn(fill(RETAINED_MSG.skipped, { file: filename, heading })); for (const heading of composed.collisions) warn(fill(MSG.collision, { file: filename, heading })); if (!json) { const headings = composed.retainedSections.join(', '); @@ -619,7 +623,7 @@ function runCheckMode( // file's own retained block, so an edit inside the block is never stale (#303). const onDisk = fs.readFileSync(file, 'utf-8'); const result = compileAdf({ target, aiDir, displayAiDir, readFile }); - const composed = composeCompiled(result.output, onDisk, file); + const composed = composeCompiled(result.output, onDisk, file, { aiLines: readAiModuleLines(aiDir) }); if (onDisk === composed.output) { print(fill(MSG.checkFresh, { file })); } else { diff --git a/packages/cli/src/commands/adf-retained.ts b/packages/cli/src/commands/adf-retained.ts index 1592aff..252d3af 100644 --- a/packages/cli/src/commands/adf-retained.ts +++ b/packages/cli/src/commands/adf-retained.ts @@ -18,11 +18,16 @@ * use it, so they cannot disagree. score deliberately does not: it asks whether * .ai/ alone is substantive, so it keeps the pure compileAdf output. * + * Section parsing is code-fence aware: a `## ` line or a marker inside a ``` or + * ~~~ fence is text, and only /^ {0,3}## / is a heading (CommonMark). + * * Nothing here imports adf.ts: pointer templates and markers are reached at * call time through adf-compile and adf-vendor, so the #355 circular-import * class cannot recur here. */ +import * as fs from 'node:fs'; +import * as path from 'node:path'; import { readRetainedSections } from './adf-tidy'; import { inferVendorMode, linesRestoreWouldDrop } from './adf-vendor'; import type { VendorFile } from './adf-vendor'; @@ -33,6 +38,17 @@ export const RETAINED_END = ''; /** First line inside a new block. The reader drops it. */ export const RETAINED_NOTE = ''; +/** + * Marker matchers. They mirror stripCharterSentinels' SENTINEL_START/END + * (packages/adf/src/sentinels.ts) for the `retained` name exactly, so compile + * and tidy/doctor/migrate agree on what a marker is (parity pinned by test). + */ +const START_RE = /^$/; +const END_RE = /^$/; +export const isRetainedStart = (line: string): boolean => START_RE.test(line.trim()); +export const isRetainedEnd = (line: string): boolean => END_RE.test(line.trim()); +const isNote = (line: string): boolean => line.trim() === RETAINED_NOTE; + /** * A heading the renderer could have produced: all caps, with an optional weight * suffix. Never treated as user content in a compiled file without a block, so @@ -43,6 +59,10 @@ export const GENERATED_HEADING_RE = /^## [A-Z][A-Z0-9_]*( _\((load-bearing|advis /** The on-demand module index fence lines the compiler emits (html and hash forms). */ const MODULE_INDEX_FENCE_LINES = new Set(['', '', '# agents-modules', '# /agents-modules']); +export const RETAINED_MSG = { + skipped: '{file}: {heading} was not carried: it looks like compiler output from .ai/. If it is yours, move it into the retained block.', +} as const; + /** * Where retained content was read from: * block the existing file has a retained block @@ -53,6 +73,11 @@ const MODULE_INDEX_FENCE_LINES = new Set(['', '` … `` block at the end of the file, and every later `--write` copies that block as it stands. A compiled file with nothing to carry gets no block and compiles byte-identically to before. The block uses HTML markers in every target, `.cursorrules` included. All-caps headings the compiler could have produced (a module since removed from `.ai/`) and `## Module Index` are never carried. `adf compile` to stdout and `charter score` still use the plain render. - **`adf compile --check` reads such a file as stale until one `--write`** (`#303`) — the expected bytes are now the render plus the file's own retained block, so an edit inside the block is never stale. A compiled file whose retained sections sit outside the block reads `[stale]` with `staleReason: "retained-unfenced"`, and the remedy (`compile --target --write`) keeps them. Each `--check` JSON row adds `staleReason` (`content`, `retained-unfenced`, or null). -- **`refused-retained` is retired from `adf compile --write`, and `retained-divergent` from `adf vendor restore`** (`#303`) — JSON consumers no longer see either value. A pointer converted with `--mode compiled` now carries the retained sections that differ from its template into the block, instead of refusing. It still refuses, exit 1 with nothing written and `VENDORS` unchanged, over content the output would not carry: `refused-bloated` (items `adf tidy` would extract; the first remedy is `adf tidy --source `, then `--force`, which discards them) and `refused-would-drop` (other lines of yours). `--mode compiled --force` over a hand-authored file carries its retained sections and discards the rest. `--force` never drops retained sections. Write JSON rows add `retainedSections`, `retainedSource` and `retainedSkipped`. That last one lists retained-looking sections that are really rendered `.ai/` text left behind by an edit or removal in `.ai/`; they are not carried, and each one is named on stderr. Parsing is code-fence aware, and marker lines match as loosely as `stripCharterSentinels`. `vendor restore` of a compiled file carries its retained sections into the pointer and lists them in `keptSections`, and the block is not counted as a `compiled-differs` difference. A block holding a non-retained heading, or text before its first heading, is refused as `would-drop` (new value) with `vendor set compiled` as the first remedy, because a pointer cannot keep it. +- **`refused-retained` is retired from `adf compile --write`, and `retained-divergent` from `adf vendor restore`** (`#303`) — JSON consumers no longer see either value. A pointer converted with `--mode compiled` now carries the retained sections that differ from its template into the block, instead of refusing. It still refuses, exit 1 with nothing written and `VENDORS` unchanged, over content the output would not carry: `refused-bloated` (items `adf tidy` would extract; the first remedy is `adf tidy --source `, then `--force`, which discards them) and `refused-would-drop` (other lines of yours). `--mode compiled --force` over a hand-authored file carries its retained sections and discards the rest. `--force` never drops retained sections. Write JSON rows add `retainedSections`, `retainedSource`, `retainedSkipped` and `retainedAmbiguous`. `retainedSkipped` lists retained-looking sections whose heading is still in a `.ai/` module, so they are rendered `.ai/` text left behind by an edit; they are not carried, and each one is named on stderr. A section left inside a generated section that `.ai/` no longer renders, with no `.ai/` match, may be stale output or your own. `--write` refuses that target as `refused-ambiguous` (new value), with "wrap it in the retained block" as the first remedy and `--force` (which discards it) as the second. `--check` reports it with `staleReason: "retained-ambiguous"`. Parsing is code-fence aware, and marker lines match as loosely as `stripCharterSentinels`. `vendor restore` of a compiled file carries its retained sections into the pointer and lists them in `keptSections`, and the block is not counted as a `compiled-differs` difference. A block holding a non-retained heading, or text before its first heading, is refused as `would-drop` (new value) with `vendor set compiled` as the first remedy, because a pointer cannot keep it. - **The pre-commit hook tidies only the staged vendor files and re-stages only the files tidy wrote** (`#320`, `#318` part 4) — re-run `charter hook install --pre-commit` to pick up staged-only tidy. An installed hook keeps its old behaviour until you do, and `charter doctor` reports it with an INFO `pre-commit hook` row (INFO, so `--ci` does not fail). Reinstalling overwrites a Charter-managed hook without `--force`, as before. The old hook ran a whole-tree `adf tidy` whenever any vendor file was staged, then ran `git add` on every staged vendor file and on all of `.ai/*.adf`. That swept unrelated `.ai/` edits into the commit and rewrote vendor files that were not part of it. The new hook passes each staged vendor file to `adf tidy` with `--source` and stages only the paths that both the dry run and the real run report. A staged vendor file that also has unstaged changes is skipped with a message, because tidy works on the working tree and staging its result would commit those hunks. If tidy would write into a module that has unstaged changes or is untracked, nothing is tidied for that commit and a message says why. A gitignored module (for example an `.ai/` kept out of git) is not treated as work in progress: tidy writes it, and the hook says it was left unstaged. Any file the hook cannot stage, and any file the real run wrote that the dry run did not plan, is named and left unstaged rather than aborting the hook. A staged lowercase `agents.md` is now tidied (`#302`), and a commit staging only compiled output still makes no CLI call. The tidy block itself never fails a commit, but the ADF evidence gate after it can, and it is unchanged. In the generated hook (no `verify:adf` script), vendor bloat that a skipped tidy leaves in the working tree still fails `charter doctor --adf-only --ci`. The skip message now says so and names the ways out: stage or stash the unstaged work; run `charter adf tidy --source ` and `git add` the result; or `git commit --no-verify`. When the gate then fails, one line says which skipped file still carries how many items. A new hook with an older local CLI that does not know `--print-paths` reports the files as clean and tidies nothing, so upgrade the CLI too. - **BREAKING: `charter adf compile --write` no longer converts pointer files without `--mode compiled`** (`#318`, part 2) — compile used to overwrite any Charter thin pointer with compiled output, silently changing its mode. Each target's mode now comes from the `VENDORS` record, or from its markers when there is no entry, and `--write` only writes targets that are compiled or absent. Under `--target all`, pointer and unmanaged targets are skipped (exit 0), and each `[skip]` line names the command that converts it. A single such `--target` is refused (exit 1). `--force` alone never changes a mode. `--mode compiled` records `compiled` before writing. If that cannot be recorded, nothing is written (`refused-record`). If the write then fails, the record is restored and the run continues. A converted file is reported as `Written CLAUDE.md (mode: pointer -> compiled)`. JSON output adds `skipped` and per-target `targets`. A compiled-only repo with no `VENDORS` section compiles as before. diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 4406af4..73b60df 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -542,7 +542,7 @@ Default with no `--write` or `--check` prints to stdout, single target only. | pointer | skipped (its `[skip]` line names the command), exit 0 | refused, exit 1 | record set to compiled, then written as `Written CLAUDE.md (mode: pointer -> compiled)` | | unmanaged | skipped, exit 0 | refused, exit 1 | as pointer, but an existing hand-authored file also needs `--force` | -When every target is skipped, compile says that nothing was written and how to switch. JSON output keeps `written`, `refused` and `nextActions`, and adds `skipped` and `targets` (one row per target: `state`, `source`, `status`, `recordedMode`, `previousMode`, `remedy`, `writeError`, `retainedSections`, `retainedSource`, `retainedSkipped`). `previousMode` (`pointer` or `unmanaged`) is how a JSON consumer tells a conversion from a re-compile. +When every target is skipped, compile says that nothing was written and how to switch. JSON output keeps `written`, `refused` and `nextActions`, and adds `skipped` and `targets` (one row per target: `state`, `source`, `status`, `recordedMode`, `previousMode`, `remedy`, `writeError`, `retainedSections`, `retainedSource`, `retainedSkipped`, `retainedAmbiguous`). `previousMode` (`pointer` or `unmanaged`) is how a JSON consumer tells a conversion from a re-compile. **Retained sections.** Compiled output keeps your own `## Environment` section and the other sections `adf tidy` retains (`## Session ...` other than `## Session Start`, and headings containing Protocol, Receipt, Provenance or Registration). They live in a block at the very end of the file, after the module index: @@ -564,7 +564,11 @@ When every target is skipped, compile says that nothing was written and how to s A file with nothing to carry gets no block, so its output is byte-identical to the plain render. The block uses the HTML markers in every target, `.cursorrules` included. Sections keep their source order, and the output is LF. A carried section whose heading matches a compiled one (say `## CONTEXT`) is kept, with a warning on stderr. The written line names what was kept: `Written CLAUDE.md (kept ## Environment)`. `adf compile` without `--write` prints the plain render, with no block. `adf tidy`, `doctor` and `adf migrate` ignore the block, because it is a Charter sentinel. -A retained-looking section that is really rendered `.ai/` text is not carried. This covers a `## Session notes` or `## Environment` line inside a `.ai/` text value, left in the compiled file after you edited or removed that value. Two signs mark such a section. It sits inside a generated section's span (from a `## KEY` heading to the next generated heading or the module index) that no longer matches the current render. Or its heading appears in a current `.ai/` module. Compile prints `: was not carried: it looks like compiler output from .ai/. If it is yours, move it into the retained block.` and lists the heading in the row's `retainedSkipped`. A section appended after the last rendered section is still carried when that section matches the current render. +A retained-looking section can really be rendered `.ai/` text: a `## Session notes` or `## Environment` line inside a `.ai/` text value, left in the compiled file after you edited or removed that value. Compile handles such sections as follows: + +- If its heading appears in a current `.ai/` module, it is compiler output. It is not carried. Compile prints `: was not carried: it looks like compiler output from .ai/. If it is yours, move it into the retained block.` and lists it in the row's `retainedSkipped`. +- If it sits inside a generated section's span (from a `## KEY` heading to the next generated heading or the module index) that no longer matches the current render, and no `.ai/` module holds its heading, it could be stale output or your own text. `--write` will not drop it. That target is refused as `refused-ambiguous` (exit 1, nothing written, `VENDORS` unchanged), named in `retainedAmbiguous`. The first remedy keeps it: wrap it in place between `` and `` (the two lines are printed), then re-run `--write`. The second, `--write --force`, discards it and lists it in `retainedSkipped`. `--check` reads such a file as `stale` with `staleReason: "retained-ambiguous"` and gives the same two remedies. +- A section appended after the last rendered section is still carried when that section matches the current render. Parsing follows Markdown. A `## ` line, a module index fence or a retained marker inside a ```` ``` ```` or `~~~` code fence is text, and an indented ` ## x` is code, not a heading. Marker lines match with the same loose whitespace that `stripCharterSentinels` accepts (`` works). A marker line that the render itself contains, quoted in a `.ai/` text value, never starts the block. @@ -584,7 +588,7 @@ Limitations: | Status | When | Exit | |---|---|---| -| `fresh` / `stale` | compiled: the file equals / differs from what `--write` would write, which is the render plus the file's own retained block (remedy: `compile --target --write`). `staleReason` is `retained-unfenced` when retained sections sit outside the block (the remedy keeps them), else `content` | 0 / 1 | +| `fresh` / `stale` | compiled: the file equals / differs from what `--write` would write, which is the render plus the file's own retained block (remedy: `compile --target --write`). `staleReason` is `retained-ambiguous` when a legacy section may be stale `.ai/` output or yours (see retained sections), `retained-unfenced` when retained sections sit outside the block (the remedy keeps them), else `content` | 0 / 1 | | `missing` | no file on disk (a recorded pointer suggests `adf vendor restore ` or `adf vendor set unmanaged`) | 1 | | `pointer-ok` | pointer with nothing `adf tidy` would extract | 0 | | `pointer-bloated` | pointer carrying content that belongs in `.ai/` (remedy: `adf tidy --source `) | 1 | diff --git a/packages/cli/src/__tests__/adf-retained.test.ts b/packages/cli/src/__tests__/adf-retained.test.ts index cdcadd7..e9f1924 100644 --- a/packages/cli/src/__tests__/adf-retained.test.ts +++ b/packages/cli/src/__tests__/adf-retained.test.ts @@ -88,7 +88,7 @@ async function fresh(target: string): Promise { return (await run(['compile', '--target', target])).out; } -interface WriteRow { file: string; status: string; previousMode: string | null; recordedMode: string | null; remedy: string | null; retainedSections: string[]; retainedSource: string; retainedSkipped: string[] } +interface WriteRow { file: string; status: string; previousMode: string | null; recordedMode: string | null; remedy: string | null; retainedSections: string[]; retainedSource: string; retainedSkipped: string[]; retainedAmbiguous: string[] } interface WriteJson { written: string[]; refused: string[]; targets: WriteRow[]; nextActions: string[] } interface CheckRow { file: string; status: string; staleReason: string | null; remedy: string | null } interface CheckJson { upToDate: boolean; targets: CheckRow[]; nextActions: string[] } @@ -507,23 +507,21 @@ const TARGET_OF_FILE: Record = { 'CLAUDE.md': 'claude', 'AGENTS. type SkipJson = WriteJson; describe('rendered .ai/ text is never carried as the user\'s (review item 1)', () => { - const cases: Array<[string, string, string | null]> = [ + const edited: Array<[string, string, string]> = [ ['edited', 'Intro line\n## Session notes\n- keep tabs', 'Intro line\n## Session notes\n- keep spaces'], ['edited (## Environment)', '## Environment\nUse WSL.', '## Environment\nUse Docker.'], - ['removed', 'Intro line\n## Session notes\n- keep tabs', null], ]; - for (const [name, before, after] of cases) { - it(`does not carry the old rendered copy when the .ai/ value is ${name}, and reports it`, async () => { + for (const [name, before, after] of edited) { + it(`does not carry the old rendered copy when the .ai/ value is ${name} (rule b: skip + warn)`, async () => { scaffold(); fs.writeFileSync('.ai/core.adf', NOTES_CORE(before)); await run(['compile', '--target', 'all', '--write']); - fs.writeFileSync('.ai/core.adf', after === null ? 'ADF: 0.1\n\n\u{1F4CB} CONTEXT:\n - TypeScript monorepo\n' : NOTES_CORE(after)); + fs.writeFileSync('.ai/core.adf', NOTES_CORE(after)); const heading = before.split('\n').find(l => l.startsWith('## ')) as string; const r = await runJson(['compile', '--target', 'all', '--write']); expect(r.code).toBe(0); for (const t of r.json.targets) { - expect(t.retainedSkipped, t.file).toEqual([heading]); - expect(t.retainedSource, t.file).toBe('none'); + expect(t, t.file).toMatchObject({ status: 'written', retainedSkipped: [heading], retainedAmbiguous: [], retainedSource: 'none' }); } expect(r.err).toContain(`CLAUDE.md: ${heading} was not carried: it looks like compiler output from .ai/. If it is yours, move it into the retained block.`); for (const f of TARGET_FILES) { @@ -535,6 +533,71 @@ describe('rendered .ai/ text is never carried as the user\'s (review item 1)', ( }); } + describe('a removed .ai/ value with no .ai/ match is ambiguous: refused, never dropped silently', () => { + async function setupRemoved(): Promise> { + scaffold(); + fs.writeFileSync('.ai/core.adf', NOTES_CORE('Intro line\n## Session notes\n- keep tabs')); + await run(['compile', '--target', 'all', '--write']); + fs.writeFileSync('.ai/core.adf', 'ADF: 0.1\n\n\u{1F4CB} CONTEXT:\n - TypeScript monorepo\n'); + return Object.fromEntries(TARGET_FILES.map(f => [f, read(f)])); + } + + it('--write refuses every target (exit 1, nothing written), keep remedy first', async () => { + const before = await setupRemoved(); + const manifest = read('.ai/manifest.adf'); + const text = await run(['compile', '--target', 'claude', '--write']); + expect(text.code).toBe(1); + expect(text.err).toContain('Refused to compile CLAUDE.md: ## Session notes sit(s) inside a section .ai/ no longer renders that way'); + expect(text.err).toContain(`Keep it: wrap it in place between these two lines, then re-run charter adf compile --target claude --write\n ${RETAINED_START}\n ${RETAINED_END}\n Or discard it: charter adf compile --target claude --write --force (discards: ## Session notes)`); + const r = await runJson(['compile', '--target', 'all', '--write']); + expect(r.code).toBe(1); + for (const t of r.json.targets) expect(t, t.file).toMatchObject({ status: 'refused-ambiguous', retainedAmbiguous: ['## Session notes'] }); + expect(r.json.nextActions[0]).toBe(`charter adf compile --target claude --write # after wrapping ## Session notes in CLAUDE.md between ${RETAINED_START} and ${RETAINED_END}`); + expect(r.json.nextActions[1]).toBe('charter adf compile --target claude --write --force'); + for (const f of TARGET_FILES) expect(read(f), f).toBe(before[f]); + expect(read('.ai/manifest.adf')).toBe(manifest); + }); + + it('--check reads it stale (retained-ambiguous) with the same remedies, not collapsed', async () => { + await setupRemoved(); + const text = await run(['compile', '--target', 'claude', '--check']); + expect(text.code).toBe(1); + expect(text.out).toContain('[stale] CLAUDE.md — ## Session notes may be stale compiler output from .ai/ or your own text'); + expect(text.out).toContain('(discards: ## Session notes)'); + const r = await runJson(['compile', '--target', 'all', '--check']); + expect(r.json.targets.every(t => t.status === 'stale' && t.staleReason === 'retained-ambiguous')).toBe(true); + expect(r.json.nextActions.slice(0, 2)).toEqual([ + `charter adf compile --target claude --write # after wrapping ## Session notes in CLAUDE.md between ${RETAINED_START} and ${RETAINED_END}`, + 'charter adf compile --target claude --write --force', + ]); + }); + + it('wrapping it in place keeps it, stably', async () => { + await setupRemoved(); + for (const f of TARGET_FILES) { + fs.writeFileSync(f, read(f).replace('## Session notes\n- keep tabs', `${RETAINED_START}\n## Session notes\n- keep tabs\n${RETAINED_END}`)); + } + const r = await runJson(['compile', '--target', 'all', '--write']); + expect(r.code).toBe(0); + for (const f of TARGET_FILES) { + const kept = read(f); + expect(kept.endsWith(`${RETAINED_START}\n## Session notes\n- keep tabs\n${RETAINED_END}\n`), f).toBe(true); + await run(['compile', '--target', TARGET_OF_FILE[f], '--write']); + expect(read(f), f).toBe(kept); + } + expect((await run(['compile', '--target', 'all', '--check'])).code).toBe(0); + }); + + it('--force discards it and lists it in retainedSkipped', async () => { + await setupRemoved(); + const r = await runJson(['compile', '--target', 'all', '--write', '--force']); + expect(r.code).toBe(0); + for (const t of r.json.targets) expect(t, t.file).toMatchObject({ status: 'written', retainedSkipped: ['## Session notes'], retainedAmbiguous: [] }); + expect(r.err).toContain('CLAUDE.md: ## Session notes was discarded (--force)'); + for (const f of TARGET_FILES) expect(read(f), f).toBe(await fresh(TARGET_OF_FILE[f])); + }); + }); + it('does not carry a legacy section whose heading is in a current .ai/ module (rule b)', () => { const c = composeCompiled(FRESH, `${FRESH}\n## Session notes\n- old\n`, 'CLAUDE.md', { aiLines: new Set(['## Session notes']) }); expect(c).toMatchObject({ output: FRESH, skipped: ['## Session notes'], retainedSource: 'none' }); diff --git a/packages/cli/src/commands/adf-compile.ts b/packages/cli/src/commands/adf-compile.ts index 2c4aab7..578b155 100644 --- a/packages/cli/src/commands/adf-compile.ts +++ b/packages/cli/src/commands/adf-compile.ts @@ -46,7 +46,7 @@ import { import { countExtractable, readRetainedSections } from './adf-tidy'; import { inferVendorMode, readVendorRecord, recordedSpelling, recordVendorModes, resolveVendorMode } from './adf-vendor'; import type { ModeSource, VendorFile, VendorMode } from './adf-vendor'; -import { composeCompiled, linesConvertWouldDrop, readAiModuleLines, RETAINED_MSG } from './adf-retained'; +import { composeCompiled, linesConvertWouldDrop, readAiModuleLines, RETAINED_END, RETAINED_MSG, RETAINED_START } from './adf-retained'; import type { RetainedSource } from './adf-retained'; // ============================================================================ @@ -75,6 +75,8 @@ export const MSG = { refusedBloated: ' [warn] Refused to compile {file}: it carries {n} item(s) beyond the pointer that .ai/ does not hold yet.\n Move them into .ai/ first: charter adf tidy --source {file}\n Or discard them: charter adf compile --target {target} --write --mode compiled --force (discards {n} item(s))', refusedWouldDrop: " [warn] Refused to compile {file}: {n} line(s) of yours would be lost ({lines}).\n Move them under ## Environment or into a .ai/ module, or discard them: charter adf compile --target {target} --write --mode compiled --force (discards {n} line(s))", collision: '{file}: your retained section {heading} has the same heading as a compiled section; both are kept.', + refusedAmbiguous: ' [warn] Refused to compile {file}: {headings} sit(s) inside a section .ai/ no longer renders that way, and no .ai/ module holds that heading.\n It may be stale compiler output or your own text, so --write will not drop it.\n Keep it: wrap it in place between these two lines, then re-run charter adf compile --target {target} --write\n {start}\n {end}\n Or discard it: charter adf compile --target {target} --write --force (discards: {headings})', + checkStaleAmbiguous: ' [stale] {file} — {headings} may be stale compiler output from .ai/ or your own text; --write refuses until you decide.\n Keep it: wrap it in place between these two lines, then run charter adf compile --target {target} --write\n {start}\n {end}\n Or discard it: charter adf compile --target {target} --write --force (discards: {headings})', checkStaleUnfenced: ' [stale] {file} — your {headings} section(s) sit outside the retained block.\n Run: charter adf compile --target {target} --write (keeps them)', nothingCompiled: ' No target is in compiled mode, so nothing was written. Pointer files stay pointers.\n To switch them: charter adf compile --target all --write --mode compiled. To see modes: charter adf vendor status', checkFresh: ' [ok] {file} is up-to-date', @@ -115,6 +117,11 @@ function targetState(aiDir: string, filename: string): TargetState { return { state: 'absent-unrecorded', source: null, exists: false, disagrees: false, inferredMode: null }; } +/** The content-preserving remedy for an ambiguous legacy section: wrap it, then re-run (a comment names the manual step). */ +function ambiguousKeepRemedy(target: CompileTarget, file: string, headings: string): string { + return `charter adf compile --target ${target} --write # after wrapping ${headings} in ${file} between ${RETAINED_START} and ${RETAINED_END}`; +} + function dedupe(items: string[]): string[] { return [...new Set(items)]; } @@ -314,6 +321,7 @@ type WriteStatus = | 'refused-hand-authored' | 'refused-bloated' | 'refused-would-drop' + | 'refused-ambiguous' | 'refused-record'; // 'refused-retained' is retired (#303): retained sections are carried, not refused over. @@ -333,6 +341,8 @@ interface WriteTargetOutcome { retainedSource: RetainedSource; /** Legacy retained-heading sections not carried because they look like rendered .ai/ text. */ retainedSkipped: string[]; + /** Legacy sections that may be the user's or stale .ai/ output: the reason for refused-ambiguous. */ + retainedAmbiguous: string[]; } function runWriteMode( @@ -357,7 +367,7 @@ function runWriteMode( const { state, source } = targetState(aiDir, filename); const row: WriteTargetOutcome = { target, file: filename, state, source, status: 'written', recordedMode: null, previousMode: null, remedy: null, writeError: null, - retainedSections: [], retainedSource: 'none', retainedSkipped: [], + retainedSections: [], retainedSource: 'none', retainedSkipped: [], retainedAmbiguous: [], }; rows.push(row); @@ -417,6 +427,17 @@ function runWriteMode( // output would NOT carry: bloat tidy can fold into .ai/, or other stray lines. const existing = fs.existsSync(filename) ? fs.readFileSync(filename, 'utf-8') : null; const composed = composeCompiled(result.output, existing, filename, { aiLines: readAiModuleLines(aiDir) }); + // A legacy section that is either stale .ai/ output or the user's own is + // never dropped without --force: the user decides (wrap it, or --force). + if (composed.ambiguous.length > 0 && !flags.force) { + const headings = composed.ambiguous.join(', '); + row.status = 'refused-ambiguous'; + row.retainedAmbiguous = composed.ambiguous; + row.remedy = ambiguousKeepRemedy(target, filename, headings); + extraActions.set(filename, `charter adf compile --target ${target} --write${modeSuffix} --force`); + if (!json) console.error(fill(MSG.refusedAmbiguous, { file: filename, headings, target, start: RETAINED_START, end: RETAINED_END })); + continue; + } // Decided by the file's markers, not by whether anything is carried: a // pointer with no retained sections can still hold bloat or stray lines. if (existing !== null && inferVendorMode(filename as VendorFile, existing).inferredMode === 'pointer' && !flags.force) { @@ -487,8 +508,9 @@ function runWriteMode( if (state === 'pointer' || state === 'unmanaged') row.previousMode = state; row.retainedSections = composed.retainedSections; row.retainedSource = composed.retainedSource; - row.retainedSkipped = composed.skipped; + row.retainedSkipped = [...composed.skipped, ...composed.ambiguous]; for (const heading of composed.skipped) warn(fill(RETAINED_MSG.skipped, { file: filename, heading })); + for (const heading of composed.ambiguous) warn(fill(RETAINED_MSG.discarded, { file: filename, heading })); for (const heading of composed.collisions) warn(fill(MSG.collision, { file: filename, heading })); if (!json) { const headings = composed.retainedSections.join(', '); @@ -554,7 +576,7 @@ interface CheckTargetResult { extractable: number | null; remedy: string | null; /** Set iff status is stale: 'retained-unfenced' when retained sections sit outside the retained block, else 'content'. */ - staleReason: 'content' | 'retained-unfenced' | null; + staleReason: 'content' | 'retained-unfenced' | 'retained-ambiguous' | null; } /** @@ -563,7 +585,7 @@ interface CheckTargetResult { */ function collapseCompileRemedies(rows: CheckTargetResult[], nextActions: string[]): string[] { const failing = rows.filter(r => FAILING_CHECK_STATUSES.includes(r.status)); - const plain = failing.every(r => (r.status === 'stale' || r.status === 'missing') && r.mode !== 'pointer'); + const plain = failing.every(r => (r.status === 'stale' || r.status === 'missing') && r.mode !== 'pointer' && r.staleReason !== 'retained-ambiguous'); return rows.length > 1 && failing.length > 0 && plain ? ['charter adf compile --target all --write'] : nextActions; } @@ -629,10 +651,18 @@ function runCheckMode( } else { row.status = 'stale'; row.remedy = compileRemedy; - row.staleReason = composed.unfenced.length > 0 ? 'retained-unfenced' : 'content'; - print(row.staleReason === 'retained-unfenced' - ? fill(MSG.checkStaleUnfenced, { file, headings: composed.unfenced.join(', '), target }) - : fill(MSG.checkStale, { file, aiDir: displayAiDir, target })); + if (composed.ambiguous.length > 0) { + const headings = composed.ambiguous.join(', '); + row.staleReason = 'retained-ambiguous'; + row.remedy = ambiguousKeepRemedy(target, file, headings); + nextActions.push(row.remedy, `${compileRemedy} --force`); + print(fill(MSG.checkStaleAmbiguous, { file, headings, target, start: RETAINED_START, end: RETAINED_END })); + } else { + row.staleReason = composed.unfenced.length > 0 ? 'retained-unfenced' : 'content'; + print(row.staleReason === 'retained-unfenced' + ? fill(MSG.checkStaleUnfenced, { file, headings: composed.unfenced.join(', '), target }) + : fill(MSG.checkStale, { file, aiDir: displayAiDir, target })); + } } } else if (ts.state === 'pointer') { // A pointer is checked for being intact, never against compiled output. diff --git a/packages/cli/src/commands/adf-retained.ts b/packages/cli/src/commands/adf-retained.ts index 252d3af..b7b1e45 100644 --- a/packages/cli/src/commands/adf-retained.ts +++ b/packages/cli/src/commands/adf-retained.ts @@ -61,6 +61,7 @@ const MODULE_INDEX_FENCE_LINES = new Set(['', '