From 830ea2c62f48e2591429016d85ab91f9d9abc8d5 Mon Sep 17 00:00:00 2001 From: Joshua Wink Date: Sat, 10 Oct 2026 15:04:59 -0500 Subject: [PATCH] feat(typescript): a compiler that writes a pipeline as plain TypeScript packages/codeupipe-compile-ts reads the source of a pipeline class and writes one module with no import of codeupipe: a function for each step and an async run(input). The data is a Map, each operation of the Payload is an operation of the Map, and each setting of an instance is a constant. A pipeline for which the compiled code cannot give the same output is refused, with the place named and the fix: Hooks, a constructor that does more than add steps, settings that are not literals, a step that keeps state, a Payload that is copied or given to other code, a Tap or a Predicate that writes. Nothing is written for a refused pipeline. The conformance suite has the target typescript:static. The runner writes each case as a pipeline class, compiles it, imports the file and runs it. 11 cases pass, and the 16 cases that also check State or Hooks pass on what plain code can show, as for the Python fast export. The compiler is a codeupipe pipeline. The Taps of the TypeScript runner now record to a module, so that compiled code can do the same. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 29 ++ README.md | 10 +- conformance/API.md | 2 +- conformance/MATRIX.md | 66 +++-- conformance/README.md | 1 + conformance/run.py | 17 +- .../runners/typescript/cup-ops/record-tap.ts | 7 +- .../runners/typescript/cup-ops/sink.ts | 2 + conformance/runners/typescript/runner.ts | 127 +++++++- packages/codeupipe-compile-ts/.gitignore | 2 + packages/codeupipe-compile-ts/LICENSE | 177 +++++++++++ packages/codeupipe-compile-ts/NOTICE | 4 + packages/codeupipe-compile-ts/README.md | 121 ++++++++ .../codeupipe-compile-ts/package-lock.json | 73 +++++ packages/codeupipe-compile-ts/package.json | 38 +++ packages/codeupipe-compile-ts/src/cli.ts | 45 +++ .../src/compile-pipeline.ts | 15 + .../src/compile-refusal.ts | 18 ++ packages/codeupipe-compile-ts/src/compile.ts | 28 ++ .../src/emit-module-filter.ts | 123 ++++++++ packages/codeupipe-compile-ts/src/index.ts | 4 + packages/codeupipe-compile-ts/src/literal.ts | 32 ++ .../codeupipe-compile-ts/src/lower-class.ts | 277 ++++++++++++++++++ .../src/lower-steps-filter.ts | 36 +++ .../codeupipe-compile-ts/src/real-path.ts | 16 + .../src/resolve-pipeline-filter.ts | 93 ++++++ .../codeupipe-compile-ts/src/source-files.ts | 132 +++++++++ .../codeupipe-compile-ts/src/step-plan.ts | 32 ++ .../codeupipe-compile-ts/tests/cli.test.ts | 42 +++ .../codeupipe-compile-ts/tests/compile-in.ts | 26 ++ .../tests/compile.test.ts | 139 +++++++++ .../tests/refusals.test.ts | 92 ++++++ .../codeupipe-compile-ts/tsconfig.build.json | 10 + packages/codeupipe-compile-ts/tsconfig.json | 19 ++ packages/codeupipe-lint-ts/package.json | 2 +- packages/codeupipe-lint-ts/tests/cli.test.ts | 5 +- packages/codeupipe-ts/README.md | 8 +- packages/codeupipe-ts/tsconfig.json | 2 +- tests/conformance/test_typescript_target.py | 13 + 39 files changed, 1820 insertions(+), 65 deletions(-) create mode 100644 conformance/runners/typescript/cup-ops/sink.ts create mode 100644 packages/codeupipe-compile-ts/.gitignore create mode 100644 packages/codeupipe-compile-ts/LICENSE create mode 100644 packages/codeupipe-compile-ts/NOTICE create mode 100644 packages/codeupipe-compile-ts/README.md create mode 100644 packages/codeupipe-compile-ts/package-lock.json create mode 100644 packages/codeupipe-compile-ts/package.json create mode 100644 packages/codeupipe-compile-ts/src/cli.ts create mode 100644 packages/codeupipe-compile-ts/src/compile-pipeline.ts create mode 100644 packages/codeupipe-compile-ts/src/compile-refusal.ts create mode 100644 packages/codeupipe-compile-ts/src/compile.ts create mode 100644 packages/codeupipe-compile-ts/src/emit-module-filter.ts create mode 100644 packages/codeupipe-compile-ts/src/index.ts create mode 100644 packages/codeupipe-compile-ts/src/literal.ts create mode 100644 packages/codeupipe-compile-ts/src/lower-class.ts create mode 100644 packages/codeupipe-compile-ts/src/lower-steps-filter.ts create mode 100644 packages/codeupipe-compile-ts/src/real-path.ts create mode 100644 packages/codeupipe-compile-ts/src/resolve-pipeline-filter.ts create mode 100644 packages/codeupipe-compile-ts/src/source-files.ts create mode 100644 packages/codeupipe-compile-ts/src/step-plan.ts create mode 100644 packages/codeupipe-compile-ts/tests/cli.test.ts create mode 100644 packages/codeupipe-compile-ts/tests/compile-in.ts create mode 100644 packages/codeupipe-compile-ts/tests/compile.test.ts create mode 100644 packages/codeupipe-compile-ts/tests/refusals.test.ts create mode 100644 packages/codeupipe-compile-ts/tsconfig.build.json create mode 100644 packages/codeupipe-compile-ts/tsconfig.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 540d642..a722404 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,6 +93,35 @@ jobs: working-directory: packages/codeupipe-lint-ts run: npm run lint + typescript-compile: + name: TypeScript compiler + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + - uses: actions/setup-node@v5 + with: + node-version: "24" + - name: Install the core package + working-directory: packages/codeupipe-ts + run: npm ci + - name: Install + working-directory: packages/codeupipe-compile-ts + run: npm ci + - name: Type check + working-directory: packages/codeupipe-compile-ts + run: npm run typecheck + - name: Tests + working-directory: packages/codeupipe-compile-ts + run: npm test + - name: Build + working-directory: packages/codeupipe-compile-ts + run: npm run build + - name: Conformance + run: python conformance/run.py typescript:static + export: name: Export against dynamic run runs-on: ubuntu-latest diff --git a/README.md b/README.md index b5c83e7..bc89b96 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ If you know Unix, these are close to things you know. | Hook | A git hook: code that runs before and after a step | | State | `$?` and `PIPESTATUS` | -This repository has four packages. +This repository has five packages. | Package | Contents | Reference | |---|---|---| @@ -37,6 +37,7 @@ This repository has four packages. | `codeupipe-lint` | Two linters for pipeline code. Depends on `codeupipe` only | [packages/codeupipe-lint/README.md](packages/codeupipe-lint/README.md) | | `codeupipe` for TypeScript | The eight primitives. No dependencies | [packages/codeupipe-ts/README.md](packages/codeupipe-ts/README.md) | | `codeupipe-lint` for TypeScript | A linter for pipeline code in TypeScript | [packages/codeupipe-lint-ts/README.md](packages/codeupipe-lint-ts/README.md) | +| `codeupipe-compile` for TypeScript | Writes a pipeline as plain TypeScript with no import of codeupipe | [packages/codeupipe-compile-ts/README.md](packages/codeupipe-compile-ts/README.md) | ## Install @@ -153,7 +154,7 @@ An editable install needs pip 21.3 or later. | Command | What it does | |---|---| | `python -m pytest tests -q` | Runs the tests | -| `python conformance/run.py` | Runs the 27 conformance cases against the Python dynamic run and the three exports, and against the TypeScript run when the machine has Node 22.18 or later | +| `python conformance/run.py` | Runs the 27 conformance cases against the Python dynamic run and the three exports, and against the TypeScript targets that the machine can run | | `python -m codeupipe_lint.anonymous_steps` | Runs the anonymous-step linter. Paths come from `cup.config.json` | | `python -m codeupipe_lint.invariants` | Runs the invariant linter. Paths come from `cup.config.json` | | `python -m codeupipe.lower examples.pricing.pricing_pipeline:PricingPipeline --out build/pricing` | Exports the example to `build/pricing/src/steps/` | @@ -168,6 +169,7 @@ An editable install needs pip 21.3 or later. | `packages/codeupipe-lint/` | The `codeupipe-lint` package: `anonymous_steps/`, `invariants/` | | `packages/codeupipe-ts/` | The TypeScript runtime, with its tests and its example | | `packages/codeupipe-lint-ts/` | The TypeScript linter, with its tests | +| `packages/codeupipe-compile-ts/` | The TypeScript compiler, with its tests | | `conformance/` | The contract ([API.md](conformance/API.md)), the cases, the runners, the benchmark | | `tests/` | The tests for the two Python packages | | `examples/pricing/` | The example, with its test in `tests/examples/` | @@ -180,8 +182,8 @@ The root `pyproject.toml` is not a package. It holds the test settings. The contract and the conformance cases do not depend on a language. Python is the first runtime and TypeScript is the second -([packages/codeupipe-ts/](packages/codeupipe-ts/)), with its own linter. The -export is for Python only. A runtime in another language must implement +([packages/codeupipe-ts/](packages/codeupipe-ts/)), with its own linter and +its own compiler. A runtime in another language must implement the contract and add a conformance runner. ## License diff --git a/conformance/API.md b/conformance/API.md index 1059875..7d8778a 100644 --- a/conformance/API.md +++ b/conformance/API.md @@ -213,7 +213,7 @@ map and fails the comparison. | Run | `run` is sync. `run_async` is opt-in | One `run`. It returns a Promise. A step can be sync or async | | Absent | `None` | `null` | | Label of a step | `type(step).__name__` | `step.constructor.name` | -| Export | `codeupipe.lower` | none | +| Export | `codeupipe.lower` | `packages/codeupipe-compile-ts` | | Linter | `packages/codeupipe-lint` | `packages/codeupipe-lint-ts` | Notes on TypeScript: diff --git a/conformance/MATRIX.md b/conformance/MATRIX.md index 7771eba..8db7cd3 100644 --- a/conformance/MATRIX.md +++ b/conformance/MATRIX.md @@ -6,42 +6,43 @@ if any target has a ❌. **Every target passes every case, and nothing is unsupported.** The targets are the Python dynamic run, async run and three static exports, and the -TypeScript run. +TypeScript run and its compiled form. -The two fast exports (`static-steps`, `static-inline`) expose no State or -Hooks, so they pass those cases as ✅\*. `static-observe` (export with +The fast exports (`python:static-steps`, `python:static-inline`, +`typescript:static`) expose no State or Hooks, so they pass those cases as +✅\*. `static-observe` (export with `observe=True`) records both and passes every case outright. See [docs/static-export.md](../docs/static-export.md). -| case | python:run | python:run_async | python:static-steps | python:static-inline | python:static-observe | typescript:run | -|---|---|---|---|---|---|---| -| `hook/lifecycle-on-success` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `hook/on-error-when-a-filter-fails` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `hook/not-notified-of-a-tap-failure` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `hook/on-error-names-a-failing-valve` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `payload/insert-adds-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/insert-overwrites` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/missing-key-reads-as-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/nested-input-is-never-mutated` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/json-value-types-round-trip` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/remove-drops-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/remove-of-a-missing-key-is-a-no-op` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/merge-right-side-wins` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/has-reports-presence-even-for-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `payload/mutable-escape-hatch-sets-in-place` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `pipeline/empty-returns-input` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `pipeline/steps-run-in-order` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `pipeline/filter-error-halts-the-run` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `pipeline/nested-pipeline-is-a-step` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `tap/observes-without-changing-output` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `tap/failure-never-halts-the-pipeline` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `tap/failure-does-not-stop-later-taps` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `tap/sees-payload-after-a-skipped-valve` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| `valve/runs-when-predicate-true` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `valve/skips-when-predicate-false` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `valve/missing-key-is-falsy` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `valve/sees-upstream-changes` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | -| `valve/independent-gates` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | +| case | python:run | python:run_async | python:static-steps | python:static-inline | python:static-observe | typescript:run | typescript:static | +|---|---|---|---|---|---|---|---| +| `hook/lifecycle-on-success` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `hook/on-error-when-a-filter-fails` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `hook/not-notified-of-a-tap-failure` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `hook/on-error-names-a-failing-valve` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `payload/insert-adds-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/insert-overwrites` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/missing-key-reads-as-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/nested-input-is-never-mutated` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/json-value-types-round-trip` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/remove-drops-a-key` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/remove-of-a-missing-key-is-a-no-op` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/merge-right-side-wins` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/has-reports-presence-even-for-null` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `payload/mutable-escape-hatch-sets-in-place` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `pipeline/empty-returns-input` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `pipeline/steps-run-in-order` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `pipeline/filter-error-halts-the-run` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `pipeline/nested-pipeline-is-a-step` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `tap/observes-without-changing-output` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `tap/failure-never-halts-the-pipeline` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `tap/failure-does-not-stop-later-taps` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `tap/sees-payload-after-a-skipped-valve` | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| `valve/runs-when-predicate-true` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `valve/skips-when-predicate-false` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `valve/missing-key-is-falsy` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `valve/sees-upstream-changes` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | +| `valve/independent-gates` | ✅ | ✅ | ✅* | ✅* | ✅ | ✅ | ✅* | - **python:run**: 27 pass, 0 pass*, 0 fail, 0 unsupported - **python:run_async**: 27 pass, 0 pass*, 0 fail, 0 unsupported @@ -49,5 +50,6 @@ Hooks, so they pass those cases as ✅\*. `static-observe` (export with - **python:static-inline**: 11 pass, 16 pass*, 0 fail, 0 unsupported - **python:static-observe**: 27 pass, 0 pass*, 0 fail, 0 unsupported - **typescript:run**: 27 pass, 0 pass*, 0 fail, 0 unsupported +- **typescript:static**: 11 pass, 16 pass*, 0 fail, 0 unsupported ✅* passed everything the target can observe; it does not expose State or Hooks (the fast static export). ➖ the target cannot express the case. diff --git a/conformance/README.md b/conformance/README.md index 43d2a75..9d1f338 100644 --- a/conformance/README.md +++ b/conformance/README.md @@ -43,6 +43,7 @@ load a number can leave the band. | `python:static-steps`, `python:static-inline` | `codeupipe.lower` fast export, loaded from disk (no State or Hooks: ✅\*) | | `python:static-observe` | `codeupipe.lower` export with `observe=True`: State and Hooks judged too | | `typescript:run` | `packages/codeupipe-ts` Pipeline. Node runs the `.ts` runner as it is (Node 22.18 or later). With no arguments, `run.py` leaves this target out when the machine has no such Node | +| `typescript:static` | `packages/codeupipe-compile-ts`: the pipeline as plain code, written to a file and imported (no State or Hooks: ✅\*). Needs `npm ci` in that package and a built core package; `run.py` leaves it out when they are not there | ## Statuses diff --git a/conformance/run.py b/conformance/run.py index 3497db9..a502af7 100644 --- a/conformance/run.py +++ b/conformance/run.py @@ -71,12 +71,25 @@ def _node_runs_typescript(): "python:static-inline": _python("static-inline"), "python:static-observe": _python("static-observe"), "typescript:run": _typescript("run"), + "typescript:static": _typescript("static"), +} + +# The static target runs the compile package, which must be installed, and +# that package uses the built core package. +_TYPESCRIPT_NEEDS = { + "typescript:static": [REPO / "packages/codeupipe-compile-ts/node_modules", + REPO / "packages/codeupipe-ts/dist/index.js"], } def default_targets(): - """Every target that this machine can run. TypeScript needs Node.""" - return [name for name in TARGETS if not name.startswith("typescript:") or _node_runs_typescript()] + """Every target that this machine can run. TypeScript needs Node, and the + static TypeScript target needs its package to be installed.""" + def can_run(name): + if not name.startswith("typescript:"): + return True + return _node_runs_typescript() and all(path.exists() for path in _TYPESCRIPT_NEEDS.get(name, [])) + return [name for name in TARGETS if can_run(name)] def load_cases(): diff --git a/conformance/runners/typescript/cup-ops/record-tap.ts b/conformance/runners/typescript/cup-ops/record-tap.ts index 78dc53f..b6105f4 100644 --- a/conformance/runners/typescript/cup-ops/record-tap.ts +++ b/conformance/runners/typescript/cup-ops/record-tap.ts @@ -1,16 +1,15 @@ import type { Payload, Tap } from "../../../../packages/codeupipe-ts/src/index.ts"; +import { TAPS } from "./sink.ts"; /** append get(key) to the runner's tap log */ export class RecordTap implements Tap { readonly key: string; - readonly log: unknown[]; - constructor(key: string, log: unknown[]) { + constructor(key: string) { this.key = key; - this.log = log; } observe(payload: Payload): void { - this.log.push(payload.get(this.key)); + TAPS.push(payload.get(this.key)); } } diff --git a/conformance/runners/typescript/cup-ops/sink.ts b/conformance/runners/typescript/cup-ops/sink.ts new file mode 100644 index 0000000..05246a9 --- /dev/null +++ b/conformance/runners/typescript/cup-ops/sink.ts @@ -0,0 +1,2 @@ +/** Where the Taps of the runner record what they see. The runner empties it before each case. */ +export const TAPS: unknown[] = []; diff --git a/conformance/runners/typescript/runner.ts b/conformance/runners/typescript/runner.ts index ed36af6..5e7c7eb 100644 --- a/conformance/runners/typescript/runner.ts +++ b/conformance/runners/typescript/runner.ts @@ -1,13 +1,23 @@ /** * TypeScript conformance runner: runs the cases and reports what happened. * - * node runner.ts run < cases.json > results.json + * node runner.ts < cases.json > results.json + * + * Targets + * run the Pipeline of packages/codeupipe-ts + * static the same pipeline as plain code from packages/codeupipe-compile-ts, + * written to a file and imported from it * * The runner never judges; conformance/run.py compares the results of every * target with the same expectations. Node runs this file as it is (Node - * 22.18 or later removes the types), so nothing is built or downloaded. + * 22.18 or later removes the types). The static target needs the compile + * package to be installed (npm ci in packages/codeupipe-compile-ts). Plain + * code has no State and no Hooks, so the static target reports neither. */ -import { readFileSync } from "node:fs"; +import { mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; import { Payload, Pipeline } from "../../../packages/codeupipe-ts/src/index.ts"; import type { Filter, Predicate, Tap } from "../../../packages/codeupipe-ts/src/index.ts"; @@ -26,6 +36,7 @@ import { RecordHook } from "./cup-ops/record-hook.ts"; import { RecordTap } from "./cup-ops/record-tap.ts"; import { RemoveFilter } from "./cup-ops/remove-filter.ts"; import { SetFilter } from "./cup-ops/set-filter.ts"; +import { TAPS } from "./cup-ops/sink.ts"; import { TruthyPredicate } from "./cup-ops/truthy-predicate.ts"; type Spec = Record; @@ -48,9 +59,9 @@ function makeFilter(spec: Spec): Filter { } } -function makeTap(spec: Spec, taps: unknown[]): Tap { +function makeTap(spec: Spec): Tap { switch (spec.op) { - case "record": return new RecordTap(spec.key, taps); + case "record": return new RecordTap(spec.key); case "fail": return new FailTap(spec.message); case "noop": return new NoopTap(); default: throw new Unsupported(`unknown tap op '${spec.op}'`); @@ -65,16 +76,16 @@ function makePredicate(when: Spec): Predicate { } } -function build(steps: Spec[], pipeline: Pipeline, taps: unknown[]): Pipeline { +function build(steps: Spec[], pipeline: Pipeline): Pipeline { for (const step of steps) { if (step.kind === "filter") { pipeline.addFilter(makeFilter(step)); } else if (step.kind === "tap") { - pipeline.addTap(makeTap(step, taps)); + pipeline.addTap(makeTap(step)); } else if (step.kind === "valve") { pipeline.addValve(makePredicate(step.when), makeFilter(step.filter)); } else if (step.kind === "pipeline") { - pipeline.addPipeline(build(step.steps, new NestedPipeline(), taps)); + pipeline.addPipeline(build(step.steps, new NestedPipeline())); } else { throw new Unsupported(`unknown step kind '${step.kind}'`); } @@ -94,9 +105,8 @@ function namesByLabel(pipeline: Pipeline, steps: Spec[]): Map { } async function runCase(spec: Case): Promise> { - const taps: unknown[] = []; const events: string[] = []; - const pipeline = build(spec.steps, new Pipeline(), taps); + const pipeline = build(spec.steps, new Pipeline()); const names = namesByLabel(pipeline, spec.steps); const named = (label: string) => names.get(label) ?? label; if (spec.hooks) pipeline.useHook(new RecordHook(events)); @@ -114,7 +124,7 @@ async function runCase(spec: Case): Promise> { skipped: state.skipped.map(named), errors: state.errors.map(([label]) => named(label)), }; - result.taps = taps; + result.taps = [...TAPS]; if (spec.hooks) { result.hooks = events.map((event) => { const at = event.indexOf(":"); @@ -125,17 +135,104 @@ async function runCase(spec: Case): Promise> { return result; } +// -- the static target -------------------------------------------------- + +const HERE = import.meta.dirname; +const CLASS_OF: Record = { + set: ["SetFilter", "set-filter"], copy: ["CopyFilter", "copy-filter"], add: ["AddFilter", "add-filter"], + append: ["AppendFilter", "append-filter"], remove: ["RemoveFilter", "remove-filter"], + merge: ["MergeFilter", "merge-filter"], has: ["HasFilter", "has-filter"], mutate: ["MutateFilter", "mutate-filter"], + fail: ["FailFilter", "fail-filter"], record: ["RecordTap", "record-tap"], "tap:fail": ["FailTap", "fail-tap"], + noop: ["NoopTap", "noop-tap"], equals: ["EqualsPredicate", "equals-predicate"], truthy: ["TruthyPredicate", "truthy-predicate"], +}; +const ARGUMENTS: Record = { + set: ["key", "value"], copy: ["from", "to"], add: ["key", "by"], append: ["key", "value"], remove: ["key"], + merge: ["value"], has: ["key", "to"], mutate: ["key", "value"], fail: ["message"], record: ["key"], + "tap:fail": ["message"], noop: [], equals: ["key", "value"], truthy: ["key"], +}; + +/** The case as the source of a pipeline class: the form that the compiler reads. */ +function sourceOf(spec: Case): string { + const imports = new Map(); + const classes: string[] = []; + const made = (op: string, step: Spec): string => { + const entry = CLASS_OF[op]; + if (entry === undefined) throw new Unsupported(`unknown op '${op}'`); + imports.set(entry[0], join(HERE, "cup-ops", entry[1] + ".ts")); + return `new ${entry[0]}(${(ARGUMENTS[op] ?? []).map((name) => JSON.stringify(step[name])).join(", ")})`; + }; + const pipelineClass = (name: string, steps: Spec[]): void => { + const lines: string[] = []; + for (const step of steps) { + if (step.kind === "filter") { + lines.push(`this.addFilter(${made(step.op, step)});`); + } else if (step.kind === "tap") { + lines.push(`this.addTap(${made(step.op === "fail" ? "tap:fail" : step.op, step)});`); + } else if (step.kind === "valve") { + lines.push(`this.addValve(${made(step.when.op, step.when)}, ${made(step.filter.op, step.filter)});`); + } else if (step.kind === "pipeline") { + const inner = `Inner${classes.length + 1}Pipeline`; + classes.push(""); // keeps the place, so that each nested class has its own number + pipelineClass(inner, step.steps); + lines.push(`this.addPipeline(new ${inner}());`); + } else { + throw new Unsupported(`unknown step kind '${step.kind}'`); + } + } + classes.push(`export class ${name} extends Pipeline {\n constructor() {\n super();\n${lines.map((line) => " " + line).join("\n")}\n }\n}`); + }; + pipelineClass("CasePipeline", spec.steps); + const head = [`import { Pipeline } from ${JSON.stringify(join(HERE, "../../../packages/codeupipe-ts/src/index.ts"))};`, + ...[...imports].map(([name, file]) => `import { ${name} } from ${JSON.stringify(file)};`)]; + return [...head, "", ...classes.filter((text) => text !== "")].join("\n") + "\n"; +} + +type Compile = (file: string, className: string, options: { outFile: string }) => Promise; + +async function runStatic(spec: Case, compile: Compile, directory: string, index: number): Promise> { + const sourceFile = join(directory, `case-${index}-pipeline.ts`); + const outFile = join(directory, `case-${index}-compiled.ts`); + writeFileSync(sourceFile, sourceOf(spec)); + try { + await compile(sourceFile, "CasePipeline", { outFile }); + } catch (error) { + if (error instanceof Error && error.name === "CompileRefusal") throw new Unsupported(error.message); + throw error; + } + const compiled: { run(input: Record): Promise> } = await import(pathToFileURL(outFile).href); + + const data = structuredClone(spec.input); + const result: Record = { id: spec.id }; + try { + result.output = await compiled.run(data); + } catch (error) { + result.error = error instanceof Error ? `${error.name}: ${error.message}` : String(error); + } + result.taps = [...TAPS]; + result.input_after = data; + return result; +} + +// -- main ----------------------------------------------------------------- + const target = process.argv[2]; -if (target !== "run") { - console.error(`unknown target '${target}'. The target is: run`); +if (target !== "run" && target !== "static") { + console.error(`unknown target '${target}'. The targets are: run, static`); process.exit(2); } +let compile: Compile | undefined; +let directory = ""; +if (target === "static") { + ({ compile } = await import("../../../packages/codeupipe-compile-ts/src/index.ts")); + directory = mkdtempSync(join(tmpdir(), "cup-conformance-")); +} const cases: Case[] = JSON.parse(readFileSync(0, "utf8")); const results: Record[] = []; -for (const spec of cases) { +for (const [index, spec] of cases.entries()) { + TAPS.length = 0; try { - results.push(await runCase(spec)); + results.push(compile === undefined ? await runCase(spec) : await runStatic(spec, compile, directory, index)); } catch (error) { if (!(error instanceof Unsupported)) throw error; results.push({ id: spec.id, unsupported: error.message }); diff --git a/packages/codeupipe-compile-ts/.gitignore b/packages/codeupipe-compile-ts/.gitignore new file mode 100644 index 0000000..b947077 --- /dev/null +++ b/packages/codeupipe-compile-ts/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +dist/ diff --git a/packages/codeupipe-compile-ts/LICENSE b/packages/codeupipe-compile-ts/LICENSE new file mode 100644 index 0000000..f433b1a --- /dev/null +++ b/packages/codeupipe-compile-ts/LICENSE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/packages/codeupipe-compile-ts/NOTICE b/packages/codeupipe-compile-ts/NOTICE new file mode 100644 index 0000000..fd0d109 --- /dev/null +++ b/packages/codeupipe-compile-ts/NOTICE @@ -0,0 +1,4 @@ +codeupipe +Copyright 2026 Orchestrate LLC + +Licensed under the Apache License, Version 2.0. See LICENSE. diff --git a/packages/codeupipe-compile-ts/README.md b/packages/codeupipe-compile-ts/README.md new file mode 100644 index 0000000..b374175 --- /dev/null +++ b/packages/codeupipe-compile-ts/README.md @@ -0,0 +1,121 @@ +# codeupipe-compile for TypeScript + +Writes a codeupipe pipeline as one plain TypeScript module with no import +of codeupipe. It reads the source and does not run it. It is not on npm +yet. + +**The rule: code that compiles gives the same output as the pipeline. A +pipeline for which the compiler cannot keep that rule is refused, with the +place named.** A pipeline that compiles and then behaves in a different way +is a bug. + +The compiler is a codeupipe pipeline: +[src/compile-pipeline.ts](src/compile-pipeline.ts). It depends on +`codeupipe` and on the `typescript` package, which it uses to read and +write code. + +```bash +codeupipe-compile src/pricing-pipeline.ts:PricingPipeline --out build/pricing.ts +``` + +Exit code: **0** the module is written, **1** the pipeline is refused (the +message names the place and the fix), **2** the command could not run. +Nothing is written on 1 or 2. + +From TypeScript: `await compile(file, className, { outFile })` returns the +text of the module, and throws `CompileRefusal`. + +## Example + +The pricing example of the core package compiles to this module: + +```ts +const cup_01_discount_filter_rate: number = 0.1; +function cup_01_discount_filter(payload: Map) { + const price = cup_get(payload, "price", 0) * (1 - cup_01_discount_filter_rate); + return payload.set("price", Math.round(price * 100) / 100); +} + +// ... one function for each step ... + +export async function run(input: Record): Promise> { + const data = new Map(Object.entries(input)); + cup_01_discount_filter(data); + if (cup_02_when_at_least_predicate(data)) cup_02_mark_bulk_filter(data); + try { cup_03_audit_tap(data); } catch { /* a Tap failure does not stop the run */ } + return Object.fromEntries(data); +} +``` + +`run` takes a plain object and returns a new plain object. The object that +the caller gives is not changed. + +## What compiles + +| Shape | Plain form | +|---|---| +| A pipeline class | `export async function run(input)`. The class extends `Pipeline`, and its constructor calls `super()` and then only adds steps | +| Filter (`call`) | A function on a `Map`. An `async call` is awaited | +| Valve | `if (predicate(data)) filter(data);` | +| Tap (`observe`) | The call in `try`/`catch`: a failure does not stop the run | +| Nested pipeline | Its steps, in place | +| A setting of an instance (`this.rate`) | A constant, from the literal that `new DiscountFilter(0.1)` gives | +| `get`, `has`, `insert`, `remove`, `toDict`, `merge(new Payload(map))` | Operations on the `Map` | +| An import of a step file | The same import, relative to the compiled file | + +A step is made in the call that adds it: `this.addFilter(new X(...))`. The +arguments are literals: text, numbers, `true`, `false`, `null`, and lists +and objects of them. + +## What is refused + +| Shape | What to do | +|---|---| +| A Hook, or `observe(...)` on the pipeline | The compiled code has no lifecycle and no State. Run the pipeline itself | +| A constructor of the pipeline that does more than add steps | Keep only `super()` and the `add...` calls | +| An argument that is not a literal, or a setting that the constructor computes | Give literal settings, and compute in the method | +| A step that assigns to `this`, calls its own helper method, or extends a class | Put the code of the step in one method of one class | +| A Filter that returns something that is not the Payload it was given | Return the payload, or a write on it | +| `copy`, `withTrace`, `traceId`, `lineage` | Plain data has none of these. Run the pipeline itself | +| The Payload under another name, or given to other code | Use it only through its operations | +| A Tap or a Predicate that writes, an async Predicate | Only a Filter writes | +| A name that the step file declares beside the class | Move it to its own file and import it | +| A step class from a package | Compile steps whose source is in the project | + +The tests for each refusal are in [tests/refusals.test.ts](tests/refusals.test.ts). + +## What the compiled code does not have + +- **State.** No `executed`, `skipped`, `errors` or `timings`. +- **Hooks.** A pipeline with a Hook is refused. +- **Tap errors.** They are contained, as in the pipeline, but they are not + recorded. +- **Labels.** A function has a name such as `cup_01_discount_filter`. + +So the conformance suite judges the target `typescript:static` on output, +on Taps and on an unchanged input: 11 cases pass, and the 16 cases that +also check State or Hooks pass on what the code can show. + +The compiled module is like one instance of the pipeline: a setting that is +a list or an object is one value for every run. + +## Differences from the Python export + +Python has `python -m codeupipe.lower`. It reads a pipeline object that is +in memory. This compiler reads the source of a pipeline class. Python also +has an inline layout and an observable export (State and Hooks); this +compiler has neither. + +## Development + +```bash +npm install # links ../codeupipe-ts and installs the TypeScript compiler +npm test # builds the core package first, then runs the tests +npm run typecheck # also checks the conformance runner +npm run build # JavaScript and type files in dist/ +python ../../conformance/run.py typescript:static +``` + +## License + +Apache License 2.0. diff --git a/packages/codeupipe-compile-ts/package-lock.json b/packages/codeupipe-compile-ts/package-lock.json new file mode 100644 index 0000000..7b691b9 --- /dev/null +++ b/packages/codeupipe-compile-ts/package-lock.json @@ -0,0 +1,73 @@ +{ + "name": "codeupipe-compile", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "codeupipe-compile", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "codeupipe": "file:../codeupipe-ts", + "typescript": "^5.9.0" + }, + "bin": { + "codeupipe-compile": "dist/cli.js" + }, + "devDependencies": { + "@types/node": "^24.0.0" + }, + "engines": { + "node": ">=22.18" + } + }, + "../codeupipe-ts": { + "name": "codeupipe", + "version": "0.1.0", + "license": "Apache-2.0", + "devDependencies": { + "@types/node": "^24.0.0", + "esbuild": "^0.28.2", + "typescript": "^5.9.0" + }, + "engines": { + "node": ">=22.18" + } + }, + "node_modules/@types/node": { + "version": "24.19.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.19.2.tgz", + "integrity": "sha512-93d49wCWJkCyN87MxLpNN6KXPCwFw7bVUg79mGUV1OS40JGWwVCr8n61DXCSqz7CV6/VfPkb7xec7uP2M+QHWQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": ">=7.24.0 <7.24.7" + } + }, + "node_modules/codeupipe": { + "resolved": "../codeupipe-ts", + "link": true + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "7.24.6", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.24.6.tgz", + "integrity": "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/packages/codeupipe-compile-ts/package.json b/packages/codeupipe-compile-ts/package.json new file mode 100644 index 0000000..d5e2406 --- /dev/null +++ b/packages/codeupipe-compile-ts/package.json @@ -0,0 +1,38 @@ +{ + "name": "codeupipe-compile", + "version": "0.1.0", + "private": true, + "description": "Writes a codeupipe pipeline as plain TypeScript with no import of codeupipe.", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/orchestrate-solutions/codeupipe.git", + "directory": "packages/codeupipe-compile-ts" + }, + "type": "module", + "main": "./dist/index.js", + "types": "./dist/index.d.ts", + "bin": { + "codeupipe-compile": "./dist/cli.js" + }, + "files": ["dist", "LICENSE", "NOTICE", "README.md"], + "engines": { + "node": ">=22.18" + }, + "scripts": { + "core": "npm --prefix ../codeupipe-ts run build", + "pretest": "npm run core", + "test": "node --test \"tests/*.test.ts\"", + "pretypecheck": "npm run core", + "typecheck": "tsc --noEmit", + "prebuild": "npm run core", + "build": "tsc -p tsconfig.build.json" + }, + "dependencies": { + "codeupipe": "file:../codeupipe-ts", + "typescript": "^5.9.0" + }, + "devDependencies": { + "@types/node": "^24.0.0" + } +} diff --git a/packages/codeupipe-compile-ts/src/cli.ts b/packages/codeupipe-compile-ts/src/cli.ts new file mode 100644 index 0000000..820aabf --- /dev/null +++ b/packages/codeupipe-compile-ts/src/cli.ts @@ -0,0 +1,45 @@ +#!/usr/bin/env node +/** + * codeupipe-compile FILE:ClassName --out FILE + * + * Exit code: 0 the module is written, 1 the pipeline is refused (the message + * names the place and the fix), 2 the command could not run. Nothing is + * written on 1 or 2. + */ +import { parseArgs } from "node:util"; + +import { compile } from "./compile.ts"; +import { CompileRefusal } from "./compile-refusal.ts"; + +async function main(argv: string[]): Promise { + let target: string | undefined; + let outFile: string | undefined; + try { + const args = parseArgs({ args: argv, allowPositionals: true, options: { out: { type: "string" } } }); + target = args.positionals[0]; + outFile = args.values.out; + if (args.positionals.length !== 1) target = undefined; + } catch (error) { + console.error(`error: ${error instanceof Error ? error.message : String(error)}`); + return 2; + } + const colon = target?.lastIndexOf(":") ?? -1; + if (target === undefined || outFile === undefined || colon <= 0 || colon === target.length - 1) { + console.error("usage: codeupipe-compile FILE:ClassName --out FILE"); + return 2; + } + try { + await compile(target.slice(0, colon), target.slice(colon + 1), { outFile }); + } catch (error) { + if (error instanceof CompileRefusal) { + console.error(`refused: ${error.message}`); + return 1; + } + console.error(`error: ${error instanceof Error ? error.message : String(error)}`); + return 2; + } + console.log(outFile); + return 0; +} + +process.exitCode = await main(process.argv.slice(2)); diff --git a/packages/codeupipe-compile-ts/src/compile-pipeline.ts b/packages/codeupipe-compile-ts/src/compile-pipeline.ts new file mode 100644 index 0000000..c47a806 --- /dev/null +++ b/packages/codeupipe-compile-ts/src/compile-pipeline.ts @@ -0,0 +1,15 @@ +import { Pipeline } from "codeupipe"; + +import { EmitModuleFilter } from "./emit-module-filter.ts"; +import { LowerStepsFilter } from "./lower-steps-filter.ts"; +import { ResolvePipelineFilter } from "./resolve-pipeline-filter.ts"; + +/** The compiler, as a codeupipe pipeline. */ +export class CompilePipeline extends Pipeline { + constructor() { + super(); + this.addFilter(new ResolvePipelineFilter()); // the pipeline class to a list of steps + this.addFilter(new LowerStepsFilter()); // each step to a plain function + this.addFilter(new EmitModuleFilter()); // the functions to one module + } +} diff --git a/packages/codeupipe-compile-ts/src/compile-refusal.ts b/packages/codeupipe-compile-ts/src/compile-refusal.ts new file mode 100644 index 0000000..5f23d1d --- /dev/null +++ b/packages/codeupipe-compile-ts/src/compile-refusal.ts @@ -0,0 +1,18 @@ +/** + * The pipeline has a shape that the compiler does not write as plain code. + * + * The rule of the compiler: code that it writes behaves the same as the + * pipeline. A shape for which it cannot keep that rule is refused, with the + * place named and a hint that says what to do. + */ +export class CompileRefusal extends Error { + readonly where: string; + readonly hint: string; + + constructor(where: string, what: string, hint: string) { + super(`${where}: ${what}. ${hint}`); + this.name = "CompileRefusal"; + this.where = where; + this.hint = hint; + } +} diff --git a/packages/codeupipe-compile-ts/src/compile.ts b/packages/codeupipe-compile-ts/src/compile.ts new file mode 100644 index 0000000..e856276 --- /dev/null +++ b/packages/codeupipe-compile-ts/src/compile.ts @@ -0,0 +1,28 @@ +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; + +import { Payload } from "codeupipe"; + +import { CompilePipeline } from "./compile-pipeline.ts"; +import { realPath } from "./real-path.ts"; +import { clearSources } from "./source-files.ts"; + +/** + * Write the pipeline class `className` of `file` as one plain TypeScript + * module, and return its text. With `outFile`, the module is written there + * and its imports are relative to that place. + * + * Throws CompileRefusal for a pipeline that has no plain form. Nothing is + * written then. + */ +export async function compile(file: string, className: string, options: { outFile?: string } = {}): Promise { + clearSources(); + const outFile = options.outFile === undefined ? undefined : realPath(options.outFile); + const out = await new CompilePipeline().run(new Payload({ file: resolve(file), className, outFile: outFile ?? null })); + const code = out.get("code") ?? ""; + if (outFile !== undefined) { + mkdirSync(dirname(outFile), { recursive: true }); + writeFileSync(outFile, code); + } + return code; +} diff --git a/packages/codeupipe-compile-ts/src/emit-module-filter.ts b/packages/codeupipe-compile-ts/src/emit-module-filter.ts new file mode 100644 index 0000000..4db7e2e --- /dev/null +++ b/packages/codeupipe-compile-ts/src/emit-module-filter.ts @@ -0,0 +1,123 @@ +import { basename, dirname, relative, sep } from "node:path"; + +import type { Filter, Payload } from "codeupipe"; +import ts from "typescript"; + +import { CompileRefusal } from "./compile-refusal.ts"; +import { realPath } from "./real-path.ts"; +import { importsOf, isCoreImport } from "./source-files.ts"; +import type { ImportBinding } from "./source-files.ts"; +import type { LoweredFunction, LoweredStep } from "./step-plan.ts"; + +const HELPERS: Record = { + cup_get: `function cup_get(data: Map, key: string): T | null; +function cup_get(data: Map, key: string, fallback: T): T; +function cup_get(data: Map, key: string, fallback: unknown = null): unknown { + return data.has(key) ? data.get(key) : fallback; +}`, + cup_remove: `function cup_remove(data: Map, key: string): Map { + data.delete(key); + return data; +}`, + cup_merge: `function cup_merge(data: Map, other: Record): Map { + for (const [key, value] of Object.entries(other)) data.set(key, value); + return data; +}`, +}; + +/** The names that a file declares at its top level, without its imports. */ +function topLevelNames(tree: ts.SourceFile): Set { + const names = new Set(); + for (const statement of tree.statements) { + if (ts.isVariableStatement(statement)) { + for (const declaration of statement.declarationList.declarations) names.add(declaration.name.getText()); + } else if ((ts.isFunctionDeclaration(statement) || ts.isClassDeclaration(statement) || ts.isInterfaceDeclaration(statement) + || ts.isTypeAliasDeclaration(statement) || ts.isEnumDeclaration(statement)) && statement.name !== undefined) { + names.add(statement.name.text); + } + } + return names; +} + +/** + * Writes the module: the imports that the steps need, the helpers, one + * function for each step, and `run`. + * + * Input keys: `lowered`, `file`, `className`, optional `outFile` (the place + * of the module, so that its imports are relative to it). + * Output keys: `code`. + */ +export class EmitModuleFilter implements Filter { + call(payload: Payload): Payload { + const lowered = payload.get("lowered") ?? []; + const outFile = payload.get("outFile") ?? undefined; + const functions: LoweredFunction[] = lowered.flatMap((step) => + step.kind === "filter" ? [step.filter] : step.kind === "tap" ? [step.tap] : [step.predicate, step.filter]); + + const imports = new Map(); + const helpers = new Set(); + for (const lower of functions) { + const bindings = importsOf(lower.source); + const declared = topLevelNames(lower.source.tree); + for (const name of lower.names) { + if (HELPERS[name] !== undefined) { + helpers.add(name); + continue; + } + const binding = bindings.find((candidate) => candidate.local === name); + if (binding !== undefined) { + if (isCoreImport(binding)) { + throw new CompileRefusal(lower.name, `the step uses \`${name}\` from codeupipe`, + "Compiled code has no import of codeupipe. Use only the operations of the Payload"); + } + const other = imports.get(name); + if (other !== undefined && (other.imported !== binding.imported || (other.file ?? other.specifier) !== (binding.file ?? binding.specifier))) { + throw new CompileRefusal(lower.name, `two steps import different things with the name \`${name}\``, + "Give one of the imports another name"); + } + imports.set(name, binding); + } else if (declared.has(name)) { + throw new CompileRefusal(lower.name, `the step uses \`${name}\`, which its file declares beside the class`, + `Move \`${name}\` to its own file and import it`); + } + } + } + + const specifierOf = (binding: ImportBinding): string => { + if (binding.file === undefined) return binding.specifier; + if (outFile === undefined) return binding.file; + const path = relative(dirname(outFile), realPath(binding.file)).split(sep).join("/"); + return path.startsWith(".") ? path : "./" + path; + }; + const importLines = [...imports.values()].sort((a, b) => (a.local < b.local ? -1 : 1)).map((binding) => { + const type = binding.typeOnly ? "type " : ""; + const what = binding.imported === "default" ? binding.local + : binding.imported === "*" ? `* as ${binding.local}` + : binding.imported === binding.local ? `{ ${binding.local} }` : `{ ${binding.imported} as ${binding.local} }`; + return `import ${type}${what} from ${JSON.stringify(specifierOf(binding))};`; + }); + + const invoke = (lower: LoweredFunction) => `${lower.isAsync ? "await " : ""}${lower.name}(data)`; + const runLines = lowered.map((step) => { + if (step.kind === "filter") return ` ${invoke(step.filter)};`; + if (step.kind === "valve") return ` if (${step.predicate.name}(data)) ${invoke(step.filter)};`; + // A Tap failure never stops the run. + return ` try { ${invoke(step.tap)}; } catch { /* a Tap failure does not stop the run */ }`; + }); + + const file = payload.get("file") ?? ""; + const parts = [ + `// Made by codeupipe-compile from ${basename(file)}:${payload.get("className")}. Do not edit.\n` + + "// This file has no import of codeupipe. It gives the same output as the pipeline.", + ...(importLines.length > 0 ? [importLines.join("\n")] : []), + ...[...helpers].sort().map((name) => HELPERS[name] ?? ""), + ...functions.map((lower) => [...lower.consts, lower.code].join("\n")), + "/** Runs the steps in order on a copy of `input`, and returns the result. */\n" + + "export async function run(input: Record): Promise> {\n" + + " const data = new Map(Object.entries(input));\n" + + (runLines.length > 0 ? runLines.join("\n") + "\n" : "") + + " return Object.fromEntries(data);\n}", + ]; + return payload.insert("code", parts.join("\n\n") + "\n"); + } +} diff --git a/packages/codeupipe-compile-ts/src/index.ts b/packages/codeupipe-compile-ts/src/index.ts new file mode 100644 index 0000000..32a2bc6 --- /dev/null +++ b/packages/codeupipe-compile-ts/src/index.ts @@ -0,0 +1,4 @@ +/** codeupipe-compile: writes a codeupipe pipeline as plain TypeScript with no import of codeupipe. */ +export { compile } from "./compile.ts"; +export { CompilePipeline } from "./compile-pipeline.ts"; +export { CompileRefusal } from "./compile-refusal.ts"; diff --git a/packages/codeupipe-compile-ts/src/literal.ts b/packages/codeupipe-compile-ts/src/literal.ts new file mode 100644 index 0000000..0e843f2 --- /dev/null +++ b/packages/codeupipe-compile-ts/src/literal.ts @@ -0,0 +1,32 @@ +import ts from "typescript"; + +/** + * Whether an expression is a literal: text, a number, true, false, null, + * undefined, or a list or an object of literals. The compiler writes such a + * value into the code as a constant. + */ +export function isLiteral(node: ts.Expression): boolean { + if (ts.isParenthesizedExpression(node) || ts.isAsExpression(node) || ts.isSatisfiesExpression(node)) { + return isLiteral(node.expression); + } + if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node) || ts.isNumericLiteral(node) + || ts.isBigIntLiteral(node)) { + return true; + } + if (node.kind === ts.SyntaxKind.TrueKeyword || node.kind === ts.SyntaxKind.FalseKeyword + || node.kind === ts.SyntaxKind.NullKeyword) { + return true; + } + if (ts.isIdentifier(node)) return node.text === "undefined"; + if (ts.isPrefixUnaryExpression(node)) { + return (node.operator === ts.SyntaxKind.MinusToken || node.operator === ts.SyntaxKind.PlusToken) + && ts.isNumericLiteral(node.operand); + } + if (ts.isArrayLiteralExpression(node)) return node.elements.every((element) => isLiteral(element)); + if (ts.isObjectLiteralExpression(node)) { + return node.properties.every((property) => ts.isPropertyAssignment(property) + && (ts.isIdentifier(property.name) || ts.isStringLiteral(property.name) || ts.isNumericLiteral(property.name)) + && isLiteral(property.initializer)); + } + return false; +} diff --git a/packages/codeupipe-compile-ts/src/lower-class.ts b/packages/codeupipe-compile-ts/src/lower-class.ts new file mode 100644 index 0000000..c77222d --- /dev/null +++ b/packages/codeupipe-compile-ts/src/lower-class.ts @@ -0,0 +1,277 @@ +import ts from "typescript"; + +import { CompileRefusal } from "./compile-refusal.ts"; +import { isLiteral } from "./literal.ts"; +import type { LoweredFunction, Made } from "./step-plan.ts"; + +export type Role = "Filter" | "Predicate" | "Tap"; + +const METHOD: Record = { Filter: "call", Predicate: "test", Tap: "observe" }; +const WRITES = new Set(["insert", "remove", "merge"]); +const DATA = "Map"; + +/** `DiscountFilter` gives `discount_filter`. */ +export function snakeCase(className: string): string { + return className.replace(/(?<=[a-z0-9])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])/g, "_").toLowerCase(); +} + +function isThis(node: ts.Node): boolean { + return node.kind === ts.SyntaxKind.ThisKeyword; +} + +type Setting = { value: string; type: string | undefined }; + +/** + * The settings of one instance: each property of the class, with the value + * that this `new ClassName(...)` gives it. Each value is a literal. + */ +function settingsOf(made: Made, where: string): Map { + const settings = new Map(); + const node = made.ref.node; + const constructor = node.members.find(ts.isConstructorDeclaration); + + for (const member of node.members) { + if (!ts.isPropertyDeclaration(member)) continue; + if (!ts.isIdentifier(member.name)) { + throw new CompileRefusal(where, `the property \`${member.name.getText()}\` does not have a plain name`, "Give it a plain name"); + } + if (member.initializer !== undefined && !isLiteral(member.initializer)) { + throw new CompileRefusal(where, `the property \`${member.name.text}\` starts with a value that is not a literal`, + "Start it with a literal, or take the value from the Payload"); + } + settings.set(member.name.text, { value: member.initializer?.getText() ?? "undefined", type: member.type?.getText() }); + } + if (constructor === undefined) { + if (made.args.length > 0) throw new CompileRefusal(where, "the class has no constructor, but it is given arguments", "Remove the arguments"); + return settings; + } + + const given = new Map(); + constructor.parameters.forEach((parameter, index) => { + if (!ts.isIdentifier(parameter.name) || parameter.dotDotDotToken !== undefined || ts.getModifiers(parameter)?.length) { + throw new CompileRefusal(where, `the constructor parameter \`${parameter.getText()}\` is not a plain name`, + "Use plain parameters, and assign each one to a property"); + } + const argument = made.args[index]; + if (argument !== undefined && !isLiteral(argument)) { + throw new CompileRefusal(where, `the argument \`${argument.getText()}\` is not a literal`, + "Give literal settings (text, numbers, true, false, null, lists and objects of them)"); + } + if (argument === undefined && parameter.initializer !== undefined && !isLiteral(parameter.initializer)) { + throw new CompileRefusal(where, `the default of \`${parameter.name.text}\` is not a literal`, "Use a literal default"); + } + given.set(parameter.name.text, { + value: argument?.getText() ?? parameter.initializer?.getText() ?? "undefined", type: parameter.type?.getText(), + }); + }); + if (made.args.length > constructor.parameters.length) { + throw new CompileRefusal(where, "the class is given more arguments than its constructor has parameters", "Remove the extra arguments"); + } + + for (const statement of constructor.body?.statements ?? []) { + const assignment = ts.isExpressionStatement(statement) && ts.isBinaryExpression(statement.expression) + && statement.expression.operatorToken.kind === ts.SyntaxKind.EqualsToken ? statement.expression : undefined; + const target = assignment !== undefined && ts.isPropertyAccessExpression(assignment.left) && isThis(assignment.left.expression) + ? assignment.left.name.text : undefined; + if (assignment === undefined || target === undefined) { + throw new CompileRefusal(where, `the constructor does more than assign settings (\`${statement.getText().split("\n")[0]}\`)`, + "Keep only statements of the form this.name = parameter"); + } + const value = assignment.right; + const fromParameter = ts.isIdentifier(value) ? given.get(value.text) : undefined; + if (fromParameter === undefined && !isLiteral(value)) { + throw new CompileRefusal(where, `the constructor computes the setting \`${target}\``, + "Assign a parameter or a literal, and compute in the method"); + } + settings.set(target, { + value: fromParameter?.value ?? value.getText(), + type: settings.get(target)?.type ?? fromParameter?.type, + }); + } + return settings; +} + +/** + * One step class as one plain function. + * + * The function takes the data as a `Map`. Each operation of the Payload + * becomes an operation of the Map, and each `this.setting` becomes a + * constant. A shape with no such form is refused. + */ +export function lowerClass(made: Made, role: Role, name: string): LoweredFunction { + const node = made.ref.node; + const where = made.ref.name; + const source = made.ref.source; + const methodName = METHOD[role]; + + if ((node.heritageClauses ?? []).some((clause) => clause.token === ts.SyntaxKind.ExtendsKeyword)) { + throw new CompileRefusal(where, "the class extends another class", "Put the code of the step in one class"); + } + const method = node.members.find((member): member is ts.MethodDeclaration => + ts.isMethodDeclaration(member) && member.name.getText() === methodName); + if (method === undefined || method.body === undefined) { + throw new CompileRefusal(where, `the class has no ${methodName}(payload) method`, `A ${role} has ${methodName}(payload)`); + } + const isAsync = (ts.getModifiers(method) ?? []).some((modifier) => modifier.kind === ts.SyntaxKind.AsyncKeyword); + if (method.asteriskToken !== undefined) { + throw new CompileRefusal(where, `${methodName} is a generator`, "Make it a plain method"); + } + if (isAsync && role === "Predicate") { + throw new CompileRefusal(where, "test is async", "A Predicate gives true or false at once. Make test sync"); + } + const parameter = method.parameters[0]; + if (method.parameters.length !== 1 || parameter === undefined || !ts.isIdentifier(parameter.name)) { + throw new CompileRefusal(where, `${methodName} does not take one plain parameter`, `Write ${methodName}(payload)`); + } + const data = parameter.name.text; + const settings = settingsOf(made, where); + const used = new Set(); + const helpers = new Set(); + let returns = 0; + let depth = 0; // the number of functions inside the method that the visitor is in + + /** The parameter itself, or a chain of writes on it: `payload.insert(..).remove(..)`. */ + const isData = (expression: ts.Expression): boolean => { + if (ts.isIdentifier(expression)) return expression.text === data; + return ts.isCallExpression(expression) && ts.isPropertyAccessExpression(expression.expression) + && WRITES.has(expression.expression.name.text) && isData(expression.expression.expression); + }; + + const refuse = (what: string, hint: string): never => { + throw new CompileRefusal(where, what, hint); + }; + + const transformer: ts.TransformerFactory = (context) => { + const factory = context.factory; + const call = (helper: string, typeArguments: readonly ts.TypeNode[] | undefined, args: ts.Expression[]) => { + helpers.add(helper); + return factory.createCallExpression(factory.createIdentifier(helper), typeArguments, args); + }; + + const visit = (node: ts.Node): ts.Node => { + if (ts.isIdentifier(node) && node.text.startsWith("cup_")) { + return refuse(`the name \`${node.text}\` starts with cup_`, "The compiler uses names that start with cup_. Use another name"); + } + + // this.setting + if (ts.isPropertyAccessExpression(node) && isThis(node.expression)) { + const setting = node.name.text; + if (!settings.has(setting)) { + return refuse(`${methodName} uses \`this.${setting}\`, which is not a property with a literal value`, + "Use a setting of the instance, or a function that the file imports"); + } + const parent = node.parent; + const isWritten = (ts.isBinaryExpression(parent) && parent.left === node + && parent.operatorToken.kind >= ts.SyntaxKind.FirstAssignment && parent.operatorToken.kind <= ts.SyntaxKind.LastAssignment) + || ts.isPrefixUnaryExpression(parent) || ts.isPostfixUnaryExpression(parent); + if (isWritten) { + return refuse(`${methodName} assigns to \`this.${setting}\``, "A step keeps nothing between runs. Put the value in the Payload"); + } + used.add(setting); + return factory.createIdentifier(`${name}_${setting}`); + } + if (isThis(node)) { + return refuse(`${methodName} uses \`this\` for more than a setting`, "Read settings as this.name, and use no other member"); + } + + // payload.operation(...) + if (ts.isCallExpression(node) && ts.isPropertyAccessExpression(node.expression) && isData(node.expression.expression)) { + const operation = node.expression.name.text; + const inner = node.expression.expression; + const receiver = ts.isIdentifier(inner) ? inner : (visit(inner) as ts.Expression); + const args = node.arguments.map((argument) => visit(argument) as ts.Expression); + if (WRITES.has(operation) && role !== "Filter") { + return refuse(`${methodName} writes to the Payload (\`${data}.${operation}\`)`, "Only a Filter writes. Put the write in a Filter"); + } + if (operation === "insert" && args.length === 2) { + return factory.createCallExpression(factory.createPropertyAccessExpression(receiver, "set"), undefined, args); + } + if (operation === "remove" && args.length === 1) return call("cup_remove", undefined, [receiver, ...args]); + if (operation === "get" && (args.length === 1 || args.length === 2)) { + return call("cup_get", node.typeArguments, [receiver, ...args]); + } + if (operation === "has" && args.length === 1) { + return factory.createCallExpression(factory.createPropertyAccessExpression(receiver, "has"), undefined, args); + } + if (operation === "toDict" && args.length === 0) { + return factory.createCallExpression(factory.createPropertyAccessExpression(factory.createIdentifier("Object"), "fromEntries"), + undefined, [receiver]); + } + if (operation === "merge" && node.arguments.length === 1) { + const other = node.arguments[0]; + if (other !== undefined && ts.isNewExpression(other) && other.expression.getText() === "Payload" + && (other.arguments?.length ?? 0) <= 1) { + const map = other.arguments?.[0]; + return call("cup_merge", undefined, [receiver, map === undefined ? factory.createObjectLiteralExpression() : (visit(map) as ts.Expression)]); + } + return refuse(`${methodName} merges a Payload that it did not make there`, "Write payload.merge(new Payload(map))"); + } + return refuse(`${methodName} uses \`${data}.${operation}\` in a form that has no plain code`, + "The operations are get, has, insert, remove, merge(new Payload(map)) and toDict"); + } + + // the payload itself, in any other place + if (ts.isIdentifier(node) && node.text === data) { + const parent = node.parent; + const isName = (ts.isPropertyAccessExpression(parent) && parent.name === node) + || (ts.isPropertyAssignment(parent) && parent.name === node) + || ((ts.isMethodDeclaration(parent) || ts.isPropertyDeclaration(parent)) && parent.name === node); + if (isName) return node; + if (ts.isReturnStatement(parent) && depth === 0) return node; + if (ts.isPropertyAccessExpression(parent) && parent.expression === node) { + return refuse(`${methodName} uses \`${data}.${parent.name.text}\``, + "The operations are get, has, insert, remove, merge(new Payload(map)) and toDict"); + } + return refuse(`${methodName} gives the Payload to other code, or gives it another name`, + "Use the Payload only through its operations, and return it"); + } + + if (ts.isReturnStatement(node) && depth === 0 && role === "Filter") { + returns += 1; + if (node.expression === undefined || !isData(node.expression)) { + return refuse("call returns something that is not the Payload it was given", + "Return the payload, or a write on it: return payload.insert(key, value)"); + } + } + + if (ts.isFunctionLike(node) || ts.isClassLike(node)) { + depth += 1; + const result = ts.visitEachChild(node, visit, context); + depth -= 1; + return result; + } + return ts.visitEachChild(node, visit, context); + }; + return (body) => ts.visitEachChild(body, visit, context); + }; + + const result = ts.transform(method.body, [transformer]); + const body = result.transformed[0]; + if (body === undefined) throw new CompileRefusal(where, "the method has no body", "Give it a body"); + const last = method.body.statements[method.body.statements.length - 1]; + if (role === "Filter" && returns === 0 && !(last !== undefined && ts.isThrowStatement(last))) { + refuse("call does not return the Payload", "End call with return payload"); + } + const printed = ts.createPrinter({ removeComments: false }).printNode(ts.EmitHint.Unspecified, body, source.tree); + result.dispose(); + + const returnType = role === "Predicate" ? ": boolean" : ""; + const code = `${isAsync ? "async " : ""}function ${name}(${data}: ${DATA})${returnType} ${printed}`; + const consts = [...used].map((setting) => { + const { value, type } = settings.get(setting) ?? { value: "undefined", type: undefined }; + return `const ${name}_${setting}${type === undefined ? "" : `: ${type}`} = ${value};`; + }); + + // The identifiers that the code uses, to find which imports it needs. + const names = new Set(helpers); + const scan = ts.createSourceFile("lowered.ts", [...consts, code].join("\n"), ts.ScriptTarget.Latest, true); + const collect = (inner: ts.Node) => { + if (ts.isIdentifier(inner) && !(ts.isPropertyAccessExpression(inner.parent) && inner.parent.name === inner)) { + names.add(inner.text); + } + ts.forEachChild(inner, collect); + }; + collect(scan); + + return { name, consts, code, isAsync, names, source }; +} diff --git a/packages/codeupipe-compile-ts/src/lower-steps-filter.ts b/packages/codeupipe-compile-ts/src/lower-steps-filter.ts new file mode 100644 index 0000000..614b9e8 --- /dev/null +++ b/packages/codeupipe-compile-ts/src/lower-steps-filter.ts @@ -0,0 +1,36 @@ +import type { Filter, Payload } from "codeupipe"; + +import { lowerClass, snakeCase } from "./lower-class.ts"; +import type { LoweredStep, StepPlan } from "./step-plan.ts"; + +/** + * Writes each step as a plain function. + * + * A function has the name `cup__`. The number is the + * place of the step in the run, so two uses of one class have two names. + * + * Input keys: `steps`. + * Output keys: `lowered` (a list of LoweredStep). + */ +export class LowerStepsFilter implements Filter { + call(payload: Payload): Payload { + const steps = payload.get("steps") ?? []; + const width = Math.max(2, String(steps.length).length); + const lowered: LoweredStep[] = steps.map((step, index) => { + const number = String(index + 1).padStart(width, "0"); + const name = (prefix: string, className: string) => `cup_${number}_${prefix}${snakeCase(className)}`; + if (step.kind === "filter") { + return { kind: "filter", filter: lowerClass(step.filter, "Filter", name("", step.filter.ref.name)) }; + } + if (step.kind === "tap") { + return { kind: "tap", tap: lowerClass(step.tap, "Tap", name("", step.tap.ref.name)) }; + } + return { + kind: "valve", + predicate: lowerClass(step.predicate, "Predicate", name("when_", step.predicate.ref.name)), + filter: lowerClass(step.filter, "Filter", name("", step.filter.ref.name)), + }; + }); + return payload.insert("lowered", lowered); + } +} diff --git a/packages/codeupipe-compile-ts/src/real-path.ts b/packages/codeupipe-compile-ts/src/real-path.ts new file mode 100644 index 0000000..a39cbae --- /dev/null +++ b/packages/codeupipe-compile-ts/src/real-path.ts @@ -0,0 +1,16 @@ +import { existsSync, realpathSync } from "node:fs"; +import { basename, dirname, join, resolve } from "node:path"; + +/** + * The path with each symbolic link followed, also for a path that is not + * there yet: the part that exists is followed, and the rest is added. + * + * A relative import is correct only between two such paths, because the + * module loader follows the links before it reads the import. + */ +export function realPath(path: string): string { + const absolute = resolve(path); + if (existsSync(absolute)) return realpathSync(absolute); + const parent = dirname(absolute); + return parent === absolute ? absolute : join(realPath(parent), basename(absolute)); +} diff --git a/packages/codeupipe-compile-ts/src/resolve-pipeline-filter.ts b/packages/codeupipe-compile-ts/src/resolve-pipeline-filter.ts new file mode 100644 index 0000000..90a0c5b --- /dev/null +++ b/packages/codeupipe-compile-ts/src/resolve-pipeline-filter.ts @@ -0,0 +1,93 @@ +import { basename } from "node:path"; + +import type { Filter, Payload } from "codeupipe"; +import ts from "typescript"; + +import { CompileRefusal } from "./compile-refusal.ts"; +import { findClass, loadSource } from "./source-files.ts"; +import type { ClassRef } from "./source-files.ts"; +import type { Made, StepPlan } from "./step-plan.ts"; + +function extendsPipeline(node: ts.ClassDeclaration): boolean { + return (node.heritageClauses ?? []).some((clause) => clause.token === ts.SyntaxKind.ExtendsKeyword + && clause.types.some((type) => type.expression.getText() === "Pipeline")); +} + +/** + * Reads a pipeline class and lists its steps. + * + * The class extends `Pipeline`. Its constructor calls `super()` and then + * only adds steps: `this.addFilter(new X(...))`, `this.addValve(new P(...), + * new F(...))`, `this.addTap(new T(...))`, `this.addPipeline(new Inner())`. + * A nested pipeline is flattened in place. + * + * Input keys: `file`, `className`. + * Output keys: `steps` (a list of StepPlan). + */ +export class ResolvePipelineFilter implements Filter { + call(payload: Payload): Payload { + const file = payload.get("file") ?? ""; + const className = payload.get("className") ?? ""; + const pipeline = findClass(loadSource(file), className, `${basename(file)}:${className}`); + return payload.insert("steps", this.stepsOf(pipeline, [])); + } + + private stepsOf(pipeline: ClassRef, outer: string[]): StepPlan[] { + const where = pipeline.name; + if (outer.includes(where)) { + throw new CompileRefusal(where, "the pipeline is nested in itself", "Remove the cycle"); + } + if (!extendsPipeline(pipeline.node)) { + throw new CompileRefusal(where, "the class does not extend Pipeline", "Give the name of a class that extends Pipeline"); + } + if (pipeline.node.members.some((member) => ts.isMethodDeclaration(member) && member.name.getText() === "run")) { + throw new CompileRefusal(where, "the class has its own run", "Remove it: the compiler writes the run of Pipeline"); + } + const made = (expression: ts.Expression | undefined, role: string): Made => { + if (expression === undefined || !ts.isNewExpression(expression) || !ts.isIdentifier(expression.expression)) { + throw new CompileRefusal(where, `the ${role} is not \`new ClassName(...)\``, + "Make each step in the constructor of the pipeline, in the call that adds it"); + } + const ref = findClass(pipeline.source, expression.expression.text, where); + return { ref, args: expression.arguments ?? [], argSource: pipeline.source }; + }; + + const steps: StepPlan[] = []; + const constructor = pipeline.node.members.find(ts.isConstructorDeclaration); + for (const statement of constructor?.body?.statements ?? []) { + const call = ts.isExpressionStatement(statement) && ts.isCallExpression(statement.expression) + ? statement.expression : undefined; + if (call !== undefined && call.expression.kind === ts.SyntaxKind.SuperKeyword) continue; + const method = call !== undefined && ts.isPropertyAccessExpression(call.expression) + && call.expression.expression.kind === ts.SyntaxKind.ThisKeyword ? call.expression.name.text : undefined; + if (call === undefined || method === undefined) { + throw new CompileRefusal(where, `the constructor has a statement that does not add a step (\`${statement.getText().split("\n")[0]}\`)`, + "Keep only super() and the calls that add steps"); + } + if (method === "addFilter") { + steps.push({ kind: "filter", filter: made(call.arguments[0], "Filter") }); + } else if (method === "addValve") { + steps.push({ kind: "valve", predicate: made(call.arguments[0], "Predicate"), filter: made(call.arguments[1], "Filter") }); + } else if (method === "addTap") { + steps.push({ kind: "tap", tap: made(call.arguments[0], "Tap") }); + } else if (method === "addPipeline") { + const inner = made(call.arguments[0], "nested pipeline"); + if (inner.args.length > 0) { + throw new CompileRefusal(where, `the nested pipeline \`${inner.ref.name}\` takes arguments`, + "Give the nested pipeline no arguments"); + } + steps.push(...this.stepsOf(inner.ref, [...outer, where])); + } else if (method === "useHook") { + throw new CompileRefusal(where, "the pipeline has a Hook", + "The compiled code has no lifecycle. Remove the Hook, or run the pipeline itself"); + } else if (method === "observe") { + throw new CompileRefusal(where, "the pipeline turns on timing or lineage", + "The compiled code has no State. Remove the call, or run the pipeline itself"); + } else { + throw new CompileRefusal(where, `the constructor calls \`this.${method}\``, + "Keep only super() and the calls that add steps"); + } + } + return steps; + } +} diff --git a/packages/codeupipe-compile-ts/src/source-files.ts b/packages/codeupipe-compile-ts/src/source-files.ts new file mode 100644 index 0000000..d3ac45e --- /dev/null +++ b/packages/codeupipe-compile-ts/src/source-files.ts @@ -0,0 +1,132 @@ +import { existsSync, readFileSync, statSync } from "node:fs"; +import { basename, dirname, join, resolve } from "node:path"; + +import ts from "typescript"; + +import { CompileRefusal } from "./compile-refusal.ts"; + +/** One parsed file. */ +export type Source = { path: string; tree: ts.SourceFile }; + +/** A class, and the file that it is in. */ +export type ClassRef = { source: Source; node: ts.ClassDeclaration; name: string }; + +/** One name that a file imports. `imported` is the exported name, "default" or "*". */ +export type ImportBinding = { + local: string; + imported: string; + specifier: string; + /** The file, for a relative import. Undefined for a package. */ + file: string | undefined; + typeOnly: boolean; +}; + +const cache = new Map(); + +export function loadSource(path: string): Source { + const absolute = resolve(path); + let source = cache.get(absolute); + if (source === undefined) { + const text = readFileSync(absolute, "utf8"); + source = { path: absolute, tree: ts.createSourceFile(absolute, text, ts.ScriptTarget.Latest, true) }; + cache.set(absolute, source); + } + return source; +} + +/** Forget the files that were read. For a tool that compiles a file again after it changes. */ +export function clearSources(): void { + cache.clear(); +} + +function isFile(path: string): boolean { + return existsSync(path) && statSync(path).isFile(); +} + +/** The file of a relative import, or undefined for a package. */ +function resolveModule(from: string, specifier: string): string | undefined { + if (!specifier.startsWith(".") && !specifier.startsWith("/")) return undefined; + const base = resolve(dirname(from), specifier); + const candidates = [base, base.replace(/\.js$/, ".ts"), base.replace(/\.mjs$/, ".mts"), base + ".ts", base + ".tsx", + join(base, "index.ts")]; + return candidates.find(isFile); +} + +/** The names that a file imports. */ +export function importsOf(source: Source): ImportBinding[] { + const found: ImportBinding[] = []; + for (const statement of source.tree.statements) { + if (!ts.isImportDeclaration(statement) || !ts.isStringLiteral(statement.moduleSpecifier)) continue; + const clause = statement.importClause; + if (clause === undefined) continue; + const specifier = statement.moduleSpecifier.text; + const file = resolveModule(source.path, specifier); + const base = { specifier, file }; + if (clause.name !== undefined) { + found.push({ ...base, local: clause.name.text, imported: "default", typeOnly: clause.isTypeOnly }); + } + const named = clause.namedBindings; + if (named !== undefined && ts.isNamespaceImport(named)) { + found.push({ ...base, local: named.name.text, imported: "*", typeOnly: clause.isTypeOnly }); + } else if (named !== undefined) { + for (const element of named.elements) { + found.push({ + ...base, local: element.name.text, imported: (element.propertyName ?? element.name).text, + typeOnly: clause.isTypeOnly || element.isTypeOnly, + }); + } + } + } + return found; +} + +/** + * Whether an import is of codeupipe itself: the package by its name, or the + * index file of its source (the directory that has `payload.ts` and + * `pipeline.ts`). + */ +export function isCoreImport(binding: ImportBinding): boolean { + if (binding.specifier === "codeupipe" || binding.specifier.startsWith("codeupipe/")) return true; + if (binding.file === undefined || !/^index\.[cm]?[jt]s$/.test(basename(binding.file))) return false; + const directory = dirname(binding.file); + return ["payload", "pipeline"].every((name) => isFile(join(directory, name + ".ts")) || isFile(join(directory, name + ".js"))); +} + +function hasModifier(node: ts.Node, kind: ts.SyntaxKind): boolean { + return ts.canHaveModifiers(node) && (ts.getModifiers(node) ?? []).some((modifier) => modifier.kind === kind); +} + +function topLevelClass(source: Source, name: string): ts.ClassDeclaration | undefined { + return source.tree.statements.find( + (statement): statement is ts.ClassDeclaration => ts.isClassDeclaration(statement) && statement.name?.text === name); +} + +/** + * The class that the name `name` means in the file `source`: a class of the + * file, or a class that the file imports from a file of the project. + */ +export function findClass(source: Source, name: string, where: string): ClassRef { + const own = topLevelClass(source, name); + if (own !== undefined) return { source, node: own, name }; + + const binding = importsOf(source).find((candidate) => candidate.local === name); + if (binding === undefined) { + throw new CompileRefusal(where, `\`${name}\` is not a class of ${basename(source.path)} and is not imported there`, + "Declare the class at the top level of a file, and import it"); + } + if (binding.file === undefined) { + throw new CompileRefusal(where, `\`${name}\` comes from the package "${binding.specifier}", and its source is not read`, + "Compile steps whose source is in the project"); + } + const target = loadSource(binding.file); + if (binding.imported === "default") { + const node = target.tree.statements.find((statement): statement is ts.ClassDeclaration => + ts.isClassDeclaration(statement) && hasModifier(statement, ts.SyntaxKind.DefaultKeyword)); + if (node !== undefined) return { source: target, node, name: node.name?.text ?? name }; + } else if (binding.imported !== "*") { + const node = topLevelClass(target, binding.imported); + if (node !== undefined) return { source: target, node, name: binding.imported }; + } + throw new CompileRefusal(where, `\`${name}\` is not a class at the top level of ${basename(binding.file)}`, + "Declare the class in the file that it is imported from"); +} diff --git a/packages/codeupipe-compile-ts/src/step-plan.ts b/packages/codeupipe-compile-ts/src/step-plan.ts new file mode 100644 index 0000000..1dfcefe --- /dev/null +++ b/packages/codeupipe-compile-ts/src/step-plan.ts @@ -0,0 +1,32 @@ +import type ts from "typescript"; + +import type { ClassRef, Source } from "./source-files.ts"; + +/** A step class, with the arguments that the pipeline gives to its constructor. */ +export type Made = { ref: ClassRef; args: readonly ts.Expression[]; argSource: Source }; + +/** One step of the pipeline, in the order of the run. A nested pipeline is already flattened. */ +export type StepPlan = + | { kind: "filter"; filter: Made } + | { kind: "valve"; predicate: Made; filter: Made } + | { kind: "tap"; tap: Made }; + +/** One step as plain code. */ +export type LoweredFunction = { + /** The name of the function. */ + name: string; + /** The constants that hold the settings of the instance. */ + consts: string[]; + /** The function. */ + code: string; + isAsync: boolean; + /** The identifiers that the constants and the function use. */ + names: Set; + /** The file that the class is in. */ + source: Source; +}; + +export type LoweredStep = + | { kind: "filter"; filter: LoweredFunction } + | { kind: "valve"; predicate: LoweredFunction; filter: LoweredFunction } + | { kind: "tap"; tap: LoweredFunction }; diff --git a/packages/codeupipe-compile-ts/tests/cli.test.ts b/packages/codeupipe-compile-ts/tests/cli.test.ts new file mode 100644 index 0000000..6ff2222 --- /dev/null +++ b/packages/codeupipe-compile-ts/tests/cli.test.ts @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { PIPELINE_IMPORT, project } from "./compile-in.ts"; + +const CLI = join(import.meta.dirname, "../src/cli.ts"); +const STEP = `export class SetFilter { call(payload) { return payload.insert("a", 1); } }\n`; +const pipeline = (extra = "") => PIPELINE_IMPORT + `import { SetFilter } from "./set-filter.ts"; +export class MainPipeline extends Pipeline { constructor() { super(); this.addFilter(new SetFilter()); ${extra} } } +`; + +function run(cwd: string, ...args: string[]) { + const done = spawnSync(process.execPath, [CLI, ...args], { cwd, encoding: "utf8" }); + return { code: done.status, out: done.stdout, err: done.stderr }; +} + +test("exit 0: the module is written, and its path is printed", () => { + const root = project({ "set-filter.ts": STEP, "pipeline.ts": pipeline() }); + const done = run(root, "pipeline.ts:MainPipeline", "--out", "build/main.ts"); + assert.equal(done.code, 0, done.err); + assert.equal(done.out.trim(), "build/main.ts"); + assert.match(readFileSync(join(root, "build/main.ts"), "utf8"), /export async function run/); +}); + +test("exit 1: a refusal names the place and the fix, and nothing is written", () => { + const root = project({ "set-filter.ts": STEP, "pipeline.ts": pipeline("this.useHook(null);") }); + const done = run(root, "pipeline.ts:MainPipeline", "--out", "build/main.ts"); + assert.equal(done.code, 1); + assert.match(done.err, /^refused: MainPipeline: the pipeline has a Hook\. /); + assert.equal(existsSync(join(root, "build")), false); +}); + +test("exit 2: wrong arguments, and a file that is not there", () => { + const root = project({}); + assert.equal(run(root).code, 2); + assert.equal(run(root, "pipeline.ts:MainPipeline").code, 2); + assert.equal(run(root, "pipeline.ts", "--out", "x.ts").code, 2); + assert.equal(run(root, "nothing.ts:MainPipeline", "--out", "x.ts").code, 2); +}); diff --git a/packages/codeupipe-compile-ts/tests/compile-in.ts b/packages/codeupipe-compile-ts/tests/compile-in.ts new file mode 100644 index 0000000..476e37b --- /dev/null +++ b/packages/codeupipe-compile-ts/tests/compile-in.ts @@ -0,0 +1,26 @@ +// A helper for the tests: write source files to a new directory and compile one pipeline class. +import { mkdtempSync, realpathSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +import { compile } from "../src/index.ts"; + +export const PIPELINE_IMPORT = `import { Pipeline } from "codeupipe";\n`; + +export type Compiled = { code: string; outFile: string; run(input: Record): Promise> }; + +export function project(files: Record): string { + const root = realpathSync(mkdtempSync(join(tmpdir(), "cup-compile-"))); + for (const [name, text] of Object.entries(files)) writeFileSync(join(root, name), text); + return root; +} + +/** Compile `className` of `pipeline.ts` and import the module that was written. */ +export async function compileIn(files: Record, className = "MainPipeline"): Promise { + const root = project(files); + const outFile = join(root, "compiled.ts"); + const code = await compile(join(root, "pipeline.ts"), className, { outFile }); + const module: { run: Compiled["run"] } = await import(pathToFileURL(outFile).href); + return { code, outFile, run: module.run }; +} diff --git a/packages/codeupipe-compile-ts/tests/compile.test.ts b/packages/codeupipe-compile-ts/tests/compile.test.ts new file mode 100644 index 0000000..0f73f51 --- /dev/null +++ b/packages/codeupipe-compile-ts/tests/compile.test.ts @@ -0,0 +1,139 @@ +import assert from "node:assert/strict"; +import { join } from "node:path"; +import { test } from "node:test"; + +import ts from "typescript"; + +import { compile } from "../src/index.ts"; +import { PIPELINE_IMPORT, compileIn, project } from "./compile-in.ts"; + +const PRICING = join(import.meta.dirname, "../../codeupipe-ts/examples/pricing/pricing-pipeline.ts"); + +test("the pricing example compiles, and the code has no import", async () => { + const code = await compile(PRICING, "PricingPipeline"); + assert.doesNotMatch(code, /^import /m); + assert.doesNotMatch(code, /codeupipe"/); + assert.match(code, /export async function run\(/); +}); + +test("the compiled pricing example gives the same output as the pipeline", async () => { + const { PricingPipeline } = await import(PRICING); + const { Payload } = await import("../../codeupipe-ts/src/index.ts"); + const file = join(project({}), "pricing.ts"); + await compile(PRICING, "PricingPipeline", { outFile: file }); + const compiled: { run(input: Record): Promise> } = await import(file); + for (const input of [{ price: 100.0, qty: 12 }, { price: 19.99, qty: 2 }, {}, { qty: 10 }]) { + const given = structuredClone(input); + const dynamic = (await new PricingPipeline().run(new Payload(input))).toDict(); + assert.deepEqual(await compiled.run(given), dynamic); + assert.deepEqual(given, input); // the caller's input is not changed + } +}); + +test("the compiled code passes a strict type check", async () => { + const file = join(project({}), "pricing.ts"); + await compile(PRICING, "PricingPipeline", { outFile: file }); + const program = ts.createProgram([file], { + strict: true, noEmit: true, target: ts.ScriptTarget.ES2022, module: ts.ModuleKind.NodeNext, + moduleResolution: ts.ModuleResolutionKind.NodeNext, lib: ["lib.es2022.d.ts"], types: [], + }); + const problems = ts.getPreEmitDiagnostics(program).map((d) => ts.flattenDiagnosticMessageText(d.messageText, " ")); + assert.deepEqual(problems, []); +}); + +test("each operation of the Payload has a plain form", async () => { + const compiled = await compileIn({ + "ops-filter.ts": `import { Payload } from "codeupipe"; +export class OpsFilter { + readonly extra: Record; + constructor(extra: Record) { this.extra = extra; } + call(payload: Payload): Payload { + payload.insert("had", payload.has("a")).remove("a"); + payload.merge(new Payload(this.extra)); + payload.insert("copy", payload.toDict()); + payload.insert("missing", payload.get("zz")); + return payload.insert("n", payload.get("n", 0) + 1).insert("last", true); + } +} +`, + "pipeline.ts": PIPELINE_IMPORT + `import { OpsFilter } from "./ops-filter.ts"; +export class MainPipeline extends Pipeline { + constructor() { super(); this.addFilter(new OpsFilter({ b: 2, n: 5 })); } +} +`, + }); + assert.deepEqual(await compiled.run({ a: 1 }), { had: true, b: 2, n: 6, copy: { had: true, b: 2, n: 5 }, missing: null, last: true }); +}); + +test("a Valve, a Tap that fails, a second use of a class, and a nested pipeline", async () => { + const compiled = await compileIn({ + "add-filter.ts": `export class AddFilter { + by: number; + constructor(by = 1) { this.by = by; } + call(payload) { return payload.insert("n", payload.get("n", 0) + this.by); } +} +`, + "is-on-predicate.ts": `export class IsOnPredicate { test(payload) { return payload.get("on") === true; } }\n`, + "fail-tap.ts": `export class FailTap { observe(payload) { throw new Error("sink is down"); } }\n`, + "pipeline.ts": PIPELINE_IMPORT + `import { AddFilter } from "./add-filter.ts"; +import { FailTap } from "./fail-tap.ts"; +import { IsOnPredicate } from "./is-on-predicate.ts"; +class InnerPipeline extends Pipeline { + constructor() { super(); this.addFilter(new AddFilter(10)); this.addTap(new FailTap()); } +} +export class MainPipeline extends Pipeline { + constructor() { + super(); + this.addFilter(new AddFilter()); + this.addValve(new IsOnPredicate(), new AddFilter(100)); + this.addPipeline(new InnerPipeline()); + } +} +`, + }); + assert.deepEqual(await compiled.run({ on: true }), { on: true, n: 111 }); + assert.deepEqual(await compiled.run({ on: false }), { on: false, n: 11 }); + assert.match(compiled.code, /cup_01_add_filter\(data\);\n if \(cup_02_when_is_on_predicate\(data\)\) cup_02_add_filter\(data\);\n cup_03_add_filter\(data\);/); +}); + +test("an async step is awaited, and a Filter failure is the same error", async () => { + const compiled = await compileIn({ + "later-filter.ts": `export class LaterFilter { + async call(payload) { await Promise.resolve(); return payload.insert("later", true); } +} +`, + "check-filter.ts": `export class CheckFilter { + call(payload) { if (payload.has("bad")) throw new RangeError("bad input"); return payload; } +} +`, + "pipeline.ts": PIPELINE_IMPORT + `import { CheckFilter } from "./check-filter.ts"; +import { LaterFilter } from "./later-filter.ts"; +export class MainPipeline extends Pipeline { + constructor() { super(); this.addFilter(new LaterFilter()); this.addFilter(new CheckFilter()); } +} +`, + }); + assert.deepEqual(await compiled.run({}), { later: true }); + assert.match(compiled.code, /await cup_01_later_filter\(data\);/); + await assert.rejects(compiled.run({ bad: 1 }), { name: "RangeError", message: "bad input" }); +}); + +test("an import of a step file is kept, relative to the compiled file", async () => { + const compiled = await compileIn({ + "rates.ts": `export const RATES: Record = { gold: 0.2 };\nexport function round(value: number): number { return Math.round(value * 100) / 100; }\n`, + "rate-filter.ts": `import type { Payload } from "codeupipe"; +import { RATES, round as roundMoney } from "./rates.ts"; +export class RateFilter { + call(payload: Payload): Payload { + const rate = RATES[payload.get("tier", "")] ?? 0; + return payload.insert("price", roundMoney(payload.get("price", 0) * (1 - rate))); + } +} +`, + "pipeline.ts": PIPELINE_IMPORT + `import { RateFilter } from "./rate-filter.ts"; +export class MainPipeline extends Pipeline { constructor() { super(); this.addFilter(new RateFilter()); } } +`, + }); + assert.match(compiled.code, /^import \{ RATES \} from "\.\/rates\.ts";\nimport \{ round as roundMoney \} from "\.\/rates\.ts";$/m); + assert.deepEqual(await compiled.run({ tier: "gold", price: 10 }), { tier: "gold", price: 8 }); +}); diff --git a/packages/codeupipe-compile-ts/tests/refusals.test.ts b/packages/codeupipe-compile-ts/tests/refusals.test.ts new file mode 100644 index 0000000..b6f01e0 --- /dev/null +++ b/packages/codeupipe-compile-ts/tests/refusals.test.ts @@ -0,0 +1,92 @@ +import assert from "node:assert/strict"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { test } from "node:test"; + +import { CompileRefusal, compile } from "../src/index.ts"; +import { PIPELINE_IMPORT, project } from "./compile-in.ts"; + +/** Compile a pipeline of one step, and return the refusal. */ +async function refusal(step: string, add: string, extra: Record = {}): Promise { + const root = project({ + "step.ts": step, + "pipeline.ts": PIPELINE_IMPORT + `import { Step } from "./step.ts"; +export class MainPipeline extends Pipeline { constructor() { super(); ${add} } } +`, + ...extra, + }); + const outFile = join(root, "compiled.ts"); + try { + await compile(join(root, "pipeline.ts"), "MainPipeline", { outFile }); + } catch (error) { + assert.ok(error instanceof CompileRefusal, String(error)); + assert.equal(existsSync(outFile), false, "nothing is written when the pipeline is refused"); + return error; + } + throw new Error("the pipeline was not refused"); +} + +const FILTER = "this.addFilter(new Step());"; +const filter = (body: string) => `export class Step { call(payload) { ${body} } }\n`; + +const CASES: Array<[string, string, string, RegExp]> = [ + ["a Hook", filter("return payload;"), FILTER + " this.useHook(new Step());", /the pipeline has a Hook/], + ["timing", filter("return payload;"), FILTER + " this.observe();", /timing or lineage/], + ["other code in the constructor", filter("return payload;"), FILTER + " console.log(1);", /does not add a step/], + ["a step that is not made in the call", filter("return payload;"), "const step = new Step(); this.addFilter(step);", /does not add a step/], + ["an argument that is not a literal", `export class Step { key; constructor(key) { this.key = key; } call(payload) { return payload; } }\n`, + "this.addFilter(new Step(process.env.KEY));", /is not a literal/], + ["a setting that the constructor computes", `export class Step { key; constructor(key) { this.key = key + "!"; } call(payload) { return payload; } }\n`, + `this.addFilter(new Step("a"));`, /computes the setting `key`/], + ["a write to this", `export class Step { count = 0; call(payload) { this.count += 1; return payload; } }\n`, FILTER, /assigns to `this\.count`/], + ["a helper method", `export class Step { help() { return 1; } call(payload) { return payload.insert("a", this.help()); } }\n`, FILTER, + /`this\.help`, which is not a property/], + ["a class that extends another", `class Base {}\nexport class Step extends Base { call(payload) { return payload; } }\n`, FILTER, /extends another class/], + ["a return of another Payload", `import { Payload } from "codeupipe";\nexport class Step { call(payload) { return new Payload({}); } }\n`, FILTER, + /returns something that is not the Payload/], + ["no return", filter(`payload.insert("a", 1);`), FILTER, /does not return the Payload/], + ["copy", filter(`const mine = payload.copy(); return payload;`), FILTER, /`payload\.copy` in a form that has no plain code/], + ["the trace id", filter(`payload.insert("t", payload.traceId); return payload;`), FILTER, /uses `payload\.traceId`/], + ["the Payload under another name", filter(`const other = payload; return payload;`), FILTER, /gives it another name/], + ["the Payload given to other code", filter(`console.log(payload); return payload;`), FILTER, /gives the Payload to other code/], + ["a merge of a Payload from elsewhere", filter(`return payload.merge(globalThis.other);`), FILTER, /merges a Payload that it did not make/], + ["a name beside the class", `const RATE = 0.1;\nexport class Step { call(payload) { return payload.insert("r", RATE); } }\n`, FILTER, + /Move `RATE` to its own file and import it/], + ["a value from codeupipe", `import { State } from "codeupipe";\nexport class Step { call(payload) { return payload.insert("s", new State()); } }\n`, FILTER, + /uses `State` from codeupipe/], + ["a name that starts with cup_", filter(`const cup_x = 1; return payload;`), FILTER, /starts with cup_/], + ["a Tap that writes", `export class Step { observe(payload) { payload.insert("a", 1); } }\n`, "this.addTap(new Step());", /observe writes to the Payload/], + ["a Predicate that writes", `export class Step { test(payload) { payload.remove("a"); return true; } call(payload) { return payload; } }\n`, + "this.addValve(new Step(), new Step());", /test writes to the Payload/], + ["an async Predicate", `export class Step { async test(payload) { return true; } call(payload) { return payload; } }\n`, + "this.addValve(new Step(), new Step());", /test is async/], + ["a class with no call", `export class Step { observe(payload) {} }\n`, FILTER, /has no call\(payload\) method/], +]; + +for (const [name, step, add, message] of CASES) { + test(`refused: ${name}`, async () => { + const error = await refusal(step, add); + assert.match(error.message, message); + assert.ok(error.hint.length > 0); + }); +} + +test("refused: a step from a package", async () => { + const root = project({ + "pipeline.ts": PIPELINE_IMPORT + `import { Step } from "some-package"; +export class MainPipeline extends Pipeline { constructor() { super(); this.addFilter(new Step()); } } +`, + }); + await assert.rejects(compile(join(root, "pipeline.ts"), "MainPipeline"), /comes from the package "some-package"/); +}); + +test("refused: a class that is not a Pipeline, and a pipeline that is nested in itself", async () => { + const root = project({ + "pipeline.ts": PIPELINE_IMPORT + `export class Plain {} +export class LoopPipeline extends Pipeline { constructor() { super(); this.addPipeline(new LoopPipeline()); } } +`, + }); + await assert.rejects(compile(join(root, "pipeline.ts"), "Plain"), /does not extend Pipeline/); + await assert.rejects(compile(join(root, "pipeline.ts"), "LoopPipeline"), /nested in itself/); + await assert.rejects(compile(join(root, "pipeline.ts"), "Missing"), /`Missing` is not a class/); +}); diff --git a/packages/codeupipe-compile-ts/tsconfig.build.json b/packages/codeupipe-compile-ts/tsconfig.build.json new file mode 100644 index 0000000..7f9bdff --- /dev/null +++ b/packages/codeupipe-compile-ts/tsconfig.build.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "rootDir": "src", + "outDir": "dist", + "declaration": true + }, + "include": ["src"] +} diff --git a/packages/codeupipe-compile-ts/tsconfig.json b/packages/codeupipe-compile-ts/tsconfig.json new file mode 100644 index 0000000..f419b27 --- /dev/null +++ b/packages/codeupipe-compile-ts/tsconfig.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "types": ["node"], + "strict": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "erasableSyntaxOnly": true, + "verbatimModuleSyntax": true, + "allowImportingTsExtensions": true, + "rewriteRelativeImportExtensions": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["src", "tests", "../../conformance/runners/typescript"] +} diff --git a/packages/codeupipe-lint-ts/package.json b/packages/codeupipe-lint-ts/package.json index 6495f59..f36a28a 100644 --- a/packages/codeupipe-lint-ts/package.json +++ b/packages/codeupipe-lint-ts/package.json @@ -27,7 +27,7 @@ "typecheck": "tsc --noEmit", "prebuild": "npm run core", "build": "tsc -p tsconfig.build.json", - "lint": "node src/cli.ts src ../codeupipe-ts/src ../codeupipe-ts/examples ../../conformance/runners/typescript" + "lint": "node src/cli.ts src ../codeupipe-ts/src ../codeupipe-ts/examples ../codeupipe-compile-ts/src ../../conformance/runners/typescript" }, "dependencies": { "codeupipe": "file:../codeupipe-ts", diff --git a/packages/codeupipe-lint-ts/tests/cli.test.ts b/packages/codeupipe-lint-ts/tests/cli.test.ts index b610724..20d530c 100644 --- a/packages/codeupipe-lint-ts/tests/cli.test.ts +++ b/packages/codeupipe-lint-ts/tests/cli.test.ts @@ -73,9 +73,10 @@ test("node_modules and dist are not read", async () => { assert.deepEqual(out.found, []); }); -test("the linter is clean on its own source and on the core package", () => { +test("the linter is clean on the TypeScript code of this repository", () => { const here = join(import.meta.dirname, ".."); - const done = run(here, "src", "../codeupipe-ts/src", "../codeupipe-ts/examples", "../../conformance/runners/typescript"); + const done = run(here, "src", "../codeupipe-ts/src", "../codeupipe-ts/examples", "../codeupipe-compile-ts/src", + "../../conformance/runners/typescript"); assert.equal(done.out.trim(), "No findings."); assert.equal(done.code, 0); }); diff --git a/packages/codeupipe-ts/README.md b/packages/codeupipe-ts/README.md index 49078fd..cd80dfc 100644 --- a/packages/codeupipe-ts/README.md +++ b/packages/codeupipe-ts/README.md @@ -121,8 +121,10 @@ override the methods that you need. method, so `addFilter` refuses a function by its type. - **Two runs of one pipeline at the same time** each write to their own State. `pipeline.state` is the State of the run that started last. -- **There is no export command.** Python has one. The linter is the - package [codeupipe-lint-ts](../codeupipe-lint-ts/). +- **The linter and the compiler are separate packages**, because they + need a parser: [codeupipe-lint-ts](../codeupipe-lint-ts/) and + [codeupipe-compile-ts](../codeupipe-compile-ts/). In Python the export is + in the core package. ## Labels and minified code @@ -144,7 +146,7 @@ Node 22.18 or later runs the `.ts` files as they are. ```bash npm install # the TypeScript compiler, the Node types and esbuild, for development only npm test # the tests -npm run typecheck # the type check of the source, tests, example and conformance runner +npm run typecheck # the type check of the source, tests and example npm run build # JavaScript and type files in dist/ node examples/pricing/main.ts python ../../conformance/run.py typescript:run diff --git a/packages/codeupipe-ts/tsconfig.json b/packages/codeupipe-ts/tsconfig.json index de9cc28..ad97e01 100644 --- a/packages/codeupipe-ts/tsconfig.json +++ b/packages/codeupipe-ts/tsconfig.json @@ -15,5 +15,5 @@ "skipLibCheck": true, "noEmit": true }, - "include": ["src", "tests", "examples", "../../conformance/runners/typescript"] + "include": ["src", "tests", "examples"] } diff --git a/tests/conformance/test_typescript_target.py b/tests/conformance/test_typescript_target.py index 02f3021..47c1850 100644 --- a/tests/conformance/test_typescript_target.py +++ b/tests/conformance/test_typescript_target.py @@ -29,6 +29,19 @@ def test_typescript_passes_every_case(tmp_path): assert not_passed == {}, json.dumps(not_passed, indent=2) +@pytest.mark.skipif("typescript:static" not in conformance_run.default_targets(), + reason="needs the compile package to be installed") +def test_the_compiled_pipeline_passes_every_case_on_what_it_can_show(): + cases = conformance_run.load_cases() + results, error = conformance_run.run_target( + "typescript:static", conformance_run.TARGETS["typescript:static"], cases) + assert error is None, error + verdicts = {case["id"]: conformance_run.judge(case, results.get(case["id"])) for case in cases} + wrong = {case_id: verdict for case_id, verdict in verdicts.items() if verdict[0] not in ("pass", "partial")} + assert wrong == {}, json.dumps(wrong, indent=2) + assert sum(1 for verdict in verdicts.values() if verdict[0] == "pass") == 11 + + def test_a_target_that_cannot_run_is_a_failure(monkeypatch, tmp_path): monkeypatch.setattr(conformance_run, "RESULTS", tmp_path) monkeypatch.setitem(conformance_run.TARGETS, "broken:run",