From 0dfb9d9f8006af3d779f69c8ca4fb61a261898ab Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Tue, 17 Nov 2026 03:30:30 -0500 Subject: [PATCH 01/34] bench: the benchmark CLI moves to bench/ as leanvm-bench, its harness crate is bench_harness bins/leanvm (package leanvm-cli, binary leanvm) becomes bench/ (package and binary leanvm-bench, `cargo leanvm-bench`), with bench.sh beside it, so that bins/ holds only what users run. The harness library crates/bench becomes crates/bench_harness, so that "bench" names the folder alone. Co-Authored-By: Claude Opus 5.5 --- .cargo/config.toml | 2 +- AGENTS.md | 22 +++++++++---------- Cargo.lock | 16 +++++++------- Cargo.toml | 6 ++--- README.md | 10 ++++----- {bins/leanvm => bench}/Cargo.toml | 10 ++++----- {bins/leanvm => bench}/bench.sh | 10 ++++----- {bins/leanvm => bench}/src/benchmark.rs | 14 ++++++------ {bins/leanvm => bench}/src/fibonacci.rs | 10 ++++----- {bins/leanvm => bench}/src/main.rs | 8 +++---- {bins/leanvm => bench}/src/report.rs | 2 +- .../tests/identical_proofs.rs | 5 +---- crates/{bench => bench_harness}/Cargo.toml | 2 +- .../{bench => bench_harness}/src/allocator.rs | 0 crates/{bench => bench_harness}/src/lib.rs | 0 crates/{bench => bench_harness}/src/trace.rs | 0 .../experimental_prover/keccak_gkr/Cargo.toml | 4 ++-- .../keccak_gkr/benches/hash_batch.rs | 6 ++--- .../keccak_gkr/src/batch_bench.rs | 4 ++-- .../lean_vm_core/tests/verifiers/main.rs | 2 +- crates/experimental_prover/pcs/Cargo.toml | 2 +- .../pcs/benches/throughput.rs | 10 ++++----- crates/keccak256/Cargo.toml | 2 +- crates/keccak256/benches/hash_throughput.rs | 4 ++-- crates/stable_prover/keccak_gkr/Cargo.toml | 4 ++-- .../keccak_gkr/benches/hash_batch.rs | 6 ++--- .../keccak_gkr/src/batch_bench.rs | 4 ++-- crates/stable_prover/pcs/Cargo.toml | 2 +- .../stable_prover/pcs/benches/throughput.rs | 10 ++++----- 29 files changed, 87 insertions(+), 90 deletions(-) rename {bins/leanvm => bench}/Cargo.toml (68%) rename {bins/leanvm => bench}/bench.sh (84%) rename {bins/leanvm => bench}/src/benchmark.rs (96%) rename {bins/leanvm => bench}/src/fibonacci.rs (94%) rename {bins/leanvm => bench}/src/main.rs (95%) rename {bins/leanvm => bench}/src/report.rs (95%) rename {bins/leanvm => bench}/tests/identical_proofs.rs (98%) rename crates/{bench => bench_harness}/Cargo.toml (91%) rename crates/{bench => bench_harness}/src/allocator.rs (100%) rename crates/{bench => bench_harness}/src/lib.rs (100%) rename crates/{bench => bench_harness}/src/trace.rs (100%) diff --git a/.cargo/config.toml b/.cargo/config.toml index 6d36286517..6f6806a2f2 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -2,7 +2,7 @@ testall = ["test", "--release", "--workspace"] clippyall = ["clippy", "--release", "--workspace", "--all-targets", "--", "-D", "warnings"] docall = ["doc", "--release", "--workspace", "--no-deps"] -leanvm = ["run", "--release", "-p", "leanvm-cli", "--"] +leanvm-bench = ["run", "--release", "-p", "leanvm-bench", "--"] [build] rustdocflags = ["-D", "warnings"] diff --git a/AGENTS.md b/AGENTS.md index cb61c980ed..23e487eff5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,12 +18,12 @@ The fields are `K = GF(2)[x]/(x^64 + x^4 + x^3 + x + 1)` and `E = K[y]/(y^2 + x* ## Layout -The root `Cargo.toml` is workspace-only: libraries in `crates/`, the CLI in `bins/`. The prover exists twice, in two trees that share no prover crate: +The root `Cargo.toml` is workspace-only: libraries in `crates/`, the benchmark CLI in `bench/`. The prover exists twice, in two trees that share no prover crate: - `crates/stable_prover/` (packages `stable_*`): the reference prover, written to be read: plain, safe code, as short as the protocol allows, and slower than experimental. Architecture-specific code lives in one SIMD layer, `primitives::field::lanes` (eight field elements at a time, `F128x8`, `F64x8`, `Acc8`, `Wide8`, and the byte maps and transposes over 64 bytes, each with one portable body that defines it and NEON, AVX2+VPCLMULQDQ and AVX-512+VPCLMULQDQ arms and the portable one), and in the shared `keccak256`'s multi-way permutation; the hot loops are written over them and nothing else in stable has `core::arch` or `cfg(target_feature)`. No uninitialized buffers, no non-temporal stores, no cache-tiled pass plans, no per-machine tuning. It keeps the whole level-0 codeword and every Merkle level, so its peak memory at the largest stacks is a multiple of experimental's on x86. An optimization enters it only where leaving it out costs a measurable share of a proof and its simple form stays short. - `crates/experimental_prover/` (packages `experimental_*`): the optimized prover, where performance work lands first. The conventions below about NEON, AVX-512, streaming stores and the like are about this tree. -Both prove the same protocol and give the same proof bytes, pinned by `bins/leanvm/tests/identical_proofs.rs`; a protocol change lands in both trees, the three verifiers and `formal/`, while an optimization lands in experimental alone. What is not prover code exists once (table below), the guest (`guests/lean_ethereum.py`) and the aggregation included. It depends on the stable tree only where it needs the protocol's structure or a primitive (`aggregation`, below; `lean_da`, its NTT), importing rather than restating them, and proves only through `isa::Prover`, which each tree implements once beside `cpu::prove` (`lean_vm_core::StableProver`, `lean_vm_core::ExperimentalProver`): proving a program, its thread pool, and the deferred claims' batching kernels (`combine_rows`, `dot_rows`). `formal/`'s Rust helpers that need a prover tree build against stable, but for `whir-fold-table`, which exports the WHIR security search that only experimental's `pcs` keeps (stable reads the verifier's table). Within a tree, crates name each other through Cargo renames (`pcs = { package = "stable_pcs", path = "../pcs" }`), so sources say `pcs::`, and the shared `fiat_shamir_prover` likewise as `fiat_shamir`; an integration test reaches its own crate with `extern crate stable_pcs as pcs;`, and a doctest names it in full. The shared crates, leaves first: +Both prove the same protocol and give the same proof bytes, pinned by `bench/tests/identical_proofs.rs`; a protocol change lands in both trees, the three verifiers and `formal/`, while an optimization lands in experimental alone. What is not prover code exists once (table below), the guest (`guests/lean_ethereum.py`) and the aggregation included. It depends on the stable tree only where it needs the protocol's structure or a primitive (`aggregation`, below; `lean_da`, its NTT), importing rather than restating them, and proves only through `isa::Prover`, which each tree implements once beside `cpu::prove` (`lean_vm_core::StableProver`, `lean_vm_core::ExperimentalProver`): proving a program, its thread pool, and the deferred claims' batching kernels (`combine_rows`, `dot_rows`). `formal/`'s Rust helpers that need a prover tree build against stable, but for `whir-fold-table`, which exports the WHIR security search that only experimental's `pcs` keeps (stable reads the verifier's table). Within a tree, crates name each other through Cargo renames (`pcs = { package = "stable_pcs", path = "../pcs" }`), so sources say `pcs::`, and the shared `fiat_shamir_prover` likewise as `fiat_shamir`; an integration test reaches its own crate with `extern crate stable_pcs as pcs;`, and a doctest names it in full. The shared crates, leaves first: | crate | role | | ----------------- | ---------------------------------------------------------------------- | @@ -34,7 +34,7 @@ Both prove the same protocol and give the same proof bytes, pinned by `bins/lean | `zkdsl_compiler` | zkDSL → ISA; its tests run the stable prover | | `lean_xmss` | leanXMSS over Keccak-256 | | `lean_sphincs` | leanSPHINCS over Keccak-256 | -| `bench` | benchmark harness (`Plan`, `Timing`, `--tracing`, Bencher output, the jemalloc allocator); never linked by a prover | +| `bench_harness` | benchmark harness (`Plan`, `Timing`, `--tracing`, Bencher output, the jemalloc allocator); never linked by a prover | | `lean_da` | additive Reed-Solomon blob encoding (the stable tree's NTT), commitments, membership vectors | | `aggregation` | recursive signature and DA aggregation, generic over the prover: the guest's constants and witness, and the entry points | | `lean_vm` | the public API, not generic: it proves with the stable tree, or with the `experimental_prover` feature the experimental one | @@ -51,7 +51,7 @@ Each tree holds, in dependency order, leaves first: | `keccak_gkr` | the prover of the VM's `KECCAK` argument | | `lean_vm_core` | arithmetization: tables, bus, constraints, `cpu::execute` and `cpu::prove` of an `isa::Program` and its witness streams; `cpu::verify` wraps `verifier`; the tree's `isa::Prover` | -Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The CLI is `bins/leanvm`, run with `cargo leanvm --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. +Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. ## Building and testing @@ -67,11 +67,11 @@ cargo fmt --all # max_width = 120 cargo machete --with-metadata # unused dependencies, dev ones included ``` -An x86-only `cfg` arm never compiles on an Apple machine, so type-check the other target before pushing anything `cfg`-gated (needs only `rustup target add x86_64-unknown-linux-gnu`; `bench` and the CLI are left out because jemalloc's build script wants a cross `gcc`): +An x86-only `cfg` arm never compiles on an Apple machine, so type-check the other target before pushing anything `cfg`-gated (needs only `rustup target add x86_64-unknown-linux-gnu`; `bench_harness` and the CLI are left out because jemalloc's build script wants a cross `gcc`): ```bash RUSTFLAGS="-C target-cpu=znver4" \ - cargo check --release --workspace --exclude leanvm-cli --exclude bench --target x86_64-unknown-linux-gnu + cargo check --release --workspace --exclude leanvm-bench --exclude bench_harness --target x86_64-unknown-linux-gnu ``` `RUSTFLAGS` replaces `.cargo/config.toml`'s `target-cpu=native`; a target-specific `CARGO_TARGET_*_RUSTFLAGS` would be merged with it and hand the Apple CPU to the x86 target. @@ -85,10 +85,10 @@ Where a proof's time goes is the `tracing` span tree, and only that: `--tracing` ## Benchmarking ```bash -cargo leanvm --prover experimental aggregate --xmss 900 --log-inv-rate 1 --repeat 3 -cargo leanvm --prover experimental aggregate --sphincs 410 --log-inv-rate 1 --repeat 3 -cargo leanvm --prover experimental recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2 --repeat 3 -cargo leanvm --prover stable aggregate --xmss 900 --log-inv-rate 1 --repeat 3 +cargo leanvm-bench --prover experimental aggregate --xmss 900 --log-inv-rate 1 --repeat 3 +cargo leanvm-bench --prover experimental aggregate --sphincs 410 --log-inv-rate 1 --repeat 3 +cargo leanvm-bench --prover experimental recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2 --repeat 3 +cargo leanvm-bench --prover stable aggregate --xmss 900 --log-inv-rate 1 --repeat 3 ``` `aggregate` takes a count per scheme (both default to zero) and `--blobs`; `recursion` likewise takes `--sphincs-per-leaf` and `--blobs-per-leaf`. Compare schemes and changes by the reported VM cycle counts. @@ -97,7 +97,7 @@ CI compares every PR with its base, never with a history: `bench.yml` runs both ## The allocator -The prover's buffers are plain `Vec`s and the libraries impose no allocator. A proof allocates and frees gigabytes, so the CLI and the benches install jemalloc with `dirty_decay_ms:-1` (`bench::Jemalloc`), which keeps freed pages mapped for the next proof at the cost of resident memory, and on Linux with `thp:always`, which backs its extents with transparent huge pages. Experimental skips a buffer's zero-fill with `primitives::uninit_vec` or `filled_vec` (`T: Plain`, every element written before it is read); stable never reads uninitialized memory, and writes a large buffer's first pass with the pool (`parallel::filled`, or `parallel::map_collect` when every element is computed anyway), since a zero-fill on one thread costs a whole-proof factor on many cores. Both drop large buffers where their last use ends. +The prover's buffers are plain `Vec`s and the libraries impose no allocator. A proof allocates and frees gigabytes, so the CLI and the benches install jemalloc with `dirty_decay_ms:-1` (`bench_harness::Jemalloc`), which keeps freed pages mapped for the next proof at the cost of resident memory, and on Linux with `thp:always`, which backs its extents with transparent huge pages. Experimental skips a buffer's zero-fill with `primitives::uninit_vec` or `filled_vec` (`T: Plain`, every element written before it is read); stable never reads uninitialized memory, and writes a large buffer's first pass with the pool (`parallel::filled`, or `parallel::map_collect` when every element is computed anyway), since a zero-fill on one thread costs a whole-proof factor on many cores. Both drop large buffers where their last use ends. ## The thread pool (`parallel`) diff --git a/Cargo.lock b/Cargo.lock index abba83434d..9d77bfadb4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -90,7 +90,7 @@ dependencies = [ ] [[package]] -name = "bench" +name = "bench_harness" version = "0.1.0" dependencies = [ "isa", @@ -210,7 +210,7 @@ dependencies = [ name = "experimental_keccak_gkr" version = "0.1.0" dependencies = [ - "bench", + "bench_harness", "experimental_keccak_gkr", "experimental_parallel", "experimental_pcs", @@ -246,7 +246,7 @@ dependencies = [ name = "experimental_pcs" version = "0.1.0" dependencies = [ - "bench", + "bench_harness", "experimental_parallel", "experimental_primitives", "fiat_shamir_prover", @@ -334,7 +334,7 @@ dependencies = [ name = "keccak256" version = "0.1.0" dependencies = [ - "bench", + "bench_harness", "sha3", "stable_parallel", "verifier", @@ -387,11 +387,11 @@ dependencies = [ ] [[package]] -name = "leanvm-cli" +name = "leanvm-bench" version = "0.1.0" dependencies = [ "aggregation", - "bench", + "bench_harness", "clap", "experimental_lean_vm_core", "isa", @@ -573,7 +573,7 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" name = "stable_keccak_gkr" version = "0.1.0" dependencies = [ - "bench", + "bench_harness", "fiat_shamir_prover", "isa", "stable_keccak_gkr", @@ -607,7 +607,7 @@ version = "0.1.0" name = "stable_pcs" version = "0.1.0" dependencies = [ - "bench", + "bench_harness", "fiat_shamir_prover", "isa", "stable_parallel", diff --git a/Cargo.toml b/Cargo.toml index f6cd3f37a7..1b9abe1b5a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [workspace] members = [ "crates/verifier", - "crates/bench", + "crates/bench_harness", "crates/keccak256", "crates/fiat_shamir_prover", "crates/isa", @@ -13,7 +13,7 @@ members = [ "crates/lean_vm", "crates/stable_prover/*", "crates/experimental_prover/*", - "bins/*", + "bench", ] resolver = "3" @@ -27,7 +27,7 @@ needless_range_loop = "allow" too_many_arguments = "allow" [workspace.dependencies] -bench = { path = "crates/bench" } +bench_harness = { path = "crates/bench_harness" } verifier = { path = "crates/verifier" } keccak256 = { path = "crates/keccak256" } fiat_shamir_prover = { path = "crates/fiat_shamir_prover" } diff --git a/README.md b/README.md index 0e5762fd0f..61b27dce6f 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ In the short term, we provide two Rust implementations of the prover, `stable` ( The leanXMSS parameters are specified in [leanXMSS.pdf](https://github.com/leanEthereum/leanVM/releases/download/doc-latest/leanXMSS.pdf), with a [security proof in Lean 4](formal/statement.tex). **This is not the definitive parameter set.** ```bash -cargo leanvm --prover experimental aggregate --xmss 1400 --log-inv-rate 2 --repeat 3 +cargo leanvm-bench --prover experimental aggregate --xmss 1400 --log-inv-rate 2 --repeat 3 ``` ``` @@ -87,7 +87,7 @@ aggregation, 1,400 leanXMSS signatures The leanSPHINCS parameters are specified in [leanSPHINCS.pdf](https://github.com/leanEthereum/leanVM/releases/download/doc-latest/leanSPHINCS.pdf), with a [security proof in Lean 4](formal/statement.tex). **This is not the definitive parameter set.** ```bash -cargo leanvm --prover experimental aggregate --sphincs 500 --log-inv-rate 1 --repeat 3 +cargo leanvm-bench --prover experimental aggregate --sphincs 500 --log-inv-rate 1 --repeat 3 ``` ``` @@ -103,7 +103,7 @@ aggregation, 500 leanSPHINCS signatures ### data availability ```bash -cargo leanvm --prover experimental aggregate --blobs 32 --log-inv-rate 1 --repeat 3 +cargo leanvm-bench --prover experimental aggregate --blobs 32 --log-inv-rate 1 --repeat 3 ``` ``` @@ -119,7 +119,7 @@ aggregation, 32 blobs ### recursion ```bash -cargo leanvm --prover experimental recursion --n 2 --xmss-per-leaf 1400 --leaf-log-inv-rate 2 --log-inv-rate 1 --repeat 3 +cargo leanvm-bench --prover experimental recursion --n 2 --xmss-per-leaf 1400 --leaf-log-inv-rate 2 --log-inv-rate 1 --repeat 3 ``` ``` @@ -154,7 +154,7 @@ Keccak GKR batch proving, 1,048,576 permutations, a state committed every 12 rou ### Fibonacci ```bash -cargo leanvm --prover experimental fibonacci --n 2000000 --log-inv-rate 1 --repeat 3 +cargo leanvm-bench --prover experimental fibonacci --n 2000000 --log-inv-rate 1 --repeat 3 ``` ``` diff --git a/bins/leanvm/Cargo.toml b/bench/Cargo.toml similarity index 68% rename from bins/leanvm/Cargo.toml rename to bench/Cargo.toml index 6a3666c993..c323fbb0c4 100644 --- a/bins/leanvm/Cargo.toml +++ b/bench/Cargo.toml @@ -1,11 +1,11 @@ [package] -name = "leanvm-cli" +name = "leanvm-bench" version.workspace = true edition.workspace = true publish = false [[bin]] -name = "leanvm" +name = "leanvm-bench" path = "src/main.rs" # A binary, nothing to document. doc = false @@ -21,9 +21,9 @@ lean_da.workspace = true zkdsl_compiler.workspace = true lean_xmss.workspace = true lean_sphincs.workspace = true -stable_lean_vm_core = { path = "../../crates/stable_prover/lean_vm_core" } -experimental_lean_vm_core = { path = "../../crates/experimental_prover/lean_vm_core" } -bench.workspace = true +stable_lean_vm_core = { path = "../crates/stable_prover/lean_vm_core" } +experimental_lean_vm_core = { path = "../crates/experimental_prover/lean_vm_core" } +bench_harness.workspace = true rand.workspace = true tracing.workspace = true clap = { version = "4", features = ["derive"] } diff --git a/bins/leanvm/bench.sh b/bench/bench.sh similarity index 84% rename from bins/leanvm/bench.sh rename to bench/bench.sh index d34c68f793..3695a71547 100755 --- a/bins/leanvm/bench.sh +++ b/bench/bench.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash # Runs leanXMSS aggregation and 16-to-1 recursion at 4 and 8 threads and at the default pool ("full"), and prints a table of # proving time and peak RSS per benchmark and thread count. -# Usage: [BENCH_THREADS="4 8 full"] bins/leanvm/bench.sh stable|experimental [log file] (default: .build/bench-.log) +# Usage: [BENCH_THREADS="4 8 full"] bench/bench.sh stable|experimental [log file] (default: .build/bench-.log) set -uo pipefail PROVER=${1:-} -case $PROVER in stable | experimental) ;; *) echo "usage: bins/leanvm/bench.sh stable|experimental [log file]" >&2; exit 2 ;; esac +case $PROVER in stable | experimental) ;; *) echo "usage: bench/bench.sh stable|experimental [log file]" >&2; exit 2 ;; esac cd "$(dirname "$0")/../.." mkdir -p .build @@ -29,15 +29,15 @@ run() { local label=$1 threads=$2; shift 2 local pool=(env LEANVM_NUM_THREADS="$threads") [ "$threads" = full ] && pool=(env -u LEANVM_NUM_THREADS) - echo "=== ${pool[*]:1} leanvm --prover $PROVER $* --cooldown 5 ===" | tee -a "$LOG" - "${pool[@]}" "${CARGO_TARGET_DIR:-target}/release/leanvm" --prover "$PROVER" "$@" --cooldown 5 2>&1 | tee "$OUT" | tee -a "$LOG" + echo "=== ${pool[*]:1} leanvm-bench --prover $PROVER $* --cooldown 5 ===" | tee -a "$LOG" + "${pool[@]}" "${CARGO_TARGET_DIR:-target}/release/leanvm-bench" --prover "$PROVER" "$@" --cooldown 5 2>&1 | tee "$OUT" | tee -a "$LOG" printf '%s\t%s\t%s\n' "$label" "$threads" "$(measure)" >>"$RESULTS" echo | tee -a "$LOG" } read -ra THREADS <<<"${BENCH_THREADS:-4 8 full}" -cargo build --release -p leanvm-cli || exit 1 +cargo build --release -p leanvm-bench || exit 1 for t in "${THREADS[@]}"; do run "aggregate, 1400 leanXMSS, rate 2" "$t" aggregate --xmss 1400 --log-inv-rate 2 --repeat 3 diff --git a/bins/leanvm/src/benchmark.rs b/bench/src/benchmark.rs similarity index 96% rename from bins/leanvm/src/benchmark.rs rename to bench/src/benchmark.rs index 93a88a662d..9b97906e9a 100644 --- a/bins/leanvm/src/benchmark.rs +++ b/bench/src/benchmark.rs @@ -3,7 +3,7 @@ //! signature counts and a blob count; recursion accepts these counts per leaf. use aggregation::{AggregateClaims, AggregateProof, OutputClaims, aggregate, signers_cache}; -use bench::{Plan, Timing}; +use bench_harness::{Plan, Timing}; use isa::{Prover, Stats, pretty_f64, pretty_integer}; use lean_xmss::{LeanXmssPublicKey, LeanXmssSignature}; use rand::{Rng, SeedableRng, rngs::StdRng}; @@ -79,7 +79,7 @@ fn describe(n_lean_xmss: usize, n_lean_sphincs: usize) -> String { /// Prove signatures and `n_blobs` blobs in one leaf, then verify it. /// /// Proving runs one discarded warmup pass followed by `plan.repeat` measured -/// passes; see [`bench`] for why the first pass is not +/// passes; see [`bench_harness`] for why the first pass is not /// representative and why the cooldown matters. pub fn run_aggregation( n_lean_xmss: usize, @@ -99,7 +99,7 @@ pub fn run_aggregation( // Only the final measured pass of each stage is traced: the tree describes the // proof the reported timings are about, instead of repeating itself per pass. let ((claims, sig, stats), prove_time) = plan.warm_then_measure(|last| { - let _quiet = (!last).then(bench::suppress_tracing); + let _quiet = (!last).then(bench_harness::suppress_tracing); aggregate::

( &[], raw_lean_xmss.clone(), @@ -111,7 +111,7 @@ pub fn run_aggregation( .expect("leaf aggregates") }); let (_, verify_time) = Plan::new(plan.repeat, 0).measure_quiet(|last| { - let _quiet = (!last).then(bench::suppress_tracing); + let _quiet = (!last).then(bench_harness::suppress_tracing); assert!(sig.verify(&claims), "the leaf aggregate verifies"); }); drop(trace_span); @@ -183,14 +183,14 @@ pub fn run_recursion( let children = leaves_from_subprocess(n); if enable_tracing { - bench::init_tracing(); + bench_harness::init_tracing(); } let ((claims, sig, stats), prove_time) = plan.warm_then_measure(|last| { - let _quiet = (!last).then(bench::suppress_tracing); + let _quiet = (!last).then(bench_harness::suppress_tracing); aggregate::

(&children, vec![], vec![], None, OutputClaims::All, log_inv_rate).expect("node aggregates") }); let (_, verify_time) = Plan::new(plan.repeat, 0).measure_quiet(|last| { - let _quiet = (!last).then(bench::suppress_tracing); + let _quiet = (!last).then(bench_harness::suppress_tracing); assert!(sig.verify(&claims), "the recursive aggregate verifies"); }); diff --git a/bins/leanvm/src/fibonacci.rs b/bench/src/fibonacci.rs similarity index 94% rename from bins/leanvm/src/fibonacci.rs rename to bench/src/fibonacci.rs index 56314d415d..55311dd401 100644 --- a/bins/leanvm/src/fibonacci.rs +++ b/bench/src/fibonacci.rs @@ -1,7 +1,7 @@ //! Fibonacci in the exponent: the demo benchmark (`buff[g^k] = g^{F(k)}`, //! recurrence `buff[i·g²] = buff[i·g] · buff[i]`). -use bench::Plan; +use bench_harness::Plan; use isa::{Prover, pretty_f64, pretty_integer}; use verifier::{E, K}; use zkdsl_compiler::{compile, parse}; @@ -9,7 +9,7 @@ use zkdsl_compiler::{compile, parse}; /// Prove and verify Fibonacci-in-the-exponent over a `HeapBuf` (an unrolled /// `mul_range` recurrence), binding `g^{F(n)}` as the public input. Prints the /// benchmark report. Proving runs one discarded warmup pass followed by -/// `plan.repeat` measured passes (see [`bench`]). +/// `plan.repeat` measured passes (see [`bench_harness`]). pub fn run_fibonacci(n: usize, log_inv_rate: usize, plan: Plan) { let trace_span = tracing::info_span!("Fibonacci", n, log_inv_rate).entered(); @@ -19,11 +19,11 @@ pub fn run_fibonacci(n: usize, log_inv_rate: usize, plan: Plan) { // Only the final measured pass of each stage is traced (see `run_aggregation`). let ((proof, stats), prove_time) = plan.warm_then_measure(|last| { - let _quiet = (!last).then(bench::suppress_tracing); + let _quiet = (!last).then(bench_harness::suppress_tracing); P::prove(&program, &[], pi, log_inv_rate).expect("the Fibonacci program proves") }); let (_, verify_time) = Plan::new(plan.repeat, 0).measure_quiet(|last| { - let _quiet = (!last).then(bench::suppress_tracing); + let _quiet = (!last).then(bench_harness::suppress_tracing); verifier::verify_execution(program.bytecode(), &public_input, &proof).unwrap() }); @@ -111,7 +111,7 @@ mod tests { super::run_fibonacci::( 200_000, stable_lean_vm_core::pcs::TEST_LOG_INV_RATE, - bench::Plan::default(), + bench_harness::Plan::default(), ); } } diff --git a/bins/leanvm/src/main.rs b/bench/src/main.rs similarity index 95% rename from bins/leanvm/src/main.rs rename to bench/src/main.rs index 3f1a30ebaf..72aa26c33a 100644 --- a/bins/leanvm/src/main.rs +++ b/bench/src/main.rs @@ -1,7 +1,7 @@ //! Benchmark CLI for signature and blob proofs, recursion, and the Fibonacci demo. #[global_allocator] -static ALLOCATOR: bench::Jemalloc = bench::Jemalloc; +static ALLOCATOR: bench_harness::Jemalloc = bench_harness::Jemalloc; use clap::{Parser, Subcommand}; use experimental_lean_vm_core::ExperimentalProver; @@ -14,9 +14,9 @@ mod report; /// Run the CLI's command on prover `P`. fn run(cli: Cli) { P::init(); - let plan = bench::Plan::new(cli.repeat, cli.cooldown); + let plan = bench_harness::Plan::new(cli.repeat, cli.cooldown); if cli.tracing && !matches!(&cli.command, Command::Recursion { .. }) { - bench::init_tracing(); + bench_harness::init_tracing(); } match cli.command { Command::Aggregate { xmss, sphincs, blobs } => { @@ -54,7 +54,7 @@ enum Prover { } #[derive(Parser)] -#[command(name = "leanvm")] +#[command(name = "leanvm-bench")] struct Cli { /// WHIR inverse-rate logarithm (1 through 4). #[arg( diff --git a/bins/leanvm/src/report.rs b/bench/src/report.rs similarity index 95% rename from bins/leanvm/src/report.rs rename to bench/src/report.rs index f56078bf04..77fd8c8420 100644 --- a/bins/leanvm/src/report.rs +++ b/bench/src/report.rs @@ -17,7 +17,7 @@ pub fn pow(x: usize) -> String { /// Peak resident set size, in GiB. pub fn peak_gib() -> String { - pretty_f64(bench::peak_rss_bytes() as f64 / (1u64 << 30) as f64) + pretty_f64(bench_harness::peak_rss_bytes() as f64 / (1u64 << 30) as f64) } /// One line of per-table instruction counts and shares, largest first, followed by memory, bytecode (`code_len` diff --git a/bins/leanvm/tests/identical_proofs.rs b/bench/tests/identical_proofs.rs similarity index 98% rename from bins/leanvm/tests/identical_proofs.rs rename to bench/tests/identical_proofs.rs index f8e9c9ccfd..c6c1ce417a 100644 --- a/bins/leanvm/tests/identical_proofs.rs +++ b/bench/tests/identical_proofs.rs @@ -76,10 +76,7 @@ fn prove( /// `# witness name:`) and with a wrong public input, through experimental's interpreter and hints too, at every rate. #[test] fn compiler_programs_prove_identically() { - let dir = concat!( - env!("CARGO_MANIFEST_DIR"), - "/../../crates/zkdsl_compiler/tests/programs" - ); + let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/../crates/zkdsl_compiler/tests/programs"); let mut paths: Vec<_> = std::fs::read_dir(dir) .unwrap() .map(|entry| entry.unwrap().path()) diff --git a/crates/bench/Cargo.toml b/crates/bench_harness/Cargo.toml similarity index 91% rename from crates/bench/Cargo.toml rename to crates/bench_harness/Cargo.toml index 8397ed252c..f94ff6b85e 100644 --- a/crates/bench/Cargo.toml +++ b/crates/bench_harness/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "bench" +name = "bench_harness" version.workspace = true edition.workspace = true publish = false diff --git a/crates/bench/src/allocator.rs b/crates/bench_harness/src/allocator.rs similarity index 100% rename from crates/bench/src/allocator.rs rename to crates/bench_harness/src/allocator.rs diff --git a/crates/bench/src/lib.rs b/crates/bench_harness/src/lib.rs similarity index 100% rename from crates/bench/src/lib.rs rename to crates/bench_harness/src/lib.rs diff --git a/crates/bench/src/trace.rs b/crates/bench_harness/src/trace.rs similarity index 100% rename from crates/bench/src/trace.rs rename to crates/bench_harness/src/trace.rs diff --git a/crates/experimental_prover/keccak_gkr/Cargo.toml b/crates/experimental_prover/keccak_gkr/Cargo.toml index 95e0522c38..cf4dd71f1a 100644 --- a/crates/experimental_prover/keccak_gkr/Cargo.toml +++ b/crates/experimental_prover/keccak_gkr/Cargo.toml @@ -13,13 +13,13 @@ primitives = { package = "experimental_primitives", path = "../primitives" } fiat_shamir = { package = "fiat_shamir_prover", path = "../../fiat_shamir_prover" } pcs = { package = "experimental_pcs", path = "../pcs" } tracing.workspace = true -bench = { workspace = true, optional = true } +bench_harness = { workspace = true, optional = true } isa = { workspace = true, optional = true } verifier.workspace = true [features] # The standalone batch benchmark (`batch_bench`), for the `hash_batch` target. -bench = ["dep:bench", "dep:isa", "primitives/test-util"] +bench = ["dep:bench_harness", "dep:isa", "primitives/test-util"] [dev-dependencies] keccak_gkr = { package = "experimental_keccak_gkr", path = ".", features = ["bench"] } diff --git a/crates/experimental_prover/keccak_gkr/benches/hash_batch.rs b/crates/experimental_prover/keccak_gkr/benches/hash_batch.rs index 456280f05b..3413850fd1 100644 --- a/crates/experimental_prover/keccak_gkr/benches/hash_batch.rs +++ b/crates/experimental_prover/keccak_gkr/benches/hash_batch.rs @@ -11,14 +11,14 @@ //! `BENCH_TRACING=1` prints the final pass's span tree (`RUST_LOG` adjusts it). extern crate experimental_keccak_gkr as keccak_gkr; -use bench::{Plan, env_usize}; +use bench_harness::{Plan, env_usize}; use pcs::whir::LOG_INV_RATE_0; #[global_allocator] -static ALLOCATOR: bench::Jemalloc = bench::Jemalloc; +static ALLOCATOR: bench_harness::Jemalloc = bench_harness::Jemalloc; fn main() { - bench::init_tracing_from_env(); + bench_harness::init_tracing_from_env(); let n = 1usize << env_usize("KECCAK_N_LOG", 13); let report = keccak_gkr::batch_bench::run(n, env_usize("KECCAK_DEPTH", 12), LOG_INV_RATE_0, Plan::from_env()); if std::env::args().any(|arg| arg == "--json") { diff --git a/crates/experimental_prover/keccak_gkr/src/batch_bench.rs b/crates/experimental_prover/keccak_gkr/src/batch_bench.rs index 6ba1754321..aef8356967 100644 --- a/crates/experimental_prover/keccak_gkr/src/batch_bench.rs +++ b/crates/experimental_prover/keccak_gkr/src/batch_bench.rs @@ -3,7 +3,7 @@ use std::time::Instant; -use bench::{Metric, Plan, Timing, bencher_json}; +use bench_harness::{Metric, Plan, Timing, bencher_json}; use isa::pretty_integer; use pcs::stack_open::StackClaim; use pcs::whir::{ProverConfig, ProverData, commit, config_for_rate}; @@ -82,7 +82,7 @@ pub fn run(n: usize, depth: usize, log_inv_rate: usize, plan: Plan) -> Report { let mut stages: [Timing; 6] = std::array::from_fn(|_| Timing::default()); let mut warm = true; let (proof, _) = plan.warm_then_measure(|final_pass| { - let _quiet = (!final_pass).then(bench::suppress_tracing); + let _quiet = (!final_pass).then(bench_harness::suppress_tracing); let (proof, secs) = prove_pass(); if !std::mem::take(&mut warm) { for (timing, s) in stages.iter_mut().zip(secs) { diff --git a/crates/experimental_prover/lean_vm_core/tests/verifiers/main.rs b/crates/experimental_prover/lean_vm_core/tests/verifiers/main.rs index 25e6aa7375..a3053ea546 100644 --- a/crates/experimental_prover/lean_vm_core/tests/verifiers/main.rs +++ b/crates/experimental_prover/lean_vm_core/tests/verifiers/main.rs @@ -1,5 +1,5 @@ //! Both verifiers' WHIR query table and grinds, pinned against the security search only this tree keeps. The verifier -//! itself is tested on stable's proofs, which `bins/leanvm/tests/identical_proofs.rs` pins to this tree's. +//! itself is tested on stable's proofs, which `bench/tests/identical_proofs.rs` pins to this tree's. extern crate experimental_lean_vm_core as lean_vm_core; mod whir_query_table; diff --git a/crates/experimental_prover/pcs/Cargo.toml b/crates/experimental_prover/pcs/Cargo.toml index b370248474..0b775a1f73 100644 --- a/crates/experimental_prover/pcs/Cargo.toml +++ b/crates/experimental_prover/pcs/Cargo.toml @@ -16,7 +16,7 @@ verifier.workspace = true [dev-dependencies] primitives = { package = "experimental_primitives", path = "../primitives", features = ["test-util"] } -bench.workspace = true +bench_harness.workspace = true isa.workspace = true [[bench]] diff --git a/crates/experimental_prover/pcs/benches/throughput.rs b/crates/experimental_prover/pcs/benches/throughput.rs index a95b3da63a..d0f3c7d10f 100644 --- a/crates/experimental_prover/pcs/benches/throughput.rs +++ b/crates/experimental_prover/pcs/benches/throughput.rs @@ -3,7 +3,7 @@ //! Commits a random witness of `2^PCS_LOG_N` GF(2^64) elements at //! inverse-rate `1/2^PCS_LOG_INV_RATE` and opens it through the stacked opener with one point //! claim, after the OOD sample, times each phase, and reports GiB/s over the committed data. The -//! passes follow [`bench::Plan::from_env`] (`BENCH_REPEAT`, `BENCH_COOLDOWN`). +//! passes follow [`bench_harness::Plan::from_env`] (`BENCH_REPEAT`, `BENCH_COOLDOWN`). //! //! ```text //! PCS_LOG_N number of variables = log2(witness length) [default 22] @@ -22,7 +22,7 @@ extern crate experimental_pcs as pcs; use std::hint::black_box; use std::time::Instant; -use bench::{Plan, Timing, env_usize}; +use bench_harness::{Plan, Timing, env_usize}; use isa::pretty_integer; use pcs::ood::ood_sample; use pcs::stack_open::{RingSwitchClaim, RingSwitchOpen, StackClaim, open_batch_mixed_whir_stacked}; @@ -34,10 +34,10 @@ use primitives::{ }; #[global_allocator] -static ALLOCATOR: bench::Jemalloc = bench::Jemalloc; +static ALLOCATOR: bench_harness::Jemalloc = bench_harness::Jemalloc; fn main() { - bench::init_tracing_from_env(); + bench_harness::init_tracing_from_env(); let log_n = env_usize("PCS_LOG_N", 22); let log_inv_rate = env_usize("PCS_LOG_INV_RATE", LOG_INV_RATE_0); @@ -79,7 +79,7 @@ fn main() { let (mut commit_t, mut open_t) = (Timing::default(), Timing::default()); plan.warm_then_measure(|last| { - let _quiet = (!last).then(bench::suppress_tracing); + let _quiet = (!last).then(bench_harness::suppress_tracing); let t = Instant::now(); let (cm, pd) = tracing::info_span!("Commit").in_scope(|| commit(&witness, log_n, pc.initial_k, log_inv_rate)); diff --git a/crates/keccak256/Cargo.toml b/crates/keccak256/Cargo.toml index 1a4d58bf61..e62b79ffee 100644 --- a/crates/keccak256/Cargo.toml +++ b/crates/keccak256/Cargo.toml @@ -9,7 +9,7 @@ workspace = true [dev-dependencies] sha3.workspace = true -bench.workspace = true +bench_harness.workspace = true parallel = { package = "stable_parallel", path = "../stable_prover/parallel" } verifier.workspace = true diff --git a/crates/keccak256/benches/hash_throughput.rs b/crates/keccak256/benches/hash_throughput.rs index 77b80e947d..befba1e999 100644 --- a/crates/keccak256/benches/hash_throughput.rs +++ b/crates/keccak256/benches/hash_throughput.rs @@ -1,6 +1,6 @@ //! Batched Keccak-256 throughput across the stable prover's thread pool, to 32-byte digests: 64-byte inputs (a //! Merkle node's two children) and 512-byte ones (a PCS leaf of 64 words, four permutations). The passes follow -//! [`bench::Plan::from_env`] (`BENCH_REPEAT`, `BENCH_COOLDOWN`). +//! [`bench_harness::Plan::from_env`] (`BENCH_REPEAT`, `BENCH_COOLDOWN`). //! //! ```text //! BENCH_REPEAT=5 cargo bench -p keccak256 --bench hash_throughput @@ -14,7 +14,7 @@ fn run() { let data: Vec = (0..TASKS * BYTES).map(|i| (i & 0xff) as u8).collect(); let mut out = vec![0u8; TASKS * k * 32]; - let (_, time) = bench::Plan::from_env().warm_then_measure(|_| { + let (_, time) = bench_harness::Plan::from_env().warm_then_measure(|_| { parallel::chunks_mut(&mut out, k * 32, |i, sub| { let d = &data[i * BYTES..][..BYTES]; for _ in 0..ITERS { diff --git a/crates/stable_prover/keccak_gkr/Cargo.toml b/crates/stable_prover/keccak_gkr/Cargo.toml index dc0a0a244f..d000631966 100644 --- a/crates/stable_prover/keccak_gkr/Cargo.toml +++ b/crates/stable_prover/keccak_gkr/Cargo.toml @@ -13,13 +13,13 @@ primitives = { package = "stable_primitives", path = "../primitives" } fiat_shamir = { package = "fiat_shamir_prover", path = "../../fiat_shamir_prover" } pcs = { package = "stable_pcs", path = "../pcs" } tracing.workspace = true -bench = { workspace = true, optional = true } +bench_harness = { workspace = true, optional = true } isa = { workspace = true, optional = true } verifier.workspace = true [features] # The standalone batch benchmark (`batch_bench`), for the `hash_batch` target. -bench = ["dep:bench", "dep:isa", "primitives/test-util"] +bench = ["dep:bench_harness", "dep:isa", "primitives/test-util"] [dev-dependencies] keccak_gkr = { package = "stable_keccak_gkr", path = ".", features = ["bench"] } diff --git a/crates/stable_prover/keccak_gkr/benches/hash_batch.rs b/crates/stable_prover/keccak_gkr/benches/hash_batch.rs index cb7be20407..664379a72e 100644 --- a/crates/stable_prover/keccak_gkr/benches/hash_batch.rs +++ b/crates/stable_prover/keccak_gkr/benches/hash_batch.rs @@ -11,14 +11,14 @@ //! `BENCH_TRACING=1` prints the final pass's span tree (`RUST_LOG` adjusts it). extern crate stable_keccak_gkr as keccak_gkr; -use bench::{Plan, env_usize}; +use bench_harness::{Plan, env_usize}; use verifier::MIN_LOG_INV_RATE; #[global_allocator] -static ALLOCATOR: bench::Jemalloc = bench::Jemalloc; +static ALLOCATOR: bench_harness::Jemalloc = bench_harness::Jemalloc; fn main() { - bench::init_tracing_from_env(); + bench_harness::init_tracing_from_env(); let n = 1usize << env_usize("KECCAK_N_LOG", 13); let report = keccak_gkr::batch_bench::run(n, env_usize("KECCAK_DEPTH", 12), MIN_LOG_INV_RATE, Plan::from_env()); if std::env::args().any(|arg| arg == "--json") { diff --git a/crates/stable_prover/keccak_gkr/src/batch_bench.rs b/crates/stable_prover/keccak_gkr/src/batch_bench.rs index d930461fc2..4c38000cec 100644 --- a/crates/stable_prover/keccak_gkr/src/batch_bench.rs +++ b/crates/stable_prover/keccak_gkr/src/batch_bench.rs @@ -1,7 +1,7 @@ //! The standalone batch benchmark of the `hash_batch` bench target and the tests: witness, commitment, reduction and //! opening of `n` random permutations, then verification. Where a pass's time goes is its span tree. -use bench::{Metric, Plan, Timing, bencher_json}; +use bench_harness::{Metric, Plan, Timing, bencher_json}; use isa::pretty_integer; use pcs::Committed; use primitives::field::{F64, F128Unreduced}; @@ -32,7 +32,7 @@ pub fn run(n: usize, depth: usize, log_inv_rate: usize, plan: Plan) -> Report { let inputs: Vec<[u64; STATE_LANES]> = (0..n).map(|_| std::array::from_fn(|_| rng.next_u64())).collect(); let (proof, prove) = plan.warm_then_measure(|final_pass| { - let _quiet = (!final_pass).then(bench::suppress_tracing); + let _quiet = (!final_pass).then(bench_harness::suppress_tracing); let _span = tracing::info_span!("Keccak GKR prove", n).entered(); let mut committed = vec![F64(0); 1 << shape.stack_log()]; let witness = tracing::info_span!("Witness").in_scope(|| Witness::generate(shape, &inputs, &mut committed)); diff --git a/crates/stable_prover/pcs/Cargo.toml b/crates/stable_prover/pcs/Cargo.toml index 27c6a36b1d..b55f61fff1 100644 --- a/crates/stable_prover/pcs/Cargo.toml +++ b/crates/stable_prover/pcs/Cargo.toml @@ -16,7 +16,7 @@ verifier.workspace = true [dev-dependencies] primitives = { package = "stable_primitives", path = "../primitives", features = ["test-util"] } -bench.workspace = true +bench_harness.workspace = true isa.workspace = true [[bench]] diff --git a/crates/stable_prover/pcs/benches/throughput.rs b/crates/stable_prover/pcs/benches/throughput.rs index 2f0e615ec2..c4f463ffd3 100644 --- a/crates/stable_prover/pcs/benches/throughput.rs +++ b/crates/stable_prover/pcs/benches/throughput.rs @@ -1,6 +1,6 @@ //! PCS throughput: commits a random witness of `2^PCS_LOG_N` `K` words at inverse rate `2^PCS_LOG_INV_RATE`, with its //! OOD sample, opens it, times each phase, and reports GiB/s over the committed data. The passes follow -//! [`bench::Plan::from_env`] (`BENCH_REPEAT`, `BENCH_COOLDOWN`). +//! [`bench_harness::Plan::from_env`] (`BENCH_REPEAT`, `BENCH_COOLDOWN`). //! //! ```text //! PCS_LOG_N number of variables = log2(witness length) [default 22] @@ -18,17 +18,17 @@ extern crate stable_pcs as pcs; use std::hint::black_box; use std::time::Instant; -use bench::{Plan, Timing, env_usize}; +use bench_harness::{Plan, Timing, env_usize}; use isa::pretty_integer; use pcs::{RingSwitchOpen, StackClaim, commit, open}; use primitives::{field::F64, test_rng::Rng}; use verifier::MIN_LOG_INV_RATE; #[global_allocator] -static ALLOCATOR: bench::Jemalloc = bench::Jemalloc; +static ALLOCATOR: bench_harness::Jemalloc = bench_harness::Jemalloc; fn main() { - bench::init_tracing_from_env(); + bench_harness::init_tracing_from_env(); let log_n = env_usize("PCS_LOG_N", 22); let log_inv_rate = env_usize("PCS_LOG_INV_RATE", MIN_LOG_INV_RATE); @@ -60,7 +60,7 @@ fn main() { let (mut commit_t, mut open_t) = (Timing::default(), Timing::default()); plan.warm_then_measure(|last| { - let _quiet = (!last).then(bench::suppress_tracing); + let _quiet = (!last).then(bench_harness::suppress_tracing); let mut ps = primitives::transcript::ProverState::from_label(b"pcs-throughput"); let t = Instant::now(); From 4ac4144e28832a2558747442d89050e5bd3f7a3a Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Tue, 17 Nov 2026 06:59:42 -0500 Subject: [PATCH 02/34] thread count: set in code, LEANVM_NUM_THREADS removed `lean_vm::setup(threads)` and leanvm-bench's `--threads` fix the prover's pool size before it starts (0: one per core), through `isa::Prover::set_num_threads`, which each tree's pool implements and which panics if the pool already runs with another count. The pools read no environment variable any more; bench.sh passes `--threads`. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 3 +- bench/bench.sh | 8 ++-- bench/src/main.rs | 7 +++ .../lean_vm_core/src/prover.rs | 4 ++ .../experimental_prover/parallel/src/lib.rs | 2 +- .../parallel/src/topology.rs | 44 ++++++++++--------- crates/isa/src/prover.rs | 3 ++ crates/lean_vm/src/lib.rs | 13 ++++-- crates/lean_vm/tests/api.rs | 2 +- .../stable_prover/lean_vm_core/src/prover.rs | 4 ++ crates/stable_prover/parallel/src/lib.rs | 38 ++++++++++------ 11 files changed, 83 insertions(+), 45 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 23e487eff5..de16fa5066 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,7 +105,7 @@ No rayon. Every parallel site is "N independent items, each writing its own disj - Nested dispatch panics, because it would deadlock the dispatch lock. Each pool only sees its own nesting, so a task of either pool must not dispatch on either: inside a `P::map_collect` task, the shared crates call no dispatching helper of the stable tree. - One experimental site runs beside the pool: `keccak::while_executing` spawns half the pool's count of plain threads while a program executes, beside the pool's dispatches of a signature loop's segments and of their Keccak blocks, and gone when it ends. Stable builds the Keccak witness after execution, in the pool. -- `LEANVM_NUM_THREADS` sets the worker count, and `1` is strictly sequential. In experimental, workers fill the performance cores before the efficiency cores, and the default is one worker per logical CPU (on Linux, per physical core the process may use), holding back one performance worker when efficiency workers exist. In stable the default is `available_parallelism()`, halved where Linux reports SMT active. +- The thread count is set in code, before a pool starts (`isa::Prover::set_num_threads`, which `lean_vm::setup` and the CLI's `--threads` call); `1` is strictly sequential. In experimental, workers fill the performance cores before the efficiency cores, and the default is one worker per logical CPU (on Linux, per physical core the process may use), holding back one performance worker when efficiency workers exist. In stable the default is `available_parallelism()`, halved where Linux reports SMT active. ## Three verifiers, one protocol @@ -151,7 +151,6 @@ In the recursion program, the prover sends the verifier advice, called hints. Hi | var | effect | | ------------------------------------------------------------------------------ | ----------------------------------------------------- | -| `LEANVM_NUM_THREADS` | worker count; `1` = sequential | | `BENCH_TRACING`, `BENCH_REPEAT`, `BENCH_COOLDOWN` | `--tracing`, `--repeat`, `--cooldown` for the benches | | `KECCAK_N_LOG`, `KECCAK_DEPTH` | keccak_gkr batch size, rounds between committed states | | `PCS_LOG_N`, `PCS_LOG_INV_RATE`, `PCS_POINT_CLAIMS`, `PCS_RING_CLAIMS` | PCS throughput bench: size, rate, claims | diff --git a/bench/bench.sh b/bench/bench.sh index 3695a71547..94ac33a34f 100755 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -27,10 +27,10 @@ measure() { run() { local label=$1 threads=$2; shift 2 - local pool=(env LEANVM_NUM_THREADS="$threads") - [ "$threads" = full ] && pool=(env -u LEANVM_NUM_THREADS) - echo "=== ${pool[*]:1} leanvm-bench --prover $PROVER $* --cooldown 5 ===" | tee -a "$LOG" - "${pool[@]}" "${CARGO_TARGET_DIR:-target}/release/leanvm-bench" --prover "$PROVER" "$@" --cooldown 5 2>&1 | tee "$OUT" | tee -a "$LOG" + local count=$threads + [ "$threads" = full ] && count=0 + echo "=== leanvm-bench --prover $PROVER $* --threads $count --cooldown 5 ===" | tee -a "$LOG" + "${CARGO_TARGET_DIR:-target}/release/leanvm-bench" --prover "$PROVER" "$@" --threads "$count" --cooldown 5 2>&1 | tee "$OUT" | tee -a "$LOG" printf '%s\t%s\t%s\n' "$label" "$threads" "$(measure)" >>"$RESULTS" echo | tee -a "$LOG" } diff --git a/bench/src/main.rs b/bench/src/main.rs index 72aa26c33a..6fd3a46e43 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -13,6 +13,9 @@ mod report; /// Run the CLI's command on prover `P`. fn run(cli: Cli) { + if cli.threads > 0 { + P::set_num_threads(cli.threads); + } P::init(); let plan = bench_harness::Plan::new(cli.repeat, cli.cooldown); if cli.tracing && !matches!(&cli.command, Command::Recursion { .. }) { @@ -78,6 +81,10 @@ struct Cli { )] repeat: usize, + /// Thread count (0: one per core). + #[arg(long, global = true, default_value_t = 0)] + threads: usize, + /// Idle seconds before each measured pass. #[arg(long, global = true, default_value_t = 2)] cooldown: u64, diff --git a/crates/experimental_prover/lean_vm_core/src/prover.rs b/crates/experimental_prover/lean_vm_core/src/prover.rs index 4ee0518578..d34f4e99d6 100644 --- a/crates/experimental_prover/lean_vm_core/src/prover.rs +++ b/crates/experimental_prover/lean_vm_core/src/prover.rs @@ -16,6 +16,10 @@ impl isa::Prover for ExperimentalProver { parallel::init(); } + fn set_num_threads(n: usize) { + parallel::set_num_threads(n) + } + fn prove( program: &isa::Program, witness: &[(String, Vec>)], diff --git a/crates/experimental_prover/parallel/src/lib.rs b/crates/experimental_prover/parallel/src/lib.rs index c71df8a480..9abfa14d31 100644 --- a/crates/experimental_prover/parallel/src/lib.rs +++ b/crates/experimental_prover/parallel/src/lib.rs @@ -14,9 +14,9 @@ use std::thread::Thread; mod topology; -pub use topology::num_threads; pub(crate) use topology::topology; use topology::{Qos, set_qos}; +pub use topology::{num_threads, set_num_threads}; /// Idle spins before a worker parks: long enough to stay hot across back-to-back /// dispatches, short enough to yield the core during a sequential stretch. diff --git a/crates/experimental_prover/parallel/src/topology.rs b/crates/experimental_prover/parallel/src/topology.rs index 516ce6bea8..4d5b41d568 100644 --- a/crates/experimental_prover/parallel/src/topology.rs +++ b/crates/experimental_prover/parallel/src/topology.rs @@ -20,30 +20,27 @@ impl Topology { } } +static TOPOLOGY: OnceLock = OnceLock::new(); + /// The pool's shape, resolved once. -/// -/// `LEANVM_NUM_THREADS` sets the total worker count, as in stable: performance -/// workers first, efficiency workers only beyond the performance cores. `1` is -/// strictly sequential. #[must_use] pub(crate) fn topology() -> Topology { - static TOPOLOGY: OnceLock = OnceLock::new(); - *TOPOLOGY.get_or_init(|| { - let requested = std::env::var("LEANVM_NUM_THREADS") - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|&n| n > 0); - match requested { - Some(n) => { - let efficiency = n.saturating_sub(perf_cores()).min(efficiency_cores()); - Topology { - perf: n - efficiency, - efficiency, - } + *TOPOLOGY.get_or_init(default_topology) +} + +/// Fixes the worker count before the pool starts: performance workers first, efficiency workers only beyond the +/// performance cores; `1` is sequential. Panics if it is already another. +pub fn set_num_threads(n: usize) { + let set = TOPOLOGY + .get_or_init(|| { + let efficiency = n.saturating_sub(perf_cores()).min(efficiency_cores()); + Topology { + perf: n - efficiency, + efficiency, } - None => default_topology(), - } - }) + }) + .total(); + assert_eq!(set, n, "the thread pool already runs with {set} threads"); } /// Worker count including the dispatcher. @@ -185,6 +182,13 @@ pub(crate) fn set_qos(qos: Qos) { mod tests { use super::*; + #[test] + #[should_panic(expected = "already runs")] + fn a_fixed_shape_refuses_another() { + set_num_threads(topology().perf); + set_num_threads(topology().perf + 1); + } + #[test] fn cpu_lists_parse_ranges_and_singles() { assert_eq!(cpu_list("0-3,8,10-11"), Some(vec![0, 1, 2, 3, 8, 10, 11])); diff --git a/crates/isa/src/prover.rs b/crates/isa/src/prover.rs index c8f5cba41c..751c2356ed 100644 --- a/crates/isa/src/prover.rs +++ b/crates/isa/src/prover.rs @@ -9,6 +9,9 @@ pub trait Prover: 'static { /// Warm up the prover's thread pool, so that no timed work pays for starting it. Idempotent. fn init(); + /// Fixes the thread pool's size (at least 1) before it starts. Panics if it is already another. + fn set_num_threads(n: usize); + /// Prove an execution of `program` on `public_input` at the code rate `2^-log_inv_rate`, or say why there is none. /// `witness` names the streams of its `hint_witness` calls, each with one entry per call, in the order they are /// popped, and no two with one name. diff --git a/crates/lean_vm/src/lib.rs b/crates/lean_vm/src/lib.rs index 13fbe4a545..2e83815096 100644 --- a/crates/lean_vm/src/lib.rs +++ b/crates/lean_vm/src/lib.rs @@ -43,9 +43,15 @@ pub mod lean_sphincs { }; } -/// Starts the prover's thread pool and builds the aggregation program, which [`AggregateProof::verify`] also reads. -/// Optional: both are built on first use otherwise. -pub fn setup() { +/// Call this function before generating / verifying proofs +/// Starts the prover's thread pool, with `threads` threads (0: one per core), and builds the aggregation program. +/// +/// # Panics +/// If the pool already runs with another thread count. +pub fn setup(threads: usize) { + if threads > 0 { + P::set_num_threads(threads); + } P::init(); aggregation::unified_guest(); } @@ -67,7 +73,6 @@ pub fn setup() { /// /// # WARNING /// - Signatures and children must be valid (children checked with [`AggregateProof::verify`]): otherwise proving fails. -/// - Call [`setup`] once before the first. pub fn aggregate( children: &[(AggregateClaims, AggregateProof)], raw_lean_xmss: Vec<(LeanXmssPublicKey, lean_xmss::Message, LeanXmssSignature)>, diff --git a/crates/lean_vm/tests/api.rs b/crates/lean_vm/tests/api.rs index b79c0bfcb8..e4f06a6688 100644 --- a/crates/lean_vm/tests/api.rs +++ b/crates/lean_vm/tests/api.rs @@ -6,7 +6,7 @@ const MSG_2: lean_xmss::Message = [2; lean_xmss::MESSAGE_LEN]; #[test] fn public_api_end_to_end() { - setup(); + setup(0); // 1. Eight leanXMSS signatures over three messages, three of the first, four of the second, one of the third, // by four keys. A key signs each message at its own leaf index, which the statement does not record. diff --git a/crates/stable_prover/lean_vm_core/src/prover.rs b/crates/stable_prover/lean_vm_core/src/prover.rs index 5d359a600b..5e4e97bb4e 100644 --- a/crates/stable_prover/lean_vm_core/src/prover.rs +++ b/crates/stable_prover/lean_vm_core/src/prover.rs @@ -12,6 +12,10 @@ impl isa::Prover for StableProver { parallel::init(); } + fn set_num_threads(n: usize) { + parallel::set_num_threads(n) + } + fn prove( program: &isa::Program, witness: &[(String, Vec>)], diff --git a/crates/stable_prover/parallel/src/lib.rs b/crates/stable_prover/parallel/src/lib.rs index 87b9757eb8..49d8da5323 100644 --- a/crates/stable_prover/parallel/src/lib.rs +++ b/crates/stable_prover/parallel/src/lib.rs @@ -10,26 +10,26 @@ use std::ptr::NonNull; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Condvar, Mutex, Once, OnceLock}; -/// Worker count including the dispatcher: `LEANVM_NUM_THREADS` when set, `1` being strictly sequential, else one per -/// physical core: the logical CPUs the platform reports, halved where Linux says SMT is active (a core's second -/// hardware thread slows a proof on a many-core machine). +static NUM_THREADS: OnceLock = OnceLock::new(); + +/// Worker count including the dispatcher: [`set_num_threads`]'s, else one per physical core (the logical CPUs, halved +/// where Linux says SMT is active: a second hardware thread per core slows a proof). #[must_use] #[inline] pub fn num_threads() -> usize { - static NUM_THREADS: OnceLock = OnceLock::new(); *NUM_THREADS.get_or_init(|| { - std::env::var("LEANVM_NUM_THREADS") - .ok() - .and_then(|v| v.parse::().ok()) - .filter(|&n| n > 0) - .unwrap_or_else(|| { - let logical = std::thread::available_parallelism().map_or(1, |n| n.get()); - let smt = std::fs::read_to_string("/sys/devices/system/cpu/smt/active").is_ok_and(|s| s.trim() == "1"); - if smt { (logical / 2).max(1) } else { logical } - }) + let logical = std::thread::available_parallelism().map_or(1, |n| n.get()); + let smt = std::fs::read_to_string("/sys/devices/system/cpu/smt/active").is_ok_and(|s| s.trim() == "1"); + if smt { (logical / 2).max(1) } else { logical } }) } +/// Fixes [`num_threads`] (at least 1) before the pool starts. Panics if it is already another. +pub fn set_num_threads(n: usize) { + let set = *NUM_THREADS.get_or_init(|| n); + assert_eq!(set, n, "the thread pool already runs with {set} threads"); +} + /// How finely guided scheduling divides the remaining work. const CLAIM_DIVISOR: usize = 4; @@ -503,3 +503,15 @@ where ) .unwrap_or_else(identity) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[should_panic(expected = "already runs")] + fn a_fixed_count_refuses_another() { + set_num_threads(num_threads()); + set_num_threads(num_threads() + 1); + } +} From 7a784f01fa0c337022e235f5f697a556319686b7 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Tue, 17 Nov 2026 10:31:04 -0500 Subject: [PATCH 03/34] sidecar: leanvm-sidecar proves `aggregate` for local clients over a Unix socket `crates/sidecar` holds the protocol (length-prefixed frames, a hello with the verifier tag, the program's Fiat-Shamir IV and the prover), the server, generic over the prover, and the Rust client, re-exported as `lean_vm::sidecar`. `bins/leanvm-sidecar` runs it (`--prover`, `--socket`, `--max-request-mib`, `--threads`); its README specifies the protocol. `AggregateClaims` gains `to_bytes`/`from_bytes`, `isa::Prover` a `NAME`. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 9 +- Cargo.lock | 27 ++ Cargo.toml | 3 + bench/src/benchmark.rs | 76 +--- bins/leanvm-sidecar/Cargo.toml | 25 + bins/leanvm-sidecar/README.md | 104 +++++ bins/leanvm-sidecar/src/main.rs | 66 +++ bins/leanvm-sidecar/tests/sidecar.rs | 120 +++++ crates/aggregation/src/lib.rs | 2 +- crates/aggregation/src/node.rs | 106 +++-- crates/aggregation/src/proof.rs | 59 +++ .../lean_vm_core/src/prover.rs | 2 + crates/isa/src/prover.rs | 3 + crates/lean_vm/Cargo.toml | 4 + crates/lean_vm/src/lib.rs | 7 + crates/sidecar/Cargo.toml | 17 + crates/sidecar/src/client.rs | 139 ++++++ crates/sidecar/src/lib.rs | 82 ++++ crates/sidecar/src/server.rs | 238 ++++++++++ crates/sidecar/src/wire.rs | 428 ++++++++++++++++++ .../stable_prover/lean_vm_core/src/prover.rs | 2 + 21 files changed, 1414 insertions(+), 105 deletions(-) create mode 100644 bins/leanvm-sidecar/Cargo.toml create mode 100644 bins/leanvm-sidecar/README.md create mode 100644 bins/leanvm-sidecar/src/main.rs create mode 100644 bins/leanvm-sidecar/tests/sidecar.rs create mode 100644 crates/sidecar/Cargo.toml create mode 100644 crates/sidecar/src/client.rs create mode 100644 crates/sidecar/src/lib.rs create mode 100644 crates/sidecar/src/server.rs create mode 100644 crates/sidecar/src/wire.rs diff --git a/AGENTS.md b/AGENTS.md index de16fa5066..8bbb5d1515 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ The fields are `K = GF(2)[x]/(x^64 + x^4 + x^3 + x + 1)` and `E = K[y]/(y^2 + x* ## Layout -The root `Cargo.toml` is workspace-only: libraries in `crates/`, the benchmark CLI in `bench/`. The prover exists twice, in two trees that share no prover crate: +The root `Cargo.toml` is workspace-only: libraries in `crates/`, the benchmark CLI in `bench/`, the binaries users run in `bins/`. The prover exists twice, in two trees that share no prover crate: - `crates/stable_prover/` (packages `stable_*`): the reference prover, written to be read: plain, safe code, as short as the protocol allows, and slower than experimental. Architecture-specific code lives in one SIMD layer, `primitives::field::lanes` (eight field elements at a time, `F128x8`, `F64x8`, `Acc8`, `Wide8`, and the byte maps and transposes over 64 bytes, each with one portable body that defines it and NEON, AVX2+VPCLMULQDQ and AVX-512+VPCLMULQDQ arms and the portable one), and in the shared `keccak256`'s multi-way permutation; the hot loops are written over them and nothing else in stable has `core::arch` or `cfg(target_feature)`. No uninitialized buffers, no non-temporal stores, no cache-tiled pass plans, no per-machine tuning. It keeps the whole level-0 codeword and every Merkle level, so its peak memory at the largest stacks is a multiple of experimental's on x86. An optimization enters it only where leaving it out costs a measurable share of a proof and its simple form stays short. - `crates/experimental_prover/` (packages `experimental_*`): the optimized prover, where performance work lands first. The conventions below about NEON, AVX-512, streaming stores and the like are about this tree. @@ -37,6 +37,7 @@ Both prove the same protocol and give the same proof bytes, pinned by `bench/tes | `bench_harness` | benchmark harness (`Plan`, `Timing`, `--tracing`, Bencher output, the jemalloc allocator); never linked by a prover | | `lean_da` | additive Reed-Solomon blob encoding (the stable tree's NTT), commitments, membership vectors | | `aggregation` | recursive signature and DA aggregation, generic over the prover: the guest's constants and witness, and the entry points | +| `sidecar` | proving in another process: the protocol of `bins/leanvm-sidecar`, its server (generic over the prover) and its Rust client | | `lean_vm` | the public API, not generic: it proves with the stable tree, or with the `experimental_prover` feature the experimental one | `aggregation` proves on whichever prover it is given, but derives the guest's constants and a child's witness from the stable tree, the reference: its bus blocks and their coordinates (`lean_vm_core::leaf`, whose term order the guest's arrays follow), the stacking, the grinds, the WHIR shapes and the NTT twiddles; both trees lay a proof out alike. @@ -51,7 +52,7 @@ Each tree holds, in dependency order, leaves first: | `keccak_gkr` | the prover of the VM's `KECCAK` argument | | `lean_vm_core` | arithmetization: tables, bus, constraints, `cpu::execute` and `cpu::prove` of an `isa::Program` and its witness streams; `cpu::verify` wraps `verifier`; the tree's `isa::Prover` | -Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. +Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. The sidecar is `bins/leanvm-sidecar`, which serves `aggregate` over a Unix socket so that clients in any language get the prover in a process of its own; its `README.md` is the protocol's specification, so a change to `crates/sidecar`'s `wire.rs`, or to the claims' byte form (`AggregateClaims::to_bytes`), lands there, and verifying stays in the library. ## Building and testing @@ -67,11 +68,11 @@ cargo fmt --all # max_width = 120 cargo machete --with-metadata # unused dependencies, dev ones included ``` -An x86-only `cfg` arm never compiles on an Apple machine, so type-check the other target before pushing anything `cfg`-gated (needs only `rustup target add x86_64-unknown-linux-gnu`; `bench_harness` and the CLI are left out because jemalloc's build script wants a cross `gcc`): +An x86-only `cfg` arm never compiles on an Apple machine, so type-check the other target before pushing anything `cfg`-gated (needs only `rustup target add x86_64-unknown-linux-gnu`; `bench_harness`, the benchmark CLI and the sidecar are left out because jemalloc's build script wants a cross `gcc`): ```bash RUSTFLAGS="-C target-cpu=znver4" \ - cargo check --release --workspace --exclude leanvm-bench --exclude bench_harness --target x86_64-unknown-linux-gnu + cargo check --release --workspace --exclude leanvm-bench --exclude leanvm-sidecar --exclude bench_harness --target x86_64-unknown-linux-gnu ``` `RUSTFLAGS` replaces `.cargo/config.toml`'s `target-cpu=native`; a target-specific `CARGO_TARGET_*_RUSTFLAGS` would be merged with it and hand the Apple CPU to the x86 target. diff --git a/Cargo.lock b/Cargo.lock index 9d77bfadb4..36437a129c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -375,6 +375,7 @@ dependencies = [ "lean_da", "lean_sphincs", "lean_xmss", + "sidecar", "stable_lean_vm_core", ] @@ -405,6 +406,19 @@ dependencies = [ "zkdsl_compiler", ] +[[package]] +name = "leanvm-sidecar" +version = "0.1.0" +dependencies = [ + "aggregation", + "bench_harness", + "clap", + "experimental_lean_vm_core", + "lean_vm", + "sidecar", + "stable_lean_vm_core", +] + [[package]] name = "libc" version = "0.2.186" @@ -563,6 +577,19 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "sidecar" +version = "0.1.0" +dependencies = [ + "aggregation", + "isa", + "lean_da", + "lean_sphincs", + "lean_xmss", + "libc", + "verifier", +] + [[package]] name = "smallvec" version = "1.15.2" diff --git a/Cargo.toml b/Cargo.toml index 1b9abe1b5a..81473da221 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,9 +11,11 @@ members = [ "crates/lean_da", "crates/aggregation", "crates/lean_vm", + "crates/sidecar", "crates/stable_prover/*", "crates/experimental_prover/*", "bench", + "bins/*", ] resolver = "3" @@ -37,6 +39,7 @@ lean_xmss = { path = "crates/lean_xmss" } lean_sphincs = { path = "crates/lean_sphincs" } lean_da = { path = "crates/lean_da" } aggregation = { path = "crates/aggregation" } +sidecar = { path = "crates/sidecar" } libc = "0.2" rand = "0.9" sha3 = "0.11" diff --git a/bench/src/benchmark.rs b/bench/src/benchmark.rs index 9b97906e9a..4ad7b224bc 100644 --- a/bench/src/benchmark.rs +++ b/bench/src/benchmark.rs @@ -255,73 +255,27 @@ fn write_leaves( log_inv_rate, ) .expect("leaf aggregates"); + // The proof carries no claims; the parent trusts its subprocess for them, standing in for a receiver that + // knows them. let (claims, leaf, _) = leaf; - write_claims(&mut bytes, &claims); - let proof = leaf.to_bytes(); - bytes.extend((proof.len() as u64).to_le_bytes()); - bytes.extend(proof); + for part in [claims.to_bytes(), leaf.to_bytes()] { + bytes.extend((part.len() as u64).to_le_bytes()); + bytes.extend(part); + } } std::fs::write(out, bytes).expect("the leaves are written"); } -/// A leaf's claims in the leaves' file: each list as its length and its `(key, message)` pairs, then -/// a 0/1 byte for whether there is a DA root and 32 root bytes, zero without one. The proof carries -/// no claims; the parent trusts its subprocess for them, standing in for a receiver that knows them. -fn write_claims(out: &mut Vec, claims: &AggregateClaims) { - out.extend((claims.lean_xmss.len() as u64).to_le_bytes()); - for (pk, message) in &claims.lean_xmss { - out.extend(pk.to_bytes()); - out.extend(message); - } - out.extend((claims.lean_sphincs.len() as u64).to_le_bytes()); - for (pk, message) in &claims.lean_sphincs { - out.extend(pk.to_bytes()); - out.extend(message); - } - out.push(u8::from(claims.da_commitment.is_some())); - out.extend(claims.da_commitment.unwrap_or_default()); -} - /// The leaves' file, read back: the bytes not yet read. struct LeavesFile<'a>(&'a [u8]); impl<'a> LeavesFile<'a> { - fn take(&mut self, n: usize) -> &'a [u8] { - let (head, rest) = self.0.split_at(n); + /// The next length-prefixed part. + fn part(&mut self) -> &'a [u8] { + let (len, rest) = self.0.split_first_chunk().expect("a length"); + let (part, rest) = rest.split_at(u64::from_le_bytes(*len) as usize); self.0 = rest; - head - } - - fn array(&mut self) -> [u8; N] { - self.take(N).try_into().expect("N bytes") - } - - fn count(&mut self) -> usize { - u64::from_le_bytes(self.array()) as usize - } - - /// The inverse of [`write_claims`]. - fn claims(&mut self) -> AggregateClaims { - let n = self.count(); - let lean_xmss = (0..n) - .map(|_| (LeanXmssPublicKey::from_bytes(&self.array()), self.array())) - .collect(); - let n = self.count(); - let lean_sphincs = (0..n) - .map(|_| { - ( - lean_sphincs::LeanSphincsPublicKey::from_bytes(&self.array()), - self.array(), - ) - }) - .collect(); - let [has_root] = self.array(); - let root = self.array(); - AggregateClaims { - lean_xmss, - lean_sphincs, - da_commitment: (has_root == 1).then_some(root), - } + part } } @@ -351,12 +305,8 @@ fn leaves_from_subprocess(n: usize) -> Vec<(AggregateClaims, AggregateProof)> { let mut file = LeavesFile(&bytes); let mut leaves = Vec::with_capacity(n); while !file.0.is_empty() { - let claims = file.claims(); - let len = file.count(); - leaves.push(( - claims, - AggregateProof::from_bytes(file.take(len)).expect("a leaf proof"), - )); + let claims = AggregateClaims::from_bytes(file.part()).expect("a leaf's claims"); + leaves.push((claims, AggregateProof::from_bytes(file.part()).expect("a leaf proof"))); } assert_eq!(leaves.len(), n, "one proof per leaf"); leaves diff --git a/bins/leanvm-sidecar/Cargo.toml b/bins/leanvm-sidecar/Cargo.toml new file mode 100644 index 0000000000..8db3c6279b --- /dev/null +++ b/bins/leanvm-sidecar/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "leanvm-sidecar" +version.workspace = true +edition.workspace = true +publish = false + +[[bin]] +name = "leanvm-sidecar" +path = "src/main.rs" +# A binary, nothing to document. +doc = false + +[lints] +workspace = true + +[dependencies] +sidecar.workspace = true +stable_lean_vm_core = { path = "../../crates/stable_prover/lean_vm_core" } +experimental_lean_vm_core = { path = "../../crates/experimental_prover/lean_vm_core" } +bench_harness.workspace = true +clap = { version = "4", features = ["derive"] } + +[dev-dependencies] +lean_vm = { path = "../../crates/lean_vm" } +aggregation = { workspace = true, features = ["signers-cache"] } diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md new file mode 100644 index 0000000000..a97aafdbbe --- /dev/null +++ b/bins/leanvm-sidecar/README.md @@ -0,0 +1,104 @@ +# leanvm-sidecar + +A process that proves for clients on the same machine, over a Unix stream socket. It runs the library's one proving call, `aggregate`, so a client in any language gets the prover's full speed, with its own allocator, thread pool and retained memory, in a process of its own. Verifying stays in the client, with the library (`AggregateProof::verify`), which is fast and needs no prover: from another language, through a binding to it. + +## Running + +```bash +cargo build --release -p leanvm-sidecar +target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar.sock +``` + +| flag | default | meaning | +| --- | --- | --- | +| `--prover` | required | `stable` or `experimental`: both give the same proofs, experimental faster | +| `--socket` | required | the socket to listen on | +| `--max-request-mib` | 1024 | the longest request taken, in MiB | +| `--threads` | 0 | the prover's thread count, 0 for one per core | +| `--tracing` | off | print each proof's timing tree (`RUST_LOG` sets the level) | + +- The sidecar starts the prover's thread pool and compiles the aggregation program before it listens, so the socket appears once it is ready. It says so on standard error, then logs one line per request there. +- A sidecar holds a lock on the file beside its socket, the socket's path with `.lock` appended, while it runs. A second sidecar on the same socket fails at once. A socket left by a sidecar that stopped is replaced; a path that is some other file, or a socket another program listens on, is not. +- The socket takes the process's umask, and connecting needs write permission on it. Restrict it to the clients the sidecar serves: it trusts them with its resources. Any of them can keep it proving, and every connection may hold a request up to the max request in memory. +- Requests are proven one at a time, across all connections, in the order they are received in full. A request whose client disconnects before its proof starts is dropped; one already proving finishes, and its proof is dropped. A client that gives up on a request should therefore close its connection, and send the request again, if at all, on a new one. +- Stop the sidecar with a signal: it holds nothing to save. A request in flight is lost, and its client sees the connection close. The same goes if the prover panics, which exits the process: run the sidecar under a supervisor that restarts it. +- The first proof of a fresh sidecar is a little slower than the next ones. + +## Clients + +From Rust, `lean_vm::sidecar::Client` speaks the protocol, with the library's arguments but for the blobs, which go as they are: + +```rust +let mut sidecar = lean_vm::sidecar::Client::connect("/run/leanvm/sidecar.sock", None)?; +let (claims, proof) = sidecar.aggregate(&children, &raw_lean_xmss, &[], &[], OutputClaims::All, 2)?; +``` + +`connect`'s timeout, which `Client::set_timeout` changes, bounds how long each read or write waits; after a timeout, drop the client. The client checks that the claims it gets back are those asked for, but for the DA root of blobs, which it only checks is there. + +From another language, a client is the protocol below: connect, read the hello, then write a request and read its response, as many times as needed. + +## Protocol, version 1 + +Every integer is little-endian. A **frame** is a `u64` length, then that many bytes of body. A **bytes** field is a `u64` length, then that many bytes. A **count** is a `u64`. + +On each connection the sidecar first sends a hello frame. The client then sends request frames, and after each one reads a response frame before sending the next. Shutting down only the writing half of the connection counts as closing it: the sidecar drops a request it has not started proving. + +### Hello + +| field | type | value | +| --- | --- | --- | +| magic | 8 bytes | `leanvmsc` | +| version | `u32` | 1 | +| max request | `u64` | the longest request body the sidecar takes | +| verifier | bytes | the verifier's version tag, `leanvm-v1` | +| program | 32 bytes | the aggregation program's Fiat-Shamir IV, `Bytecode::fiat_shamir_iv` | +| prover | bytes | `stable` or `experimental` | + +Every version keeps the magic and the version first. A client checks them, the verifier tag and the program, and disconnects if any differs from what it expects: a proof for another verifier or another build's program does not verify with its library. A client must not send a request body longer than the max request: the sidecar would close the connection without reading it, and the client could fail mid-write. + +### Request + +The library's `aggregate` arguments: + +| field | type | meaning | +| --- | --- | --- | +| operation | `u8` | 1, `aggregate` | +| rate | `u8` | `log_inv_rate`, 1 to 4: higher gives a smaller proof, slower | +| children | count, then per child: claims (bytes), proof (bytes) | at most 16 | +| leanXMSS signatures | count, then per signature: pubkey (32 bytes), message (32 bytes), signature (2404 bytes) | | +| leanSPHINCS signatures | count, then per signature: pubkey (32 bytes), message (32 bytes), signature (7740 bytes) | | +| blobs | count, then per blob: 8192 symbols of 16 bytes each | at most 32 | +| output | `u8`, then for 1 the claims (bytes) | 0: every covered claim; 1: exactly these claims, each covered by the inputs | + +Keys, signatures and proofs are the libraries' byte forms (`to_bytes`), and a symbol is its 128 bits as a little-endian `u128`, as LeanDA hashes it. Signatures may come in any order, and a repeated claim, the same pubkey and message twice, counts once. As with the library, the sidecar does not verify children: an invalid one costs a failed proof, status 3 or 4, so a child from an untrusted source is checked with `AggregateProof::verify` first. + +**Claims**, as `AggregateClaims::to_bytes` writes them: + +| field | type | +| --- | --- | +| leanXMSS claims | count, then per claim: pubkey (32 bytes), message (32 bytes) | +| leanSPHINCS claims | count, then per claim: pubkey (32 bytes), message (32 bytes) | +| DA commitment | `u8`, 0 for none or 1, then for 1 the root (32 bytes) | + +A proof verifies only against claims whose lists are strictly sorted, comparing each claim's 64 bytes, pubkey then message, as a byte string. The claims a response carries are; the output claims of a request may come in any order. + +### Response + +| field | type | meaning | +| --- | --- | --- | +| status | `u8` | below | +| for 0: claims, proof | bytes, bytes | what the proof proves, and the proof (`AggregateProof::from_bytes` reads it) | +| otherwise: message | bytes | UTF-8, why there is no proof | + +| status | name | meaning | +| --- | --- | --- | +| 0 | ok | | +| 1 | invalid input | the request is inconsistent: a rate out of range, too many children or blobs, nothing to publish, an output claim the inputs do not cover | +| 2 | invalid signature | a signature is invalid | +| 3 | invalid child | a child is invalid, or its proof does not decode | +| 4 | proving failed | an input the checks above missed is invalid, or the request is too large for one proof | +| 5 | malformed | the request does not decode, or is longer than the hello's max request | + +A request with several problems gets the status of the first one found. + +After any response the connection stays usable, except after a request longer than the max request: the sidecar answers it with status 5 without reading it, then closes the connection. diff --git a/bins/leanvm-sidecar/src/main.rs b/bins/leanvm-sidecar/src/main.rs new file mode 100644 index 0000000000..2ed0bf4f00 --- /dev/null +++ b/bins/leanvm-sidecar/src/main.rs @@ -0,0 +1,66 @@ +//! The leanVM sidecar: proves for one machine's clients over a Unix socket. `README.md` says how to run it and +//! specifies the protocol. + +#[global_allocator] +static ALLOCATOR: bench_harness::Jemalloc = bench_harness::Jemalloc; + +use clap::Parser; +use experimental_lean_vm_core::ExperimentalProver; +use stable_lean_vm_core::StableProver; +use std::path::PathBuf; +use std::process::ExitCode; + +/// Which of the two provers runs. +#[derive(Clone, Copy, clap::ValueEnum)] +enum Prover { + /// The stable prover. + Stable, + /// The experimental prover. + Experimental, +} + +#[derive(Parser)] +#[command(name = "leanvm-sidecar")] +struct Cli { + /// The prover to run. + #[arg(long, value_enum)] + prover: Prover, + + /// The Unix socket to listen on. It appears once the sidecar is ready. + #[arg(long)] + socket: PathBuf, + + /// The longest request taken, in MiB. + #[arg( + long, + default_value_t = 1024, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1 << 20) + )] + max_request_mib: u64, + + /// Thread count (0: one per core). + #[arg(long, default_value_t = 0)] + threads: usize, + + /// Print each proof's timing tree. Use RUST_LOG to adjust verbosity. + #[arg(long)] + tracing: bool, +} + +fn main() -> ExitCode { + let cli = Cli::parse(); + if cli.tracing { + bench_harness::init_tracing(); + } + let config = sidecar::Config { + socket: cli.socket, + max_request: cli.max_request_mib << 20, + threads: cli.threads, + }; + let Err(e) = match cli.prover { + Prover::Stable => sidecar::serve::(&config), + Prover::Experimental => sidecar::serve::(&config), + }; + eprintln!("leanvm-sidecar: {}: {e}", config.socket.display()); + ExitCode::FAILURE +} diff --git a/bins/leanvm-sidecar/tests/sidecar.rs b/bins/leanvm-sidecar/tests/sidecar.rs new file mode 100644 index 0000000000..0c45b6fa70 --- /dev/null +++ b/bins/leanvm-sidecar/tests/sidecar.rs @@ -0,0 +1,120 @@ +//! The sidecar binary end to end, through the library's client: a leaf with every kind of input, a narrowed node over +//! it from a second connection, refusals that leave the connection usable, a request that is not one sent as another +//! language's client would, and the socket's life across a second sidecar and a restart. + +use aggregation::signers_cache; +use lean_vm::sidecar::{Client, Rejection, SidecarError}; +use lean_vm::{AggregateClaims, OutputClaims, lean_da}; +use stable_lean_vm_core::StableProver; +use std::io::{Read, Write}; +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command}; +use std::time::{Duration, Instant}; + +struct Sidecar(Child); + +impl Drop for Sidecar { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +/// The socket's path, and its lock's, removed at the end whatever happens. +struct Socket(PathBuf); + +impl Drop for Socket { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + let _ = std::fs::remove_file(self.0.with_extension("sock.lock")); + } +} + +fn command(socket: &Path) -> Command { + let mut command = Command::new(env!("CARGO_BIN_EXE_leanvm-sidecar")); + command.args(["--prover", "stable", "--socket"]).arg(socket); + command +} + +/// Starts a sidecar and waits until it serves. +fn start(socket: &Path) -> (Sidecar, Client) { + let mut sidecar = Sidecar(command(socket).spawn().unwrap()); + let start = Instant::now(); + loop { + if let Ok(client) = Client::connect(socket, None) { + return (sidecar, client); + } + assert!(sidecar.0.try_wait().unwrap().is_none(), "the sidecar exited"); + assert!(start.elapsed() < Duration::from_secs(300), "the sidecar never served"); + std::thread::sleep(Duration::from_millis(50)); + } +} + +fn read_frame(stream: &mut UnixStream) -> Vec { + let mut len = [0; 8]; + stream.read_exact(&mut len).unwrap(); + let mut body = vec![0; u64::from_le_bytes(len) as usize]; + stream.read_exact(&mut body).unwrap(); + body +} + +#[test] +fn sidecar_end_to_end() { + let guard = Socket(PathBuf::from(format!( + "/tmp/leanvm-sidecar-test-{}.sock", + std::process::id() + ))); + let socket = &guard.0; + let (first, mut client) = start(socket); + assert_eq!(client.prover(), "stable"); + + let xmss = signers_cache::get_signers::(3); + let sphincs = signers_cache::get_lean_sphincs_signers::(1); + let blobs: Vec = vec![std::array::from_fn(|i| i as u128)]; + let leaf = client + .aggregate(&[], &xmss[..2], &sphincs, &blobs, OutputClaims::All, 2) + .unwrap(); + assert!(leaf.1.verify(&leaf.0)); + assert_eq!(leaf.0.num_signature_claims(), 3); + assert_eq!( + leaf.0.da_commitment, + Some(lean_da::BlobMatrix::build(&blobs).unwrap().commitment()) + ); + + // A second connection while the first stays open: the leaf and one more signature, keeping only one leanXMSS + // claim of each. + let mut other = Client::connect(socket, None).unwrap(); + let mut kept: Vec<_> = [&xmss[0], &xmss[2]].iter().map(|(pk, m, _)| (*pk, *m)).collect(); + kept.sort(); + let only = AggregateClaims { + lean_xmss: kept, + ..AggregateClaims::default() + }; + let node = other + .aggregate(&[leaf], &xmss[2..], &[], &[], OutputClaims::Only(&only), 1) + .unwrap(); + assert_eq!(node.0, only); + assert!(node.1.verify(&only)); + + // Refusals leave the connection in step: two in a row. + for rate in [0, 300] { + match client.aggregate(&[], &xmss[..1], &[], &[], OutputClaims::All, rate) { + Err(SidecarError::Rejected(Rejection::InvalidInput, message)) => assert!(message.contains("log_inv_rate")), + other => panic!("rate {rate}: {other:?}"), + } + } + + // A client in another language sends an operation this version does not have. + let mut raw = UnixStream::connect(socket).unwrap(); + assert!(read_frame(&mut raw).starts_with(b"leanvmsc")); + raw.write_all(&[&1u64.to_le_bytes()[..], &[7]].concat()).unwrap(); + assert_eq!(read_frame(&mut raw)[0], Rejection::Malformed as u8); + + // A second sidecar leaves a live one's socket alone; a restart replaces the stale socket a killed one leaves. + let refused = command(socket).output().unwrap(); + assert!(!refused.status.success()); + drop(first); + assert!(socket.exists()); + let (_sidecar, _client) = start(socket); +} diff --git a/crates/aggregation/src/lib.rs b/crates/aggregation/src/lib.rs index 259adb897c..beab2876ee 100644 --- a/crates/aggregation/src/lib.rs +++ b/crates/aggregation/src/lib.rs @@ -58,7 +58,7 @@ pub use error::AggregateError; pub use guest::unified_guest; use lean_sphincs::LeanSphincsPublicKey; use lean_xmss::LeanXmssPublicKey; -pub use node::aggregate; +pub use node::{aggregate, published_claims}; pub use proof::{AggregateClaims, AggregateProof, OutputClaims}; /// One leanXMSS claim: a pubkey, and the message it signed at whatever leaf index its diff --git a/crates/aggregation/src/node.rs b/crates/aggregation/src/node.rs index 9b2298e2b6..45f7473025 100644 --- a/crates/aggregation/src/node.rs +++ b/crates/aggregation/src/node.rs @@ -2,7 +2,7 @@ use super::cells::count; use super::child::gen_verify; -use super::coverage::plan_coverage; +use super::coverage::{Coverage, plan_coverage}; use super::deferred::{DeferredBytecodeClaim, aggregate_deferred_claims, bytecode_sums}; use super::error::AggregateError; use super::guest::{fs_seed, unified_guest}; @@ -11,7 +11,7 @@ use super::hints::FORGE; use super::hints::{Hints, lean_sphincs_hints, lean_xmss_hints}; use super::proof::{AggregateClaims, AggregateProof, OutputClaims, is_sorted}; use super::statement::{lean_sphincs_claim_cells, lean_xmss_claim_cells, list_split, statement_digest}; -use super::{MAX_LOG_INV_RATE, MAX_RECURSIONS, MIN_LOG_INV_RATE}; +use super::{LeanSphincsClaim, LeanXmssClaim, MAX_LOG_INV_RATE, MAX_RECURSIONS, MIN_LOG_INV_RATE}; use fiat_shamir_prover::merkle::hash_to_scalars; use isa::{Prover, Stats}; use lean_da::CELLS_PER_ROW; @@ -48,10 +48,6 @@ pub fn aggregate( // Two proofs at once would share the pool and hold both proofs' memory. static PROVING: Mutex<()> = Mutex::new(()); let _proving = PROVING.lock().unwrap_or_else(PoisonError::into_inner); - let declare = match output { - OutputClaims::All => None, - OutputClaims::Only(claims) => Some(claims), - }; // Proving checks the rate too, but only after the witness is built. if !(MIN_LOG_INV_RATE..=MAX_LOG_INV_RATE).contains(&log_inv_rate) { return Err(AggregateError::InvalidInput("log_inv_rate out of range")); @@ -59,39 +55,16 @@ pub fn aggregate( if children.len() > MAX_RECURSIONS { return Err(AggregateError::InvalidInput("more than MAX_RECURSIONS children")); } - // The published root and the guest's slot for it: a child's, or `children.len()` for the node's own, its blobs' - // or none. - let own = children.len(); - let (da_pick, da_root) = match (blobs, declare.map(|claims| claims.da_commitment)) { - (Some(matrix), declared) => { - let root = Some(matrix.commitment()); - if declared.is_some_and(|declared| declared != root) { - return Err(AggregateError::InvalidInput("a node with blobs publishes their root")); - } - (own, root) - } - (None, None) => children - .iter() - .position(|(claims, _)| claims.da_commitment.is_some()) - .map_or((own, None), |i| (i, children[i].0.da_commitment)), - (None, Some(None)) => (own, None), - (None, Some(root)) => children - .iter() - .position(|(claims, _)| claims.da_commitment == root) - .map(|i| (i, root)) - .ok_or(AggregateError::InvalidInput("the requested DA root is not a child's"))?, - }; - - // The request's checks, on the claim lists alone, before any costly work. - if !children.iter().all(|(claims, _)| is_sorted(claims)) { - return Err(AggregateError::InvalidChild); - } let raw_lean_xmss_claims = sorted_claims(&mut raw_lean_xmss); let raw_lean_sphincs_claims = sorted_claims(&mut raw_lean_sphincs); - let cover = plan_coverage(&raw_lean_xmss_claims, &raw_lean_sphincs_claims, children, declare)?; - if cover.lean_xmss.signers.is_empty() && cover.lean_sphincs.signers.is_empty() && da_root.is_none() { - return Err(AggregateError::InvalidInput("nothing to publish")); - } + let blobs_root = blobs.map(lean_da::BlobMatrix::commitment); + let (cover, da_pick, da_root) = plan( + children, + &raw_lean_xmss_claims, + &raw_lean_sphincs_claims, + blobs_root, + output, + )?; // Nothing here verifies a child (see `AGENTS.md`): the replay reads out of its proof the summary `gen_verify` // builds the guest's witness from, and fails only on a proof it cannot walk. @@ -282,6 +255,65 @@ pub fn aggregate( Ok((claims, proof, stats)) } +/// The claims [`aggregate`] proves for these inputs, without proving: `blobs_root` is its blobs' root, if any. `Err` +/// for the requests it refuses on their claims alone. +pub fn published_claims( + children: &[(AggregateClaims, AggregateProof)], + raw_lean_xmss: &[LeanXmssClaim], + raw_lean_sphincs: &[LeanSphincsClaim], + blobs_root: Option<[u8; 32]>, + output: OutputClaims, +) -> Result { + let (cover, _, da_commitment) = plan(children, raw_lean_xmss, raw_lean_sphincs, blobs_root, output)?; + Ok(AggregateClaims { + lean_xmss: cover.lean_xmss.signers, + lean_sphincs: cover.lean_sphincs.signers, + da_commitment, + }) +} + +/// The request's checks, on the claim lists alone, before any costly work: the coverage, and the published root with +/// the guest's slot for it, a child's or `children.len()` for the node's own, its blobs' or none. +fn plan( + children: &[(AggregateClaims, AggregateProof)], + raw_lean_xmss: &[LeanXmssClaim], + raw_lean_sphincs: &[LeanSphincsClaim], + blobs_root: Option<[u8; 32]>, + output: OutputClaims, +) -> Result<(Coverage, usize, Option<[u8; 32]>), AggregateError> { + let declare = match output { + OutputClaims::All => None, + OutputClaims::Only(claims) => Some(claims), + }; + let own = children.len(); + let (da_pick, da_root) = match (blobs_root, declare.map(|claims| claims.da_commitment)) { + (Some(root), declared) => { + if declared.is_some_and(|declared| declared != Some(root)) { + return Err(AggregateError::InvalidInput("a node with blobs publishes their root")); + } + (own, Some(root)) + } + (None, None) => children + .iter() + .position(|(claims, _)| claims.da_commitment.is_some()) + .map_or((own, None), |i| (i, children[i].0.da_commitment)), + (None, Some(None)) => (own, None), + (None, Some(root)) => children + .iter() + .position(|(claims, _)| claims.da_commitment == root) + .map(|i| (i, root)) + .ok_or(AggregateError::InvalidInput("the requested DA root is not a child's"))?, + }; + if !children.iter().all(|(claims, _)| is_sorted(claims)) { + return Err(AggregateError::InvalidChild); + } + let cover = plan_coverage(raw_lean_xmss, raw_lean_sphincs, children, declare)?; + if cover.lean_xmss.signers.is_empty() && cover.lean_sphincs.signers.is_empty() && da_root.is_none() { + return Err(AggregateError::InvalidInput("nothing to publish")); + } + Ok((cover, da_pick, da_root)) +} + /// Sort raw signatures on their claims and drop repeats: on the whole (pubkey, message) pair, so a signer may appear /// once per message, and one claim needs one signature, whatever leaf index a leanXMSS one carries. Returns the claims. fn sorted_claims(raw: &mut Vec<(K, [u8; 32], S)>) -> Vec<(K, [u8; 32])> { diff --git a/crates/aggregation/src/proof.rs b/crates/aggregation/src/proof.rs index dcffb01e05..ed2d4a9c6f 100644 --- a/crates/aggregation/src/proof.rs +++ b/crates/aggregation/src/proof.rs @@ -65,12 +65,71 @@ pub(super) fn is_sorted(claims: &AggregateClaims) -> bool { claims.lean_xmss.windows(2).all(|w| w[0] < w[1]) && claims.lean_sphincs.windows(2).all(|w| w[0] < w[1]) } +// Wire format of claims: the leanXMSS claims (a u64 count, then each claim's pubkey and message), the leanSPHINCS +// claims alike, then the DA commitment (a byte, 0 for none or 1, then the 32-byte root). Little-endian, and +// canonical: decoding rejects any other flag and trailing bytes. Order is kept, sorted or not. + +const _: () = assert!(lean_xmss::PUB_KEY_SIZE == 32 && lean_sphincs::PUB_KEY_SIZE == 32); +const _: () = assert!(lean_xmss::MESSAGE_LEN == 32 && lean_sphincs::MESSAGE_LEN == 32); + +fn encode_claims(out: &mut Vec, claims: &[(K, [u8; 32])], pk_bytes: impl Fn(&K) -> [u8; 32]) { + out.extend((claims.len() as u64).to_le_bytes()); + for (pk, message) in claims { + out.extend(pk_bytes(pk)); + out.extend(message); + } +} + +/// Reads a count and that many `(pubkey, message)` pairs off the front of `bytes`. A forged count allocates nothing. +fn decode_claims(bytes: &mut &[u8], pk: impl Fn(&[u8; 32]) -> K) -> Option> { + let (count, rest) = bytes.split_first_chunk::<8>()?; + let count = usize::try_from(u64::from_le_bytes(*count)).ok()?; + let (pairs, rest) = rest.split_at_checked(count.checked_mul(64)?)?; + *bytes = rest; + let pairs = pairs.as_chunks::<32>().0.as_chunks::<2>().0; + Some(pairs.iter().map(|[key, message]| (pk(key), *message)).collect()) +} + impl AggregateClaims { /// The number of `(pubkey, message)` claims, not of distinct signers: a pubkey may sign /// several messages. pub fn num_signature_claims(&self) -> usize { self.lean_xmss.len() + self.lean_sphincs.len() } + + pub fn to_bytes(&self) -> Vec { + let mut out = Vec::new(); + encode_claims(&mut out, &self.lean_xmss, lean_xmss::LeanXmssPublicKey::to_bytes); + encode_claims( + &mut out, + &self.lean_sphincs, + lean_sphincs::LeanSphincsPublicKey::to_bytes, + ); + match self.da_commitment { + None => out.push(0), + Some(root) => { + out.push(1); + out.extend(root); + } + } + out + } + + /// Decode claims, `None` if the bytes are not exactly one encoding. + pub fn from_bytes(mut bytes: &[u8]) -> Option { + let lean_xmss = decode_claims(&mut bytes, lean_xmss::LeanXmssPublicKey::from_bytes)?; + let lean_sphincs = decode_claims(&mut bytes, lean_sphincs::LeanSphincsPublicKey::from_bytes)?; + let da_commitment = match bytes { + [0] => None, + [1, root @ ..] => Some(<[u8; 32]>::try_from(root).ok()?), + _ => return None, + }; + Some(Self { + lean_xmss, + lean_sphincs, + da_commitment, + }) + } } impl AggregateProof { diff --git a/crates/experimental_prover/lean_vm_core/src/prover.rs b/crates/experimental_prover/lean_vm_core/src/prover.rs index d34f4e99d6..464eb1bce1 100644 --- a/crates/experimental_prover/lean_vm_core/src/prover.rs +++ b/crates/experimental_prover/lean_vm_core/src/prover.rs @@ -12,6 +12,8 @@ pub struct ExperimentalProver; const GROUP: usize = 8; impl isa::Prover for ExperimentalProver { + const NAME: &'static str = "experimental"; + fn init() { parallel::init(); } diff --git a/crates/isa/src/prover.rs b/crates/isa/src/prover.rs index 751c2356ed..175b9bbc0d 100644 --- a/crates/isa/src/prover.rs +++ b/crates/isa/src/prover.rs @@ -6,6 +6,9 @@ use verifier::{E, layout::N_TABLES}; /// A prover of ISA programs, and the thread pool and field kernels it brings for the work around a proof. pub trait Prover: 'static { + /// The prover's name: `stable` or `experimental`. + const NAME: &'static str; + /// Warm up the prover's thread pool, so that no timed work pays for starting it. Idempotent. fn init(); diff --git a/crates/lean_vm/Cargo.toml b/crates/lean_vm/Cargo.toml index 01451d2cd9..c0ced4e277 100644 --- a/crates/lean_vm/Cargo.toml +++ b/crates/lean_vm/Cargo.toml @@ -15,6 +15,10 @@ lean_sphincs.workspace = true stable_lean_vm_core = { path = "../stable_prover/lean_vm_core" } experimental_lean_vm_core = { path = "../experimental_prover/lean_vm_core", optional = true } +# The sidecar speaks over a Unix socket. +[target.'cfg(unix)'.dependencies] +sidecar.workspace = true + [features] # Prove with the experimental tree rather than the stable one. experimental_prover = ["dep:experimental_lean_vm_core"] diff --git a/crates/lean_vm/src/lib.rs b/crates/lean_vm/src/lib.rs index 2e83815096..879deff7f4 100644 --- a/crates/lean_vm/src/lib.rs +++ b/crates/lean_vm/src/lib.rs @@ -43,6 +43,13 @@ pub mod lean_sphincs { }; } +/// [`aggregate`] in another process, the sidecar (`bins/leanvm-sidecar`), which keeps the prover's memory, threads +/// and warm state out of this one. Verifying stays here, with [`AggregateProof::verify`]. +#[cfg(unix)] +pub mod sidecar { + pub use ::sidecar::{Client, Rejection, SidecarError}; +} + /// Call this function before generating / verifying proofs /// Starts the prover's thread pool, with `threads` threads (0: one per core), and builds the aggregation program. /// diff --git a/crates/sidecar/Cargo.toml b/crates/sidecar/Cargo.toml new file mode 100644 index 0000000000..06f02cbd1d --- /dev/null +++ b/crates/sidecar/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "sidecar" +version.workspace = true +edition.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +isa.workspace = true +verifier.workspace = true +aggregation.workspace = true +lean_da.workspace = true +lean_xmss.workspace = true +lean_sphincs.workspace = true +libc.workspace = true diff --git a/crates/sidecar/src/client.rs b/crates/sidecar/src/client.rs new file mode 100644 index 0000000000..57c0a406f1 --- /dev/null +++ b/crates/sidecar/src/client.rs @@ -0,0 +1,139 @@ +//! The protocol from Rust. + +use crate::wire::{self, Incoming}; +use crate::{Rejection, SidecarError}; +use aggregation::{AggregateClaims, AggregateProof, OutputClaims}; +use lean_da::Blob; +use lean_sphincs::{LeanSphincsPublicKey, LeanSphincsSignature}; +use lean_xmss::{LeanXmssPublicKey, LeanXmssSignature}; +use std::io::Write; +use std::os::unix::net::UnixStream; +use std::path::Path; +use std::time::Duration; + +/// A connection to a sidecar, for one request at a time. +/// +/// After an error other than [`SidecarError::Rejected`], the connection is in an unknown state: drop it and connect +/// again. +#[derive(Debug)] +pub struct Client { + stream: UnixStream, + prover: String, + max_request: u64, +} + +impl Client { + /// Connects to the sidecar listening on `socket`, and checks it speaks this protocol version and proves this + /// build's aggregation program, which the first connection in a process builds. `timeout` is + /// [`Self::set_timeout`]'s, already for reading the sidecar's hello. + pub fn connect(socket: impl AsRef, timeout: Option) -> Result { + let mut stream = UnixStream::connect(socket)?; + stream.set_read_timeout(timeout)?; + stream.set_write_timeout(timeout)?; + let body = match wire::read_frame(&mut stream, 1 << 16)? { + Incoming::Body(body) => body, + Incoming::End => return Err(SidecarError::Protocol("closed before its hello".into())), + // Not a hello, as `hello_version` then says. + Incoming::TooLarge(_) => Vec::new(), + }; + let version = + wire::hello_version(&body).ok_or_else(|| SidecarError::Incompatible("not a leanVM sidecar".into()))?; + if version != wire::VERSION { + return Err(SidecarError::Incompatible(format!( + "protocol version {version}, this client speaks {}", + wire::VERSION + ))); + } + let hello = + wire::decode_hello(&body).ok_or_else(|| SidecarError::Protocol("a hello that does not decode".into()))?; + if hello.tag != verifier::TAG { + return Err(SidecarError::Incompatible(format!( + "proves for verifier {}, this client verifies {}", + String::from_utf8_lossy(&hello.tag), + String::from_utf8_lossy(verifier::TAG) + ))); + } + if hello.program != aggregation::unified_guest().bytecode().fiat_shamir_iv() { + return Err(SidecarError::Incompatible( + "proves another aggregation program: another leanVM build".into(), + )); + } + Ok(Self { + stream, + prover: hello.prover, + max_request: hello.max_request, + }) + } + + /// How long a call may wait on the sidecar, reading or writing, before it fails with [`SidecarError::Io`]; `None`, + /// the default, waits as long as it takes. Dropping the client then closes the connection, so the sidecar drops + /// the request unless it has started proving it. + pub fn set_timeout(&self, timeout: Option) -> std::io::Result<()> { + self.stream.set_read_timeout(timeout)?; + self.stream.set_write_timeout(timeout) + } + + /// The prover the sidecar runs: `stable` or `experimental`. + pub fn prover(&self) -> &str { + &self.prover + } + + /// The library's `aggregate`, proven by the sidecar. The arguments are the library's but for the blobs, which + /// go as they are (none for no blobs) and which the sidecar builds the matrix of. + /// + /// As with the library, children must be valid (checked with [`AggregateProof::verify`]), or proving fails. The + /// claims returned are checked to be those asked for, but for the root of `blobs`, which is only checked to be there. + pub fn aggregate( + &mut self, + children: &[(AggregateClaims, AggregateProof)], + raw_lean_xmss: &[(LeanXmssPublicKey, lean_xmss::Message, LeanXmssSignature)], + raw_lean_sphincs: &[(LeanSphincsPublicKey, lean_sphincs::Message, LeanSphincsSignature)], + blobs: &[Blob], + output: OutputClaims, + log_inv_rate: usize, + ) -> Result<(AggregateClaims, AggregateProof), SidecarError> { + let request = wire::encode_request(children, raw_lean_xmss, raw_lean_sphincs, blobs, output, log_inv_rate); + let len = request.len() as u64 - 8; + if len > self.max_request { + return Err(SidecarError::Rejected( + Rejection::Malformed, + format!( + "a request of {len} bytes, the sidecar takes at most {}", + self.max_request + ), + )); + } + self.stream.write_all(&request)?; + // The claims a response carries are no longer than the request. + let body = match wire::read_frame(&mut self.stream, len + PROOF_ROOM)? { + Incoming::Body(body) => body, + Incoming::End => return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof).into()), + Incoming::TooLarge(_) => return Err(SidecarError::Protocol("a response longer than any".into())), + }; + let (claims, proof) = match wire::decode_response(&body) { + Some(Ok(proven)) => proven, + Some(Err((rejection, message))) => return Err(SidecarError::Rejected(rejection, message)), + None => return Err(SidecarError::Protocol("a response that does not decode".into())), + }; + // The root of blobs is the sidecar's to compute: it is taken as given, but must be there. + let blobs_root = (!blobs.is_empty()).then_some(claims.da_commitment); + let asked = aggregation::published_claims( + children, + &claims_of(raw_lean_xmss), + &claims_of(raw_lean_sphincs), + blobs_root.flatten(), + output, + ); + if blobs_root == Some(None) || asked.as_ref() != Ok(&claims) { + return Err(SidecarError::Protocol("a proof of other claims than asked for".into())); + } + Ok((claims, proof)) + } +} + +/// Room for a proof in a response, far more than one takes. +const PROOF_ROOM: u64 = 1 << 28; + +fn claims_of(raw: &[(K, [u8; 32], S)]) -> Vec<(K, [u8; 32])> { + raw.iter().map(|(pk, message, _)| (*pk, *message)).collect() +} diff --git a/crates/sidecar/src/lib.rs b/crates/sidecar/src/lib.rs new file mode 100644 index 0000000000..3a122e98ac --- /dev/null +++ b/crates/sidecar/src/lib.rs @@ -0,0 +1,82 @@ +//! The leanVM sidecar: a process that proves, and nothing else, for a client on the same machine. +//! +//! A client calls the sidecar's one operation, the library's `aggregate`, over a Unix stream socket, and verifies +//! the proofs it gets back with the library. The sidecar keeps the prover's allocator, threads and warm state out of +//! the client's process. `bins/leanvm-sidecar` runs [`serve`]; its `README.md` specifies the protocol, which any +//! language can speak, and [`Client`] speaks it from Rust. + +mod client; +mod server; +mod wire; + +pub use client::Client; +pub use server::{Config, serve}; + +/// Why the sidecar made no proof. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum Rejection { + /// [`aggregation::AggregateError::InvalidInput`], or more blobs than one proof takes. + InvalidInput = 1, + /// [`aggregation::AggregateError::InvalidSignature`]. + InvalidSignature = 2, + /// [`aggregation::AggregateError::InvalidChild`], or a child proof's bytes that do not decode. + InvalidChild = 3, + /// [`aggregation::AggregateError::ProvingFailed`]. + ProvingFailed = 4, + /// The request is not one: its bytes do not decode, or it is longer than the sidecar takes. + Malformed = 5, +} + +impl Rejection { + fn from_code(code: u8) -> Option { + [ + Self::InvalidInput, + Self::InvalidSignature, + Self::InvalidChild, + Self::ProvingFailed, + Self::Malformed, + ] + .into_iter() + .find(|r| *r as u8 == code) + } +} + +/// Why a [`Client`] call failed. +#[derive(Debug)] +pub enum SidecarError { + /// The socket failed, or the sidecar closed it: it stopped, or crashed. Connect again. + Io(std::io::Error), + /// The sidecar speaks another protocol version, or proves for another verifier or aggregation program. + Incompatible(String), + /// The sidecar's bytes are not the protocol, or prove other claims than asked for. Connect again. + Protocol(String), + /// The sidecar made no proof, and says why. The connection stays usable. + Rejected(Rejection, String), +} + +impl std::fmt::Display for SidecarError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Io(e) => write!(f, "sidecar connection: {e}"), + Self::Incompatible(why) => write!(f, "incompatible sidecar: {why}"), + Self::Protocol(why) => write!(f, "sidecar protocol: {why}"), + Self::Rejected(rejection, message) => write!(f, "sidecar refused ({rejection:?}): {message}"), + } + } +} + +impl std::error::Error for SidecarError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Io(e) => Some(e), + _ => None, + } + } +} + +impl From for SidecarError { + fn from(e: std::io::Error) -> Self { + Self::Io(e) + } +} diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs new file mode 100644 index 0000000000..2c23255ae6 --- /dev/null +++ b/crates/sidecar/src/server.rs @@ -0,0 +1,238 @@ +//! The server: a thread per connection reads and decodes requests, and the thread that calls [`serve`], the one +//! that started the prover's pool, proves them one at a time. + +use crate::Rejection; +use crate::wire::{self, Hello, Incoming, Outcome, Request}; +use aggregation::{AggregateError, OutputClaims}; +use isa::Prover; +use std::convert::Infallible; +use std::fs::{File, TryLockError}; +use std::io::{self, Write}; +use std::os::fd::AsRawFd; +use std::os::unix::fs::FileTypeExt; +use std::os::unix::net::{UnixListener, UnixStream}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::mpsc::{self, Receiver, Sender}; +use std::thread; +use std::time::Duration; + +/// How a sidecar serves. +#[derive(Clone, Debug)] +pub struct Config { + /// The socket to listen on. A stale one, left by a sidecar that stopped, is replaced. + pub socket: PathBuf, + /// The longest request body taken, in bytes. + pub max_request: u64, + /// The prover's thread count (0: one per core). + pub threads: usize, +} + +/// A request, its client's connection, and where its outcome goes. +type Job = (Request, UnixStream, Sender); + +/// Starts prover `P` and builds the aggregation program, then listens on `config.socket` and serves until the +/// process ends, proving on the calling thread. The socket appears once the sidecar is ready to prove. Returns only +/// if it cannot listen, which a second sidecar on the same socket learns before it starts anything. +/// +/// Call it from the main thread: a panic while proving, a bug, then ends the process. +pub fn serve(config: &Config) -> io::Result { + std::os::unix::net::SocketAddr::from_pathname(&config.socket)?; + let _lock = lock(&config.socket)?; + remove_stale(&config.socket)?; + if config.threads > 0 { + P::set_num_threads(config.threads); + } + P::init(); + let program = aggregation::unified_guest().bytecode().fiat_shamir_iv(); + let listener = UnixListener::bind(&config.socket)?; + let hello = wire::encode_hello(&Hello { + max_request: config.max_request, + tag: verifier::TAG.to_vec(), + program, + prover: P::NAME.into(), + }); + let (jobs, queue) = mpsc::channel(); + let max_request = config.max_request; + thread::Builder::new() + .name("accept".into()) + .spawn(move || accept(&listener, &jobs, hello.into(), max_request))?; + eprintln!( + "leanvm-sidecar: serving {} with the {} prover", + config.socket.display(), + P::NAME + ); + prove_forever::

(&queue) +} + +/// Locks `.lock`, which the process holds until it ends, so that one sidecar at a time owns the socket. +fn lock(socket: &Path) -> io::Result { + let mut path = socket.as_os_str().to_owned(); + path.push(".lock"); + let file = File::options().create(true).truncate(false).write(true).open(&path)?; + match file.try_lock() { + Ok(()) => Ok(file), + Err(TryLockError::WouldBlock) => Err(io::Error::new( + io::ErrorKind::AddrInUse, + format!("another sidecar serves {}", socket.display()), + )), + Err(TryLockError::Error(e)) => Err(e), + } +} + +/// Removes the socket a sidecar that stopped left, and nothing else: not another file, nor a socket something listens +/// on. Holding the lock, no other sidecar binds meanwhile. +fn remove_stale(socket: &Path) -> io::Result<()> { + let metadata = match std::fs::symlink_metadata(socket) { + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + metadata => metadata?, + }; + if !metadata.file_type().is_socket() { + return Err(io::Error::new( + io::ErrorKind::AlreadyExists, + format!("{} is not a socket", socket.display()), + )); + } + // Only a refused connection says nothing listens: one we may not open can be another program's. + match UnixStream::connect(socket) { + Err(e) if e.kind() == io::ErrorKind::ConnectionRefused => std::fs::remove_file(socket), + Ok(_) => Err(io::Error::new( + io::ErrorKind::AddrInUse, + format!("something else listens on {}", socket.display()), + )), + Err(e) => Err(e), + } +} + +fn prove_forever(queue: &Receiver) -> ! { + loop { + let (request, client, reply) = queue.recv().expect("the accept thread never ends"); + let summary = format!( + "{} children, {} leanXMSS, {} leanSPHINCS, {} blobs, rate {}", + request.children.len(), + request.raw_lean_xmss.len(), + request.raw_lean_sphincs.len(), + request.blobs.len(), + request.log_inv_rate, + ); + if hung_up(&client) { + eprintln!("{summary}: dropped, its client hung up"); + continue; + } + let outcome = aggregate::

(request); + match &outcome { + Ok(_) => eprintln!("{summary}: proven"), + Err((rejection, message)) => eprintln!("{summary}: refused ({rejection:?}): {message}"), + } + // The connection may have closed meanwhile. + let _ = reply.send(outcome); + } +} + +/// Whether the client closed its connection, so no longer waits for the request it sent: one that timed out, say. A +/// client that only shut down its writing half reads the same, as the protocol says. Nothing else reads the stream +/// meanwhile, its connection thread waiting on the outcome. +fn hung_up(client: &UnixStream) -> bool { + let mut byte = 0u8; + // SAFETY: a one-byte buffer of ours, on an open socket; peeking leaves the stream as it is. + let n = unsafe { + libc::recv( + client.as_raw_fd(), + (&raw mut byte).cast(), + 1, + libc::MSG_PEEK | libc::MSG_DONTWAIT, + ) + }; + match n { + 0 => true, + 1.. => false, + _ => !matches!( + io::Error::last_os_error().kind(), + io::ErrorKind::WouldBlock | io::ErrorKind::Interrupted + ), + } +} + +fn aggregate(request: Request) -> Outcome { + let blobs = match request.blobs.len() { + 0 => None, + _ => Some(lean_da::BlobMatrix::build(&request.blobs).map_err(|e| (Rejection::InvalidInput, e.to_string()))?), + }; + drop(request.blobs); + let output = request.output.as_ref().map_or(OutputClaims::All, OutputClaims::Only); + aggregation::aggregate::

( + &request.children, + request.raw_lean_xmss, + request.raw_lean_sphincs, + blobs.as_ref(), + output, + request.log_inv_rate, + ) + .map(|(claims, proof, _)| (claims, proof)) + .map_err(|e| match e { + AggregateError::InvalidInput(reason) => (Rejection::InvalidInput, reason.into()), + AggregateError::InvalidSignature => (Rejection::InvalidSignature, e.to_string()), + AggregateError::InvalidChild => (Rejection::InvalidChild, e.to_string()), + AggregateError::ProvingFailed => (Rejection::ProvingFailed, e.to_string()), + }) +} + +fn accept(listener: &UnixListener, jobs: &Sender, hello: Arc<[u8]>, max_request: u64) { + for stream in listener.incoming() { + match stream { + Ok(stream) => { + let (jobs, hello) = (jobs.clone(), hello.clone()); + let spawned = thread::Builder::new() + .name("connection".into()) + .spawn(move || connection(stream, &jobs, &hello, max_request)); + if let Err(e) = spawned { + eprintln!("leanvm-sidecar: a connection dropped: {e}"); + } + } + Err(e) => { + // Out of file descriptors, say: wait for some to close rather than spin. + eprintln!("leanvm-sidecar: accept: {e}"); + thread::sleep(Duration::from_millis(100)); + } + } + } +} + +/// Serves one connection: its hello, then each request in turn, until the client closes it. +fn connection(mut stream: UnixStream, jobs: &Sender, hello: &[u8], max_request: u64) { + if stream.write_all(hello).is_err() { + return; + } + loop { + let body = match wire::read_frame(&mut stream, max_request) { + Ok(Incoming::Body(body)) => body, + // The body is left unread, so the stream is lost: say why, and close. + Ok(Incoming::TooLarge(len)) => { + let message = format!("a request of {len} bytes, the sidecar takes at most {max_request}"); + eprintln!("a request refused (Malformed): {message}"); + let _ = stream.write_all(&wire::encode_response(&Err((Rejection::Malformed, message)))); + return; + } + Ok(Incoming::End) | Err(_) => return, + }; + let outcome = match wire::decode_request(&body) { + Ok(request) => { + drop(body); + let Ok(client) = stream.try_clone() else { return }; + let (reply, outcome) = mpsc::channel(); + jobs.send((request, client, reply)) + .expect("the prover outlives every connection"); + // No outcome: the prover dropped the request, its client having hung up. + let Ok(outcome) = outcome.recv() else { return }; + outcome + } + Err((rejection, message)) => { + eprintln!("a request refused ({rejection:?}): {message}"); + Err((rejection, message)) + } + }; + if stream.write_all(&wire::encode_response(&outcome)).is_err() { + return; + } + } +} diff --git a/crates/sidecar/src/wire.rs b/crates/sidecar/src/wire.rs new file mode 100644 index 0000000000..6b4c589d54 --- /dev/null +++ b/crates/sidecar/src/wire.rs @@ -0,0 +1,428 @@ +//! The protocol's bytes, as `bins/leanvm-sidecar/README.md` specifies them: frames, the hello, requests and +//! responses. Every integer is little-endian; every variable-length field is a `u64` length, then its bytes. + +use crate::Rejection; +use aggregation::{AggregateClaims, AggregateProof, OutputClaims}; +use lean_da::Blob; +use lean_sphincs::{LeanSphincsPublicKey, LeanSphincsSignature}; +use lean_xmss::{LeanXmssPublicKey, LeanXmssSignature}; +use std::io::{self, Read}; + +/// The hello's first bytes. +pub(crate) const MAGIC: [u8; 8] = *b"leanvmsc"; + +/// The protocol's version, which the hello announces right after [`MAGIC`], where every version keeps it. +pub(crate) const VERSION: u32 = 1; + +const OP_AGGREGATE: u8 = 1; +const OK: u8 = 0; + +const XMSS_RECORD: usize = lean_xmss::PUB_KEY_SIZE + lean_xmss::MESSAGE_LEN + lean_xmss::SIG_SIZE; +const SPHINCS_RECORD: usize = lean_sphincs::PUB_KEY_SIZE + lean_sphincs::MESSAGE_LEN + lean_sphincs::SIG_SIZE; +const BLOB_BYTES: usize = 16 * lean_da::BLOB_SYMBOLS; + +pub(crate) type RawLeanXmss = (LeanXmssPublicKey, lean_xmss::Message, LeanXmssSignature); +pub(crate) type RawLeanSphincs = (LeanSphincsPublicKey, lean_sphincs::Message, LeanSphincsSignature); +pub(crate) type Proven = (AggregateClaims, AggregateProof); +/// A request's proof, or why there is none. +pub(crate) type Outcome = Result; + +/// A frame being built: its length is patched in by [`Self::finish`]. +struct Frame(Vec); + +impl Frame { + fn new() -> Self { + Self(vec![0; 8]) + } + + fn u8(&mut self, x: u8) { + self.0.push(x); + } + + fn u64(&mut self, x: u64) { + self.0.extend(x.to_le_bytes()); + } + + fn raw(&mut self, bytes: &[u8]) { + self.0.extend(bytes); + } + + fn bytes(&mut self, bytes: &[u8]) { + self.u64(bytes.len() as u64); + self.raw(bytes); + } + + fn finish(mut self) -> Vec { + let len = (self.0.len() as u64 - 8).to_le_bytes(); + self.0[..8].copy_from_slice(&len); + self.0 + } +} + +/// What [`read_frame`] read. +pub(crate) enum Incoming { + Body(Vec), + /// The stream ended cleanly, between frames. + End, + /// A frame longer than the reader takes: its announced length. The body is left unread. + TooLarge(u64), +} + +/// Reads one frame. A body is read as it arrives, so a forged length allocates nothing it does not receive. +pub(crate) fn read_frame(r: &mut impl Read, max_len: u64) -> io::Result { + let mut len = [0; 8]; + let mut got = 0; + while got < len.len() { + match r.read(&mut len[got..]) { + Ok(0) if got == 0 => return Ok(Incoming::End), + Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()), + Ok(n) => got += n, + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + let len = u64::from_le_bytes(len); + if len > max_len { + return Ok(Incoming::TooLarge(len)); + } + let mut body = Vec::new(); + r.take(len).read_to_end(&mut body)?; + if body.len() as u64 != len { + return Err(io::ErrorKind::UnexpectedEof.into()); + } + Ok(Incoming::Body(body)) +} + +/// Reads fields off the front of a body. +struct Cursor<'a>(&'a [u8]); + +impl<'a> Cursor<'a> { + fn take(&mut self, n: usize) -> Option<&'a [u8]> { + let (head, rest) = self.0.split_at_checked(n)?; + self.0 = rest; + Some(head) + } + + fn array(&mut self) -> Option<&'a [u8; N]> { + self.take(N).map(|bytes| bytes.try_into().unwrap()) + } + + fn u8(&mut self) -> Option { + self.array::<1>().map(|[x]| *x) + } + + fn u32(&mut self) -> Option { + self.array().map(|x| u32::from_le_bytes(*x)) + } + + fn u64(&mut self) -> Option { + self.array().map(|x| u64::from_le_bytes(*x)) + } + + fn bytes(&mut self) -> Option<&'a [u8]> { + let len = usize::try_from(self.u64()?).ok()?; + self.take(len) + } + + /// A count of records of `size` bytes each, all of which must follow: a forged count allocates nothing. + fn count(&mut self, size: usize) -> Option { + let n = usize::try_from(self.u64()?).ok()?; + (n.checked_mul(size)? <= self.0.len()).then_some(n) + } + + /// A count of records of `size` bytes each, then the records, each read by `record`. + fn records(&mut self, size: usize, record: impl Fn(&mut Self) -> T) -> Option> { + let n = self.count(size)?; + Some((0..n).map(|_| record(self)).collect()) + } + + fn claims(&mut self) -> Option { + AggregateClaims::from_bytes(self.bytes()?) + } + + fn string(&mut self) -> Option { + String::from_utf8(self.bytes()?.to_vec()).ok() + } + + fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +/// What the server says first on every connection, after [`MAGIC`] and [`VERSION`]. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Hello { + pub max_request: u64, + pub tag: Vec, + /// The aggregation program's Fiat-Shamir IV, which pins the program and the verifier tag. + pub program: [u8; 32], + pub prover: String, +} + +pub(crate) fn encode_hello(hello: &Hello) -> Vec { + let mut f = Frame::new(); + f.raw(&MAGIC); + f.raw(&VERSION.to_le_bytes()); + f.u64(hello.max_request); + f.bytes(&hello.tag); + f.raw(&hello.program); + f.bytes(hello.prover.as_bytes()); + f.finish() +} + +/// The version a hello announces, `None` unless the body starts with [`MAGIC`]. +pub(crate) fn hello_version(body: &[u8]) -> Option { + let mut c = Cursor(body); + (c.array::<8>()? == &MAGIC).then(|| c.u32())? +} + +/// `None` unless the body is a whole hello of this version's layout. +pub(crate) fn decode_hello(body: &[u8]) -> Option { + let mut c = Cursor(body); + hello_version(c.take(12)?)?; + let hello = Hello { + max_request: c.u64()?, + tag: c.bytes()?.to_vec(), + program: *c.array()?, + prover: c.string()?, + }; + c.is_empty().then_some(hello) +} + +/// An `aggregate` call, as the server decodes it. +pub(crate) struct Request { + pub children: Vec, + pub raw_lean_xmss: Vec, + pub raw_lean_sphincs: Vec, + pub blobs: Vec, + /// `None` for [`OutputClaims::All`]. + pub output: Option, + pub log_inv_rate: usize, +} + +pub(crate) fn encode_request( + children: &[Proven], + raw_lean_xmss: &[RawLeanXmss], + raw_lean_sphincs: &[RawLeanSphincs], + blobs: &[Blob], + output: OutputClaims, + log_inv_rate: usize, +) -> Vec { + let mut f = Frame::new(); + f.u8(OP_AGGREGATE); + // A rate past a byte is out of range anyway: the server refuses it as it would any other. + f.u8(u8::try_from(log_inv_rate).unwrap_or(u8::MAX)); + f.u64(children.len() as u64); + for (claims, proof) in children { + f.bytes(&claims.to_bytes()); + f.bytes(&proof.to_bytes()); + } + f.u64(raw_lean_xmss.len() as u64); + for (pk, message, signature) in raw_lean_xmss { + f.raw(&pk.to_bytes()); + f.raw(message); + f.raw(&signature.to_bytes()); + } + f.u64(raw_lean_sphincs.len() as u64); + for (pk, message, signature) in raw_lean_sphincs { + f.raw(&pk.to_bytes()); + f.raw(message); + f.raw(&signature.to_bytes()); + } + f.u64(blobs.len() as u64); + for blob in blobs { + for symbol in blob { + f.raw(&symbol.to_le_bytes()); + } + } + match output { + OutputClaims::All => f.u8(0), + OutputClaims::Only(claims) => { + f.u8(1); + f.bytes(&claims.to_bytes()); + } + } + f.finish() +} + +pub(crate) fn decode_request(body: &[u8]) -> Result { + let malformed = |what: &str| (Rejection::Malformed, format!("malformed request: {what}")); + let mut c = Cursor(body); + match c.u8() { + Some(OP_AGGREGATE) => {} + Some(op) => return Err((Rejection::Malformed, format!("unknown operation {op}"))), + None => return Err(malformed("empty")), + } + let log_inv_rate = c.u8().ok_or_else(|| malformed("no rate"))?.into(); + + // Each child is at least its two lengths. Nothing is reserved: `aggregate` bounds the count, not this. + let n = c.count(16).ok_or_else(|| malformed("children"))?; + let mut children = Vec::new(); + for i in 0..n { + let claims = c.claims().ok_or_else(|| malformed(&format!("child {i}'s claims")))?; + let proof = c.bytes().ok_or_else(|| malformed(&format!("child {i}'s proof")))?; + let proof = AggregateProof::from_bytes(proof) + .ok_or_else(|| (Rejection::InvalidChild, format!("child {i}'s proof does not decode")))?; + children.push((claims, proof)); + } + + let raw_lean_xmss = c + .records(XMSS_RECORD, |c| { + let pk = LeanXmssPublicKey::from_bytes(c.array().unwrap()); + ( + pk, + *c.array().unwrap(), + LeanXmssSignature::from_bytes(c.array().unwrap()), + ) + }) + .ok_or_else(|| malformed("leanXMSS signatures"))?; + let raw_lean_sphincs = c + .records(SPHINCS_RECORD, |c| { + let pk = LeanSphincsPublicKey::from_bytes(c.array().unwrap()); + ( + pk, + *c.array().unwrap(), + LeanSphincsSignature::from_bytes(c.array().unwrap()), + ) + }) + .ok_or_else(|| malformed("leanSPHINCS signatures"))?; + + let n = c.count(BLOB_BYTES).ok_or_else(|| malformed("blobs"))?; + let mut blobs = vec![[0; lean_da::BLOB_SYMBOLS]; n]; + for blob in &mut blobs { + let bytes = c.take(BLOB_BYTES).unwrap(); + for (symbol, bytes) in blob.iter_mut().zip(bytes.as_chunks::<16>().0) { + *symbol = u128::from_le_bytes(*bytes); + } + } + + let output = match c.u8() { + Some(0) => None, + Some(1) => Some(c.claims().ok_or_else(|| malformed("output claims"))?), + _ => return Err(malformed("output")), + }; + if !c.is_empty() { + return Err(malformed("trailing bytes")); + } + Ok(Request { + children, + raw_lean_xmss, + raw_lean_sphincs, + blobs, + output, + log_inv_rate, + }) +} + +pub(crate) fn encode_response(result: &Outcome) -> Vec { + let mut f = Frame::new(); + match result { + Ok((claims, proof)) => { + f.u8(OK); + f.bytes(&claims.to_bytes()); + f.bytes(&proof.to_bytes()); + } + Err((rejection, message)) => { + f.u8(*rejection as u8); + f.bytes(message.as_bytes()); + } + } + f.finish() +} + +/// `None` if the body is not a response. +pub(crate) fn decode_response(body: &[u8]) -> Option { + let mut c = Cursor(body); + let result = match c.u8()? { + OK => Ok((c.claims()?, AggregateProof::from_bytes(c.bytes()?)?)), + status => Err((Rejection::from_code(status)?, c.string()?)), + }; + c.is_empty().then_some(result) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn body(frame: Vec) -> Vec { + assert_eq!( + u64::from_le_bytes(frame[..8].try_into().unwrap()), + frame.len() as u64 - 8 + ); + frame[8..].to_vec() + } + + #[test] + fn hello_round_trips() { + let hello = Hello { + max_request: 1 << 30, + tag: verifier::TAG.to_vec(), + program: [7; 32], + prover: "stable".into(), + }; + let bytes = body(encode_hello(&hello)); + assert_eq!(hello_version(&bytes), Some(VERSION)); + assert_eq!(decode_hello(&bytes), Some(hello)); + assert_eq!(decode_hello(&bytes[..bytes.len() - 1]), None); + assert_eq!(decode_hello(&[&bytes[..], &[0]].concat()), None); + let mut other = bytes.clone(); + other[0] ^= 1; + assert_eq!(hello_version(&other), None); + } + + /// A request decodes to what was encoded; every truncation, and a trailing byte, is refused, never a panic. + #[test] + fn requests_round_trip_and_truncations_are_refused() { + let xmss = ( + LeanXmssPublicKey::from_bytes(&[1; 32]), + [2; 32], + LeanXmssSignature::from_bytes(&std::array::from_fn(|i| i as u8)), + ); + let sphincs = ( + LeanSphincsPublicKey::from_bytes(&[3; 32]), + [4; 32], + LeanSphincsSignature::from_bytes(&std::array::from_fn(|i| (i * 7) as u8)), + ); + let blob: Blob = std::array::from_fn(|i| ((i as u128) << 100) | i as u128); + let only = AggregateClaims { + lean_xmss: vec![(xmss.0, xmss.1)], + lean_sphincs: vec![(sphincs.0, sphincs.1)], + da_commitment: Some([5; 32]), + }; + let frame = encode_request( + &[], + std::slice::from_ref(&xmss), + std::slice::from_ref(&sphincs), + &[blob], + OutputClaims::Only(&only), + 3, + ); + let bytes = body(frame); + let request = decode_request(&bytes).unwrap_or_else(|(_, m)| panic!("{m}")); + assert!(request.children.is_empty()); + assert_eq!(request.raw_lean_xmss, vec![xmss]); + assert_eq!(request.raw_lean_sphincs, vec![sphincs]); + assert_eq!(request.blobs, vec![blob]); + assert_eq!(request.output, Some(only)); + assert_eq!(request.log_inv_rate, 3); + for len in 0..bytes.len() { + assert!( + decode_request(&bytes[..len]).is_err(), + "a prefix of {len} bytes decoded" + ); + } + assert!(decode_request(&[&bytes[..], &[0]].concat()).is_err()); + // A count no body could hold. + let mut huge = vec![OP_AGGREGATE, 1]; + huge.extend(u64::MAX.to_le_bytes()); + assert!(decode_request(&huge).is_err()); + } + + #[test] + fn rejections_round_trip() { + let rejection = (Rejection::InvalidInput, "log_inv_rate out of range".to_string()); + let bytes = body(encode_response(&Err(rejection.clone()))); + assert_eq!(decode_response(&bytes).unwrap().unwrap_err(), rejection); + assert!(decode_response(&[42, 0, 0, 0, 0, 0, 0, 0, 0]).is_none()); + } +} diff --git a/crates/stable_prover/lean_vm_core/src/prover.rs b/crates/stable_prover/lean_vm_core/src/prover.rs index 5e4e97bb4e..d7ed580106 100644 --- a/crates/stable_prover/lean_vm_core/src/prover.rs +++ b/crates/stable_prover/lean_vm_core/src/prover.rs @@ -8,6 +8,8 @@ use verifier::E; pub struct StableProver; impl isa::Prover for StableProver { + const NAME: &'static str = "stable"; + fn init() { parallel::init(); } From d0380fa81ff028e2787e655a9564f45274a8f2d1 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Tue, 17 Nov 2026 11:06:30 -0500 Subject: [PATCH 04/34] sidecar: a verify operation, for clients without the library, taken from #768 Verify (operation 2) answers whether a proof proves claims, on the connection's thread rather than behind the proofs being made. https://github.com/leanEthereum/leanVM/pull/768 Co-Authored-By: vladimir-ea <85992906+vladimir-ea@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- bins/leanvm-sidecar/README.md | 31 ++++++++---- bins/leanvm-sidecar/tests/sidecar.rs | 6 ++- crates/sidecar/src/client.rs | 47 ++++++++++++------- crates/sidecar/src/lib.rs | 6 +-- crates/sidecar/src/server.rs | 18 +++---- crates/sidecar/src/wire.rs | 70 ++++++++++++++++++++++++---- 7 files changed, 133 insertions(+), 47 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 8bbb5d1515..2aaa4e4b8c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -52,7 +52,7 @@ Each tree holds, in dependency order, leaves first: | `keccak_gkr` | the prover of the VM's `KECCAK` argument | | `lean_vm_core` | arithmetization: tables, bus, constraints, `cpu::execute` and `cpu::prove` of an `isa::Program` and its witness streams; `cpu::verify` wraps `verifier`; the tree's `isa::Prover` | -Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. The sidecar is `bins/leanvm-sidecar`, which serves `aggregate` over a Unix socket so that clients in any language get the prover in a process of its own; its `README.md` is the protocol's specification, so a change to `crates/sidecar`'s `wire.rs`, or to the claims' byte form (`AggregateClaims::to_bytes`), lands there, and verifying stays in the library. +Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. The sidecar is `bins/leanvm-sidecar`, which serves `aggregate` over a Unix socket so that clients in any language get the prover in a process of its own; its `README.md` is the protocol's specification, so a change to `crates/sidecar`'s `wire.rs`, or to the claims' byte form (`AggregateClaims::to_bytes`), lands there. It also verifies, for clients without the library. ## Building and testing diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index a97aafdbbe..6de27db904 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -1,6 +1,6 @@ # leanvm-sidecar -A process that proves for clients on the same machine, over a Unix stream socket. It runs the library's one proving call, `aggregate`, so a client in any language gets the prover's full speed, with its own allocator, thread pool and retained memory, in a process of its own. Verifying stays in the client, with the library (`AggregateProof::verify`), which is fast and needs no prover: from another language, through a binding to it. +A process that proves for clients on the same machine, over a Unix stream socket. It runs the library's one proving call, `aggregate`, so a client in any language gets the prover's full speed, with its own allocator, thread pool and retained memory, in a process of its own. It also verifies proofs, for clients without the library; a Rust client verifies with the library (`AggregateProof::verify`), which is faster. ## Running @@ -17,7 +17,7 @@ target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar | `--threads` | 0 | the prover's thread count, 0 for one per core | | `--tracing` | off | print each proof's timing tree (`RUST_LOG` sets the level) | -- The sidecar starts the prover's thread pool and compiles the aggregation program before it listens, so the socket appears once it is ready. It says so on standard error, then logs one line per request there. +- The sidecar starts the prover's thread pool, compiles the aggregation program before it listens, so the socket appears once it is ready. It says so on standard error, then logs one line per `aggregate` request there. - A sidecar holds a lock on the file beside its socket, the socket's path with `.lock` appended, while it runs. A second sidecar on the same socket fails at once. A socket left by a sidecar that stopped is replaced; a path that is some other file, or a socket another program listens on, is not. - The socket takes the process's umask, and connecting needs write permission on it. Restrict it to the clients the sidecar serves: it trusts them with its resources. Any of them can keep it proving, and every connection may hold a request up to the max request in memory. - Requests are proven one at a time, across all connections, in the order they are received in full. A request whose client disconnects before its proof starts is dropped; one already proving finishes, and its proof is dropped. A client that gives up on a request should therefore close its connection, and send the request again, if at all, on a new one. @@ -56,13 +56,15 @@ On each connection the sidecar first sends a hello frame. The client then sends Every version keeps the magic and the version first. A client checks them, the verifier tag and the program, and disconnects if any differs from what it expects: a proof for another verifier or another build's program does not verify with its library. A client must not send a request body longer than the max request: the sidecar would close the connection without reading it, and the client could fail mid-write. -### Request +### Requests -The library's `aggregate` arguments: +A request's first byte is its operation: 1 for `aggregate`, 2 for `verify`. + +**Aggregate**, the library's `aggregate` arguments: | field | type | meaning | | --- | --- | --- | -| operation | `u8` | 1, `aggregate` | +| operation | `u8` | 1 | | rate | `u8` | `log_inv_rate`, 1 to 4: higher gives a smaller proof, slower | | children | count, then per child: claims (bytes), proof (bytes) | at most 16 | | leanXMSS signatures | count, then per signature: pubkey (32 bytes), message (32 bytes), signature (2404 bytes) | | @@ -70,7 +72,17 @@ The library's `aggregate` arguments: | blobs | count, then per blob: 8192 symbols of 16 bytes each | at most 32 | | output | `u8`, then for 1 the claims (bytes) | 0: every covered claim; 1: exactly these claims, each covered by the inputs | -Keys, signatures and proofs are the libraries' byte forms (`to_bytes`), and a symbol is its 128 bits as a little-endian `u128`, as LeanDA hashes it. Signatures may come in any order, and a repeated claim, the same pubkey and message twice, counts once. As with the library, the sidecar does not verify children: an invalid one costs a failed proof, status 3 or 4, so a child from an untrusted source is checked with `AggregateProof::verify` first. +Keys, signatures and proofs are the libraries' byte forms (`to_bytes`), and a symbol is its 128 bits as a little-endian `u128`, as LeanDA hashes it. Signatures may come in any order, and a repeated claim, the same pubkey and message twice, counts once. As with the library, the sidecar does not verify children: an invalid one costs a failed proof, status 3 or 4, so a child from an untrusted source is verified first. + +**Verify**, whether a proof proves claims, as `AggregateProof::verify` says. It does not wait for the proofs being made. + +| field | type | +| --- | --- | +| operation | `u8`, 2 | +| claims | bytes | +| proof | bytes | + +A proof whose bytes do not decode is invalid; claims that do not decode make the request malformed. **Claims**, as `AggregateClaims::to_bytes` writes them: @@ -87,8 +99,9 @@ A proof verifies only against claims whose lists are strictly sorted, comparing | field | type | meaning | | --- | --- | --- | | status | `u8` | below | -| for 0: claims, proof | bytes, bytes | what the proof proves, and the proof (`AggregateProof::from_bytes` reads it) | -| otherwise: message | bytes | UTF-8, why there is no proof | +| for 0, to `aggregate`: claims, proof | bytes, bytes | what the proof proves, and the proof (`AggregateProof::from_bytes` reads it) | +| for 0, to `verify`: valid | `u8` | 1 if the proof proves the claims, else 0 | +| otherwise: message | bytes | UTF-8, why there is no answer | | status | name | meaning | | --- | --- | --- | @@ -99,6 +112,6 @@ A proof verifies only against claims whose lists are strictly sorted, comparing | 4 | proving failed | an input the checks above missed is invalid, or the request is too large for one proof | | 5 | malformed | the request does not decode, or is longer than the hello's max request | -A request with several problems gets the status of the first one found. +A request with several problems gets the status of the first one found. A `verify` request gets only 0 or 5. After any response the connection stays usable, except after a request longer than the max request: the sidecar answers it with status 5 without reading it, then closes the connection. diff --git a/bins/leanvm-sidecar/tests/sidecar.rs b/bins/leanvm-sidecar/tests/sidecar.rs index 0c45b6fa70..fb65245eea 100644 --- a/bins/leanvm-sidecar/tests/sidecar.rs +++ b/bins/leanvm-sidecar/tests/sidecar.rs @@ -1,5 +1,5 @@ //! The sidecar binary end to end, through the library's client: a leaf with every kind of input, a narrowed node over -//! it from a second connection, refusals that leave the connection usable, a request that is not one sent as another +//! it from a second connection, verifying, refusals that leave the connection usable, a request that is not one sent as another //! language's client would, and the socket's life across a second sidecar and a restart. use aggregation::signers_cache; @@ -97,6 +97,10 @@ fn sidecar_end_to_end() { assert_eq!(node.0, only); assert!(node.1.verify(&only)); + // Verifying, from either connection, while nothing proves. + assert!(client.verify(&node.0, &node.1).unwrap()); + assert!(!other.verify(&AggregateClaims::default(), &node.1).unwrap()); + // Refusals leave the connection in step: two in a row. for rate in [0, 300] { match client.aggregate(&[], &xmss[..1], &[], &[], OutputClaims::All, rate) { diff --git a/crates/sidecar/src/client.rs b/crates/sidecar/src/client.rs index 57c0a406f1..54005d3358 100644 --- a/crates/sidecar/src/client.rs +++ b/crates/sidecar/src/client.rs @@ -93,23 +93,8 @@ impl Client { log_inv_rate: usize, ) -> Result<(AggregateClaims, AggregateProof), SidecarError> { let request = wire::encode_request(children, raw_lean_xmss, raw_lean_sphincs, blobs, output, log_inv_rate); - let len = request.len() as u64 - 8; - if len > self.max_request { - return Err(SidecarError::Rejected( - Rejection::Malformed, - format!( - "a request of {len} bytes, the sidecar takes at most {}", - self.max_request - ), - )); - } - self.stream.write_all(&request)?; // The claims a response carries are no longer than the request. - let body = match wire::read_frame(&mut self.stream, len + PROOF_ROOM)? { - Incoming::Body(body) => body, - Incoming::End => return Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof).into()), - Incoming::TooLarge(_) => return Err(SidecarError::Protocol("a response longer than any".into())), - }; + let body = self.call(&request, request.len() as u64 + PROOF_ROOM)?; let (claims, proof) = match wire::decode_response(&body) { Some(Ok(proven)) => proven, Some(Err((rejection, message))) => return Err(SidecarError::Rejected(rejection, message)), @@ -129,6 +114,36 @@ impl Client { } Ok((claims, proof)) } + + /// The sidecar's verify operation, for clients without the library: from Rust, [`AggregateProof::verify`] is faster. + pub fn verify(&mut self, claims: &AggregateClaims, proof: &AggregateProof) -> Result { + let body = self.call(&wire::encode_verify(claims, proof), 1 << 16)?; + match wire::decode_verdict(&body) { + Some(Ok(valid)) => Ok(valid), + Some(Err((rejection, message))) => Err(SidecarError::Rejected(rejection, message)), + None => Err(SidecarError::Protocol("a response that does not decode".into())), + } + } + + /// Sends a request frame and reads the response's body, at most `max_response` bytes. + fn call(&mut self, request: &[u8], max_response: u64) -> Result, SidecarError> { + let len = request.len() as u64 - 8; + if len > self.max_request { + return Err(SidecarError::Rejected( + Rejection::Malformed, + format!( + "a request of {len} bytes, the sidecar takes at most {}", + self.max_request + ), + )); + } + self.stream.write_all(request)?; + match wire::read_frame(&mut self.stream, max_response)? { + Incoming::Body(body) => Ok(body), + Incoming::End => Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof).into()), + Incoming::TooLarge(_) => Err(SidecarError::Protocol("a response longer than any".into())), + } + } } /// Room for a proof in a response, far more than one takes. diff --git a/crates/sidecar/src/lib.rs b/crates/sidecar/src/lib.rs index 3a122e98ac..40110ea7e2 100644 --- a/crates/sidecar/src/lib.rs +++ b/crates/sidecar/src/lib.rs @@ -1,7 +1,7 @@ -//! The leanVM sidecar: a process that proves, and nothing else, for a client on the same machine. +//! The leanVM sidecar: a process that proves for clients on the same machine. //! -//! A client calls the sidecar's one operation, the library's `aggregate`, over a Unix stream socket, and verifies -//! the proofs it gets back with the library. The sidecar keeps the prover's allocator, threads and warm state out of +//! A client calls the library's `aggregate` over a Unix stream socket, and verifies proofs with the library, or with +//! the sidecar if it has no binding to it. The sidecar keeps the prover's allocator, threads and warm state out of //! the client's process. `bins/leanvm-sidecar` runs [`serve`]; its `README.md` specifies the protocol, which any //! language can speak, and [`Client`] speaks it from Rust. diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs index 2c23255ae6..10e106859f 100644 --- a/crates/sidecar/src/server.rs +++ b/crates/sidecar/src/server.rs @@ -1,8 +1,8 @@ -//! The server: a thread per connection reads and decodes requests, and the thread that calls [`serve`], the one -//! that started the prover's pool, proves them one at a time. +//! The server: a thread per connection reads and decodes requests and verifies proofs, and the thread that calls +//! [`serve`], the one that started the prover's pool, proves them one at a time. use crate::Rejection; -use crate::wire::{self, Hello, Incoming, Outcome, Request}; +use crate::wire::{self, Call, Hello, Incoming, Outcome, Request}; use aggregation::{AggregateError, OutputClaims}; use isa::Prover; use std::convert::Infallible; @@ -215,8 +215,8 @@ fn connection(mut stream: UnixStream, jobs: &Sender, hello: &[u8], max_requ } Ok(Incoming::End) | Err(_) => return, }; - let outcome = match wire::decode_request(&body) { - Ok(request) => { + let response = match wire::decode_request(&body) { + Ok(Call::Aggregate(request)) => { drop(body); let Ok(client) = stream.try_clone() else { return }; let (reply, outcome) = mpsc::channel(); @@ -224,14 +224,16 @@ fn connection(mut stream: UnixStream, jobs: &Sender, hello: &[u8], max_requ .expect("the prover outlives every connection"); // No outcome: the prover dropped the request, its client having hung up. let Ok(outcome) = outcome.recv() else { return }; - outcome + wire::encode_response(&outcome) } + // Here rather than behind the proofs: it is fast, and on one thread. + Ok(Call::Verify(claims, proof)) => wire::encode_verdict(proof.is_some_and(|proof| proof.verify(&claims))), Err((rejection, message)) => { eprintln!("a request refused ({rejection:?}): {message}"); - Err((rejection, message)) + wire::encode_response(&Err((rejection, message))) } }; - if stream.write_all(&wire::encode_response(&outcome)).is_err() { + if stream.write_all(&response).is_err() { return; } } diff --git a/crates/sidecar/src/wire.rs b/crates/sidecar/src/wire.rs index 6b4c589d54..7e3375511f 100644 --- a/crates/sidecar/src/wire.rs +++ b/crates/sidecar/src/wire.rs @@ -15,6 +15,7 @@ pub(crate) const MAGIC: [u8; 8] = *b"leanvmsc"; pub(crate) const VERSION: u32 = 1; const OP_AGGREGATE: u8 = 1; +const OP_VERIFY: u8 = 2; const OK: u8 = 0; const XMSS_RECORD: usize = lean_xmss::PUB_KEY_SIZE + lean_xmss::MESSAGE_LEN + lean_xmss::SIG_SIZE; @@ -189,7 +190,14 @@ pub(crate) fn decode_hello(body: &[u8]) -> Option { c.is_empty().then_some(hello) } -/// An `aggregate` call, as the server decodes it. +/// A request, as the server decodes it. +pub(crate) enum Call { + Aggregate(Request), + /// Claims, and their proof unless its bytes do not decode, which makes it invalid. + Verify(AggregateClaims, Option), +} + +/// An `aggregate` call. pub(crate) struct Request { pub children: Vec, pub raw_lean_xmss: Vec, @@ -245,14 +253,37 @@ pub(crate) fn encode_request( f.finish() } -pub(crate) fn decode_request(body: &[u8]) -> Result { +pub(crate) fn encode_verify(claims: &AggregateClaims, proof: &AggregateProof) -> Vec { + let mut f = Frame::new(); + f.u8(OP_VERIFY); + f.bytes(&claims.to_bytes()); + f.bytes(&proof.to_bytes()); + f.finish() +} + +pub(crate) fn decode_request(body: &[u8]) -> Result { let malformed = |what: &str| (Rejection::Malformed, format!("malformed request: {what}")); let mut c = Cursor(body); - match c.u8() { - Some(OP_AGGREGATE) => {} + let call = match c.u8() { + Some(OP_AGGREGATE) => Call::Aggregate(decode_aggregate(&mut c, malformed)?), + Some(OP_VERIFY) => { + let claims = c.claims().ok_or_else(|| malformed("claims"))?; + let proof = c.bytes().ok_or_else(|| malformed("proof"))?; + Call::Verify(claims, AggregateProof::from_bytes(proof)) + } Some(op) => return Err((Rejection::Malformed, format!("unknown operation {op}"))), None => return Err(malformed("empty")), + }; + if !c.is_empty() { + return Err(malformed("trailing bytes")); } + Ok(call) +} + +fn decode_aggregate( + c: &mut Cursor, + malformed: impl Fn(&str) -> (Rejection, String), +) -> Result { let log_inv_rate = c.u8().ok_or_else(|| malformed("no rate"))?.into(); // Each child is at least its two lengths. Nothing is reserved: `aggregate` bounds the count, not this. @@ -301,9 +332,6 @@ pub(crate) fn decode_request(body: &[u8]) -> Result Some(c.claims().ok_or_else(|| malformed("output claims"))?), _ => return Err(malformed("output")), }; - if !c.is_empty() { - return Err(malformed("trailing bytes")); - } Ok(Request { children, raw_lean_xmss, @@ -330,7 +358,29 @@ pub(crate) fn encode_response(result: &Outcome) -> Vec { f.finish() } -/// `None` if the body is not a response. +/// A verify request's response: whether the proof is valid. +pub(crate) fn encode_verdict(valid: bool) -> Vec { + let mut f = Frame::new(); + f.u8(OK); + f.u8(valid.into()); + f.finish() +} + +/// `None` if the body is not a verify request's response. +pub(crate) fn decode_verdict(body: &[u8]) -> Option> { + let mut c = Cursor(body); + let result = match c.u8()? { + OK => Ok(match c.u8()? { + 0 => false, + 1 => true, + _ => return None, + }), + status => Err((Rejection::from_code(status)?, c.string()?)), + }; + c.is_empty().then_some(result) +} + +/// `None` if the body is not an aggregate request's response. pub(crate) fn decode_response(body: &[u8]) -> Option { let mut c = Cursor(body); let result = match c.u8()? { @@ -398,7 +448,9 @@ mod tests { 3, ); let bytes = body(frame); - let request = decode_request(&bytes).unwrap_or_else(|(_, m)| panic!("{m}")); + let Ok(Call::Aggregate(request)) = decode_request(&bytes) else { + panic!("not an aggregate request") + }; assert!(request.children.is_empty()); assert_eq!(request.raw_lean_xmss, vec![xmss]); assert_eq!(request.raw_lean_sphincs, vec![sphincs]); From 176446de0cdb4db1676fd68780de05a5ad14055b Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Tue, 17 Nov 2026 11:06:51 -0500 Subject: [PATCH 05/34] sidecar: warm up before serving, taken from #768 Two discarded proofs before the socket appears, a one-blob leaf and a 16-child recursion over it at the highest rate, so the first requests no longer pay for first-touched memory. On an M-series Mac, experimental prover: the first 900-signature leaf at rate 1 went from 12-30% slower than the next ones to 4%, the first 16-child recursion from 7-70% to 2%, rates 1 and 2 both covered; the warm-up costs about 1.8 s at startup. https://github.com/leanEthereum/leanVM/pull/768 Co-Authored-By: vladimir-ea <85992906+vladimir-ea@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- bins/leanvm-sidecar/README.md | 4 ++-- crates/sidecar/src/server.rs | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index 6de27db904..4009d955db 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -17,12 +17,12 @@ target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar | `--threads` | 0 | the prover's thread count, 0 for one per core | | `--tracing` | off | print each proof's timing tree (`RUST_LOG` sets the level) | -- The sidecar starts the prover's thread pool, compiles the aggregation program before it listens, so the socket appears once it is ready. It says so on standard error, then logs one line per `aggregate` request there. +- The sidecar starts the prover's thread pool, compiles the aggregation program and warms up before it listens, so the socket appears once it is ready. It says so on standard error, then logs one line per `aggregate` request there. +- The warm-up, two discarded proofs, makes the first proofs as fast as the next ones: without it they pay for touching the memory the prover then keeps. A proof larger than any before it still pays for its extra memory. - A sidecar holds a lock on the file beside its socket, the socket's path with `.lock` appended, while it runs. A second sidecar on the same socket fails at once. A socket left by a sidecar that stopped is replaced; a path that is some other file, or a socket another program listens on, is not. - The socket takes the process's umask, and connecting needs write permission on it. Restrict it to the clients the sidecar serves: it trusts them with its resources. Any of them can keep it proving, and every connection may hold a request up to the max request in memory. - Requests are proven one at a time, across all connections, in the order they are received in full. A request whose client disconnects before its proof starts is dropped; one already proving finishes, and its proof is dropped. A client that gives up on a request should therefore close its connection, and send the request again, if at all, on a new one. - Stop the sidecar with a signal: it holds nothing to save. A request in flight is lost, and its client sees the connection close. The same goes if the prover panics, which exits the process: run the sidecar under a supervisor that restarts it. -- The first proof of a fresh sidecar is a little slower than the next ones. ## Clients diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs index 10e106859f..c7a513f9ab 100644 --- a/crates/sidecar/src/server.rs +++ b/crates/sidecar/src/server.rs @@ -31,7 +31,7 @@ pub struct Config { /// A request, its client's connection, and where its outcome goes. type Job = (Request, UnixStream, Sender); -/// Starts prover `P` and builds the aggregation program, then listens on `config.socket` and serves until the +/// Starts prover `P`, builds the aggregation program and warms up, then listens on `config.socket` and serves until the /// process ends, proving on the calling thread. The socket appears once the sidecar is ready to prove. Returns only /// if it cannot listen, which a second sidecar on the same socket learns before it starts anything. /// @@ -45,6 +45,7 @@ pub fn serve(config: &Config) -> io::Result { } P::init(); let program = aggregation::unified_guest().bytecode().fiat_shamir_iv(); + warm_up::

(); let listener = UnixListener::bind(&config.socket)?; let hello = wire::encode_hello(&Hello { max_request: config.max_request, @@ -65,6 +66,19 @@ pub fn serve(config: &Config) -> io::Result { prove_forever::

(&queue) } +/// A fresh process's first proofs are slower, touching memory the allocator then keeps: this pays for it before the +/// first request, with a discarded leaf and a full recursion over it, at the highest rate, whose memory is the +/// largest. A blob is the one input that needs no key. +fn warm_up() { + let blobs = lean_da::BlobMatrix::build(&[[0; lean_da::BLOB_SYMBOLS]]).expect("one blob"); + let rate = aggregation::MAX_LOG_INV_RATE; + let leaf = aggregation::aggregate::

(&[], vec![], vec![], Some(&blobs), OutputClaims::All, rate) + .expect("the warm-up's leaf proves"); + let children = vec![(leaf.0, leaf.1); aggregation::MAX_RECURSIONS]; + aggregation::aggregate::

(&children, vec![], vec![], None, OutputClaims::All, rate) + .expect("the warm-up's recursion proves"); +} + /// Locks `.lock`, which the process holds until it ends, so that one sidecar at a time owns the socket. fn lock(socket: &Path) -> io::Result { let mut path = socket.as_os_str().to_owned(); From cbe67fc302173fc31400482697dcc771317734b3 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Tue, 17 Nov 2026 15:21:04 -0500 Subject: [PATCH 06/34] review fixes: bench.sh runs from the repo root again, and doc touch-ups bench/bench.sh moved up a level with the rename but still went up two, leaving the repository. The rest is documentation: setup's first sentence ran into the next, AggregateClaims::to_bytes had no doc, Client::verify's doc spoke of clients without the library, and the sidecar test's header and two doc lines were rewrapped. Co-Authored-By: Claude Opus 5.5 --- bench/bench.sh | 2 +- bins/leanvm-sidecar/tests/sidecar.rs | 4 ++-- crates/aggregation/src/proof.rs | 1 + crates/lean_vm/src/lib.rs | 4 ++-- crates/sidecar/src/client.rs | 4 ++-- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/bench/bench.sh b/bench/bench.sh index 94ac33a34f..6b62878f57 100755 --- a/bench/bench.sh +++ b/bench/bench.sh @@ -7,7 +7,7 @@ set -uo pipefail PROVER=${1:-} case $PROVER in stable | experimental) ;; *) echo "usage: bench/bench.sh stable|experimental [log file]" >&2; exit 2 ;; esac -cd "$(dirname "$0")/../.." +cd "$(dirname "$0")/.." mkdir -p .build LOG="${2:-$PWD/.build/bench-$(date +%Y%m%d-%H%M%S).log}" RESULTS=$(mktemp) diff --git a/bins/leanvm-sidecar/tests/sidecar.rs b/bins/leanvm-sidecar/tests/sidecar.rs index fb65245eea..7887e4022a 100644 --- a/bins/leanvm-sidecar/tests/sidecar.rs +++ b/bins/leanvm-sidecar/tests/sidecar.rs @@ -1,6 +1,6 @@ //! The sidecar binary end to end, through the library's client: a leaf with every kind of input, a narrowed node over -//! it from a second connection, verifying, refusals that leave the connection usable, a request that is not one sent as another -//! language's client would, and the socket's life across a second sidecar and a restart. +//! it from a second connection, verifying, refusals that leave the connection usable, an unknown operation sent as +//! another language's client would, and the socket's life across a second sidecar and a restart. use aggregation::signers_cache; use lean_vm::sidecar::{Client, Rejection, SidecarError}; diff --git a/crates/aggregation/src/proof.rs b/crates/aggregation/src/proof.rs index ed2d4a9c6f..8dc997c11a 100644 --- a/crates/aggregation/src/proof.rs +++ b/crates/aggregation/src/proof.rs @@ -97,6 +97,7 @@ impl AggregateClaims { self.lean_xmss.len() + self.lean_sphincs.len() } + /// Encode claims, which [`Self::from_bytes`] reads back. pub fn to_bytes(&self) -> Vec { let mut out = Vec::new(); encode_claims(&mut out, &self.lean_xmss, lean_xmss::LeanXmssPublicKey::to_bytes); diff --git a/crates/lean_vm/src/lib.rs b/crates/lean_vm/src/lib.rs index 879deff7f4..ec32d52fbf 100644 --- a/crates/lean_vm/src/lib.rs +++ b/crates/lean_vm/src/lib.rs @@ -50,8 +50,8 @@ pub mod sidecar { pub use ::sidecar::{Client, Rejection, SidecarError}; } -/// Call this function before generating / verifying proofs -/// Starts the prover's thread pool, with `threads` threads (0: one per core), and builds the aggregation program. +/// Call this function before generating / verifying proofs: it starts the prover's thread pool, with `threads` threads +/// (0: one per core), and builds the aggregation program. /// /// # Panics /// If the pool already runs with another thread count. diff --git a/crates/sidecar/src/client.rs b/crates/sidecar/src/client.rs index 54005d3358..d02af67619 100644 --- a/crates/sidecar/src/client.rs +++ b/crates/sidecar/src/client.rs @@ -82,7 +82,7 @@ impl Client { /// go as they are (none for no blobs) and which the sidecar builds the matrix of. /// /// As with the library, children must be valid (checked with [`AggregateProof::verify`]), or proving fails. The - /// claims returned are checked to be those asked for, but for the root of `blobs`, which is only checked to be there. + /// claims returned are checked to be those asked for, but for the root of `blobs`, only checked to be there. pub fn aggregate( &mut self, children: &[(AggregateClaims, AggregateProof)], @@ -115,7 +115,7 @@ impl Client { Ok((claims, proof)) } - /// The sidecar's verify operation, for clients without the library: from Rust, [`AggregateProof::verify`] is faster. + /// The sidecar's verify operation, which [`AggregateProof::verify`] answers faster in this process. pub fn verify(&mut self, claims: &AggregateClaims, proof: &AggregateProof) -> Result { let body = self.call(&wire::encode_verify(claims, proof), 1 << 16)?; match wire::decode_verdict(&body) { From 97b518425b4db55bdf324a33876c11617f06c6dd Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Tue, 17 Nov 2026 16:20:01 -0500 Subject: [PATCH 07/34] AGENTS.md: the verifier's Keccak under Zen 4 tuning Built for znver4 (target-cpu=native on Zen 4), LLVM's SLP vectorizer packs the verifier's Keccak-f lanes 4 or 8 wide and spends the gain on cross-lane shuffles. On h1 (Ryzen 7 PRO 8700GE), one permutation takes 437 ns, against 196 ns with SLP off and 197 ns with -slp-max-vf=2; generic x86-64 (232 ns), x86-64-v3 (205 ns) and x86-64-v4 (203 ns) are unaffected, as is an M4 Max (136 ns). Decoding a 1000-leanXMSS leaf proof, which rebuilds its pruned Merkle paths, and verifying it take 2.96 and 8.12 ms there, against 1.60 and 6.37 ms built without AVX-512. Tried in the source, none kept: a black_box barrier per round (Zen 4 down to 256 ns at best, other targets 15 to 60% slower), row and column loops the compiler cannot unroll (2 to 4 times slower everywhere), and fully unrolled locals or the shared crate's row schedule (vectorized the same). Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 2aaa4e4b8c..e207bd6bea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -135,7 +135,7 @@ In the recursion program, the prover sends the verifier advice, called hints. Hi - **A hot NEON helper that is `#[inline(always)]` stays so.** A `#[target_feature]` function cannot be forced inline, and LLVM then leaves field helpers out of line in hot loops. Split a product's halves by hand rather than with `array::map`. A helper that needs `unsafe` is an `unsafe fn` its callers wrap. - **On Zen 4, 512-bit cross-lane moves are half-rate**: prefer kernels where both qwords of every 128-bit lane carry a product and nothing crosses lanes. - **On Zen 5 the carry-less multiplier is the scarce unit**: reduce modulo `x^64 + x^4 + x^3 + x + 1` with shifts and XORs (`gf2_64::reduce`), and count CLMUL instructions, not products. The exception is the scalar `F64` product (`gf2_64::x86_64::mul`): one CLMUL fold, then a byte lookup of its spill (CLHASH's reduction). -- **`crates/verifier` is still auto-vectorized**: on AVX-512, LLVM turns table lookups into gathers that lose to scalar loads. Check its x86 disassembly after touching a lookup. +- **`crates/verifier` is still auto-vectorized**: on AVX-512, LLVM turns table lookups into gathers that lose to scalar loads. Check its x86 disassembly after touching a lookup. Tuned for Zen 4 (`znver4`, which `native` is there), the SLP vectorizer also packs the Keccak permutation's lanes into vectors and spends the gain on shuffles, slower than scalar; no portable rewrite tried avoids it without slowing other targets, and `-C llvm-args=-slp-max-vf=2` measures the verifier without it. - **Fiat-Shamir:** `ProverState::add_scalar` and `Transcript::next_scalar` bind into the transcript as a side effect, and the public statement seeds it at construction. Never re-observe data that rode the stream. - **Provers and verifiers derive the layout identically** from announced sizes: a change to `placements_of` or the schema lands in both provers and all three verifiers (`crates/verifier`'s `layout.rs` is the native one). `col_kappas` is derived from `col_kappa_sources`; keep it that way. - **The opening's lane fold binds the committed witness's top variables** (`whir::commit` documents the layout), so every verifier rotates the terminal point left by `INITIAL_FOLDING_FACTOR` before evaluating the basis, while every other read of the opening's point stays in round order. From 9e31e9e0dca1af2fb73c251b54e35cc5f63355a4 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Tue, 17 Nov 2026 17:08:36 -0500 Subject: [PATCH 08/34] sidecar: a client that builds no prover, proving and verifying through the sidecar lean_vm gets a default `prover` feature. Without it, it builds no prover tree, no aggregation, lean_da or zkDSL compiler: it offers the sidecar's client, the claim and signature types and the rate bounds, and proves and verifies through the sidecar alone, so not while the sidecar is down. With it, the library proves and verifies in the process as before; experimental_prover implies it. - claims (new): AggregateClaims and its byte form, OutputClaims and the two claim aliases, moved out of aggregation, which re-exports them, so that both the library and the client name them. It depends on lean_xmss and lean_sphincs alone. - sidecar: the protocol and the client by default; the server, and the protocol's server half, behind a `server` feature, which bins/leanvm-sidecar turns on. The client takes and returns proofs as bytes, so that its API is the same with or without the prover, and no longer checks the program IV or the claims a response carries: it trusts the sidecar it verifies through. aggregation's published_claims, which only that check used, goes, and node.rs is main's again. - formal's guest export, which compiles aggregation's sources, depends on claims and imports the two key types aggregation no longer brings. - CI runs clippy on lean_vm without default features. Verifying through the sidecar costs about what verifying in-process does once the proof is decoded, which a client receiving bytes pays either way: for a 1000-leanXMSS leaf at rate 1 (277 KiB), 6.7 ms through the sidecar against 6.5 ms decode and verify in-process on an M4 Max, and 11.1 against 11.2 ms on a Ryzen 7 PRO 8700GE. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/rust.yml | 2 + AGENTS.md | 7 +- Cargo.lock | 12 + Cargo.toml | 2 + bins/leanvm-sidecar/Cargo.toml | 2 +- bins/leanvm-sidecar/README.md | 9 +- bins/leanvm-sidecar/tests/sidecar.rs | 6 +- crates/aggregation/Cargo.toml | 1 + crates/aggregation/src/coverage.rs | 4 +- crates/aggregation/src/lib.rs | 14 +- crates/aggregation/src/node.rs | 108 +++---- crates/aggregation/src/proof.rs | 93 +----- crates/aggregation/src/statement.rs | 3 +- crates/aggregation/src/tests.rs | 2 +- crates/claims/Cargo.toml | 12 + crates/claims/src/lib.rs | 99 ++++++ crates/lean_vm/Cargo.toml | 15 +- crates/lean_vm/src/lib.rs | 79 +---- crates/lean_vm/src/prover.rs | 61 ++++ crates/sidecar/Cargo.toml | 13 +- crates/sidecar/src/client.rs | 63 ++-- crates/sidecar/src/lib.rs | 27 +- crates/sidecar/src/server.rs | 279 ++++++++++++++++- crates/sidecar/src/wire.rs | 296 ++---------------- .../lean_vm/scripts/guest-export/Cargo.lock | 9 + .../lean_vm/scripts/guest-export/Cargo.toml | 1 + .../guest-export/src/export/guest_diff.rs | 4 +- .../guest-export/src/export/node_vectors.rs | 2 + 28 files changed, 629 insertions(+), 596 deletions(-) create mode 100644 crates/claims/Cargo.toml create mode 100644 crates/claims/src/lib.rs create mode 100644 crates/lean_vm/src/prover.rs diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index fc417204e6..8a61996d48 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -66,6 +66,8 @@ jobs: run: cargo clippyall - name: Clippy (the API on the experimental prover) run: cargo clippy --release -p lean_vm --all-targets --features experimental_prover -- -D warnings + - name: Clippy (the API without the prover, the sidecar's client alone) + run: cargo clippy --release -p lean_vm --no-default-features -- -D warnings # Clippy type-checks every arm the ISA selects, so it also does `cargo check`'s work. # Nothing runs: GitHub's x86-64 runners do not reliably have AVX-512. diff --git a/AGENTS.md b/AGENTS.md index e207bd6bea..aae43b3249 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,9 +36,10 @@ Both prove the same protocol and give the same proof bytes, pinned by `bench/tes | `lean_sphincs` | leanSPHINCS over Keccak-256 | | `bench_harness` | benchmark harness (`Plan`, `Timing`, `--tracing`, Bencher output, the jemalloc allocator); never linked by a prover | | `lean_da` | additive Reed-Solomon blob encoding (the stable tree's NTT), commitments, membership vectors | +| `claims` | `AggregateClaims` and `OutputClaims`, and the claims' byte form: what both the library and the sidecar's client name | | `aggregation` | recursive signature and DA aggregation, generic over the prover: the guest's constants and witness, and the entry points | -| `sidecar` | proving in another process: the protocol of `bins/leanvm-sidecar`, its server (generic over the prover) and its Rust client | -| `lean_vm` | the public API, not generic: it proves with the stable tree, or with the `experimental_prover` feature the experimental one | +| `sidecar` | proving and verifying in another process: the protocol of `bins/leanvm-sidecar` and its Rust client, which builds no prover; with the `server` feature, the server (generic over the prover) | +| `lean_vm` | the public API, not generic: it proves with the stable tree, or with the `experimental_prover` feature the experimental one; without its default `prover` feature, it builds no prover and offers the sidecar's client alone | `aggregation` proves on whichever prover it is given, but derives the guest's constants and a child's witness from the stable tree, the reference: its bus blocks and their coordinates (`lean_vm_core::leaf`, whose term order the guest's arrays follow), the stacking, the grinds, the WHIR shapes and the NTT twiddles; both trees lay a proof out alike. @@ -52,7 +53,7 @@ Each tree holds, in dependency order, leaves first: | `keccak_gkr` | the prover of the VM's `KECCAK` argument | | `lean_vm_core` | arithmetization: tables, bus, constraints, `cpu::execute` and `cpu::prove` of an `isa::Program` and its witness streams; `cpu::verify` wraps `verifier`; the tree's `isa::Prover` | -Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. The sidecar is `bins/leanvm-sidecar`, which serves `aggregate` over a Unix socket so that clients in any language get the prover in a process of its own; its `README.md` is the protocol's specification, so a change to `crates/sidecar`'s `wire.rs`, or to the claims' byte form (`AggregateClaims::to_bytes`), lands there. It also verifies, for clients without the library. +Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. The sidecar is `bins/leanvm-sidecar`, which serves `aggregate` over a Unix socket so that clients in any language get the prover in a process of its own; its `README.md` is the protocol's specification, so a change to the protocol in `crates/sidecar` (`wire.rs` and the server's half in `server.rs`), or to the claims' byte form (`AggregateClaims::to_bytes`), lands there. It also verifies, for clients without the library's prover. ## Building and testing diff --git a/Cargo.lock b/Cargo.lock index 36437a129c..ae37fea9ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6,6 +6,7 @@ version = 4 name = "aggregation" version = "0.1.0" dependencies = [ + "claims", "fiat_shamir_prover", "isa", "keccak256", @@ -125,6 +126,14 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "claims" +version = "0.1.0" +dependencies = [ + "lean_sphincs", + "lean_xmss", +] + [[package]] name = "clap" version = "4.6.1" @@ -370,6 +379,7 @@ name = "lean_vm" version = "0.1.0" dependencies = [ "aggregation", + "claims", "experimental_lean_vm_core", "isa", "lean_da", @@ -377,6 +387,7 @@ dependencies = [ "lean_xmss", "sidecar", "stable_lean_vm_core", + "verifier", ] [[package]] @@ -582,6 +593,7 @@ name = "sidecar" version = "0.1.0" dependencies = [ "aggregation", + "claims", "isa", "lean_da", "lean_sphincs", diff --git a/Cargo.toml b/Cargo.toml index 81473da221..15c3fb53ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "crates/lean_xmss", "crates/lean_sphincs", "crates/lean_da", + "crates/claims", "crates/aggregation", "crates/lean_vm", "crates/sidecar", @@ -38,6 +39,7 @@ zkdsl_compiler = { path = "crates/zkdsl_compiler" } lean_xmss = { path = "crates/lean_xmss" } lean_sphincs = { path = "crates/lean_sphincs" } lean_da = { path = "crates/lean_da" } +claims = { path = "crates/claims" } aggregation = { path = "crates/aggregation" } sidecar = { path = "crates/sidecar" } libc = "0.2" diff --git a/bins/leanvm-sidecar/Cargo.toml b/bins/leanvm-sidecar/Cargo.toml index 8db3c6279b..047a660451 100644 --- a/bins/leanvm-sidecar/Cargo.toml +++ b/bins/leanvm-sidecar/Cargo.toml @@ -14,7 +14,7 @@ doc = false workspace = true [dependencies] -sidecar.workspace = true +sidecar = { workspace = true, features = ["server"] } stable_lean_vm_core = { path = "../../crates/stable_prover/lean_vm_core" } experimental_lean_vm_core = { path = "../../crates/experimental_prover/lean_vm_core" } bench_harness.workspace = true diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index 4009d955db..a33b3ff984 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -1,6 +1,6 @@ # leanvm-sidecar -A process that proves for clients on the same machine, over a Unix stream socket. It runs the library's one proving call, `aggregate`, so a client in any language gets the prover's full speed, with its own allocator, thread pool and retained memory, in a process of its own. It also verifies proofs, for clients without the library; a Rust client verifies with the library (`AggregateProof::verify`), which is faster. +A process that proves for clients on the same machine, over a Unix stream socket. It runs the library's one proving call, `aggregate`, so a client in any language gets the prover's full speed, with its own allocator, thread pool and retained memory, in a process of its own. It also verifies proofs, so a client builds no prover at all. ## Running @@ -26,14 +26,17 @@ target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar ## Clients -From Rust, `lean_vm::sidecar::Client` speaks the protocol, with the library's arguments but for the blobs, which go as they are: +From Rust, `lean_vm::sidecar::Client` speaks the protocol, with the library's arguments but for the blobs, which go as they are, and proofs, which are their bytes: ```rust let mut sidecar = lean_vm::sidecar::Client::connect("/run/leanvm/sidecar.sock", None)?; let (claims, proof) = sidecar.aggregate(&children, &raw_lean_xmss, &[], &[], OutputClaims::All, 2)?; +assert!(sidecar.verify(&claims, &proof)?); ``` -`connect`'s timeout, which `Client::set_timeout` changes, bounds how long each read or write waits; after a timeout, drop the client. The client checks that the claims it gets back are those asked for, but for the DA root of blobs, which it only checks is there. +With `lean_vm = { default-features = false }`, a client builds no prover: it proves and verifies through the sidecar alone, so it cannot verify while the sidecar is down or restarting. With the default `prover` feature, the library also proves and verifies in the client's process (`AggregateProof::from_bytes`, then `verify`), which takes a sidecar of the same leanVM build. + +`connect`'s timeout, which `Client::set_timeout` changes, bounds how long each read or write waits; after a timeout, drop the client. From another language, a client is the protocol below: connect, read the hello, then write a request and read its response, as many times as needed. diff --git a/bins/leanvm-sidecar/tests/sidecar.rs b/bins/leanvm-sidecar/tests/sidecar.rs index 7887e4022a..be22806a6b 100644 --- a/bins/leanvm-sidecar/tests/sidecar.rs +++ b/bins/leanvm-sidecar/tests/sidecar.rs @@ -4,7 +4,7 @@ use aggregation::signers_cache; use lean_vm::sidecar::{Client, Rejection, SidecarError}; -use lean_vm::{AggregateClaims, OutputClaims, lean_da}; +use lean_vm::{AggregateClaims, AggregateProof, OutputClaims, lean_da}; use stable_lean_vm_core::StableProver; use std::io::{Read, Write}; use std::os::unix::net::UnixStream; @@ -75,7 +75,7 @@ fn sidecar_end_to_end() { let leaf = client .aggregate(&[], &xmss[..2], &sphincs, &blobs, OutputClaims::All, 2) .unwrap(); - assert!(leaf.1.verify(&leaf.0)); + assert!(AggregateProof::from_bytes(&leaf.1).unwrap().verify(&leaf.0)); assert_eq!(leaf.0.num_signature_claims(), 3); assert_eq!( leaf.0.da_commitment, @@ -95,7 +95,7 @@ fn sidecar_end_to_end() { .aggregate(&[leaf], &xmss[2..], &[], &[], OutputClaims::Only(&only), 1) .unwrap(); assert_eq!(node.0, only); - assert!(node.1.verify(&only)); + assert!(AggregateProof::from_bytes(&node.1).unwrap().verify(&only)); // Verifying, from either connection, while nothing proves. assert!(client.verify(&node.0, &node.1).unwrap()); diff --git a/crates/aggregation/Cargo.toml b/crates/aggregation/Cargo.toml index 0c192c0b25..087ca25e81 100644 --- a/crates/aggregation/Cargo.toml +++ b/crates/aggregation/Cargo.toml @@ -13,6 +13,7 @@ verifier.workspace = true keccak256.workspace = true fiat_shamir_prover.workspace = true zkdsl_compiler.workspace = true +claims.workspace = true lean_xmss.workspace = true lean_sphincs.workspace = true lean_da.workspace = true diff --git a/crates/aggregation/src/coverage.rs b/crates/aggregation/src/coverage.rs index f7de07c4ae..3c39e1d676 100644 --- a/crates/aggregation/src/coverage.rs +++ b/crates/aggregation/src/coverage.rs @@ -1,8 +1,8 @@ //! The coverage table: which slot each raw signature and child claim writes. use super::error::AggregateError; -use super::proof::{AggregateClaims, AggregateProof}; -use super::{LeanSphincsClaim, LeanXmssClaim}; +use super::proof::AggregateProof; +use super::{AggregateClaims, LeanSphincsClaim, LeanXmssClaim}; use std::collections::BTreeSet; /// The coverage slot each write in the guest's coverage walk targets, in walk diff --git a/crates/aggregation/src/lib.rs b/crates/aggregation/src/lib.rs index beab2876ee..15377d1672 100644 --- a/crates/aggregation/src/lib.rs +++ b/crates/aggregation/src/lib.rs @@ -54,19 +54,11 @@ mod statement; #[cfg(test)] mod tests; +pub use claims::{AggregateClaims, LeanSphincsClaim, LeanXmssClaim, OutputClaims}; pub use error::AggregateError; pub use guest::unified_guest; -use lean_sphincs::LeanSphincsPublicKey; -use lean_xmss::LeanXmssPublicKey; -pub use node::{aggregate, published_claims}; -pub use proof::{AggregateClaims, AggregateProof, OutputClaims}; - -/// One leanXMSS claim: a pubkey, and the message it signed at whatever leaf index its -/// signature carries. -pub type LeanXmssClaim = (LeanXmssPublicKey, lean_xmss::Message); - -/// One leanSPHINCS claim: a pubkey, and the message it signed. -pub type LeanSphincsClaim = (LeanSphincsPublicKey, lean_sphincs::Message); +pub use node::aggregate; +pub use proof::AggregateProof; /// The maximum number of child proofs one [`aggregate`] call can take. pub const MAX_RECURSIONS: usize = 16; diff --git a/crates/aggregation/src/node.rs b/crates/aggregation/src/node.rs index 45f7473025..8df3e452a2 100644 --- a/crates/aggregation/src/node.rs +++ b/crates/aggregation/src/node.rs @@ -2,16 +2,16 @@ use super::cells::count; use super::child::gen_verify; -use super::coverage::{Coverage, plan_coverage}; +use super::coverage::plan_coverage; use super::deferred::{DeferredBytecodeClaim, aggregate_deferred_claims, bytecode_sums}; use super::error::AggregateError; use super::guest::{fs_seed, unified_guest}; #[cfg(test)] use super::hints::FORGE; use super::hints::{Hints, lean_sphincs_hints, lean_xmss_hints}; -use super::proof::{AggregateClaims, AggregateProof, OutputClaims, is_sorted}; +use super::proof::{AggregateProof, is_sorted}; use super::statement::{lean_sphincs_claim_cells, lean_xmss_claim_cells, list_split, statement_digest}; -use super::{LeanSphincsClaim, LeanXmssClaim, MAX_LOG_INV_RATE, MAX_RECURSIONS, MIN_LOG_INV_RATE}; +use super::{AggregateClaims, MAX_LOG_INV_RATE, MAX_RECURSIONS, MIN_LOG_INV_RATE, OutputClaims}; use fiat_shamir_prover::merkle::hash_to_scalars; use isa::{Prover, Stats}; use lean_da::CELLS_PER_ROW; @@ -48,6 +48,10 @@ pub fn aggregate( // Two proofs at once would share the pool and hold both proofs' memory. static PROVING: Mutex<()> = Mutex::new(()); let _proving = PROVING.lock().unwrap_or_else(PoisonError::into_inner); + let declare = match output { + OutputClaims::All => None, + OutputClaims::Only(claims) => Some(claims), + }; // Proving checks the rate too, but only after the witness is built. if !(MIN_LOG_INV_RATE..=MAX_LOG_INV_RATE).contains(&log_inv_rate) { return Err(AggregateError::InvalidInput("log_inv_rate out of range")); @@ -55,16 +59,39 @@ pub fn aggregate( if children.len() > MAX_RECURSIONS { return Err(AggregateError::InvalidInput("more than MAX_RECURSIONS children")); } + // The published root and the guest's slot for it: a child's, or `children.len()` for the node's own, its blobs' + // or none. + let own = children.len(); + let (da_pick, da_root) = match (blobs, declare.map(|claims| claims.da_commitment)) { + (Some(matrix), declared) => { + let root = Some(matrix.commitment()); + if declared.is_some_and(|declared| declared != root) { + return Err(AggregateError::InvalidInput("a node with blobs publishes their root")); + } + (own, root) + } + (None, None) => children + .iter() + .position(|(claims, _)| claims.da_commitment.is_some()) + .map_or((own, None), |i| (i, children[i].0.da_commitment)), + (None, Some(None)) => (own, None), + (None, Some(root)) => children + .iter() + .position(|(claims, _)| claims.da_commitment == root) + .map(|i| (i, root)) + .ok_or(AggregateError::InvalidInput("the requested DA root is not a child's"))?, + }; + + // The request's checks, on the claim lists alone, before any costly work. + if !children.iter().all(|(claims, _)| is_sorted(claims)) { + return Err(AggregateError::InvalidChild); + } let raw_lean_xmss_claims = sorted_claims(&mut raw_lean_xmss); let raw_lean_sphincs_claims = sorted_claims(&mut raw_lean_sphincs); - let blobs_root = blobs.map(lean_da::BlobMatrix::commitment); - let (cover, da_pick, da_root) = plan( - children, - &raw_lean_xmss_claims, - &raw_lean_sphincs_claims, - blobs_root, - output, - )?; + let cover = plan_coverage(&raw_lean_xmss_claims, &raw_lean_sphincs_claims, children, declare)?; + if cover.lean_xmss.signers.is_empty() && cover.lean_sphincs.signers.is_empty() && da_root.is_none() { + return Err(AggregateError::InvalidInput("nothing to publish")); + } // Nothing here verifies a child (see `AGENTS.md`): the replay reads out of its proof the summary `gen_verify` // builds the guest's witness from, and fails only on a proof it cannot walk. @@ -255,65 +282,6 @@ pub fn aggregate( Ok((claims, proof, stats)) } -/// The claims [`aggregate`] proves for these inputs, without proving: `blobs_root` is its blobs' root, if any. `Err` -/// for the requests it refuses on their claims alone. -pub fn published_claims( - children: &[(AggregateClaims, AggregateProof)], - raw_lean_xmss: &[LeanXmssClaim], - raw_lean_sphincs: &[LeanSphincsClaim], - blobs_root: Option<[u8; 32]>, - output: OutputClaims, -) -> Result { - let (cover, _, da_commitment) = plan(children, raw_lean_xmss, raw_lean_sphincs, blobs_root, output)?; - Ok(AggregateClaims { - lean_xmss: cover.lean_xmss.signers, - lean_sphincs: cover.lean_sphincs.signers, - da_commitment, - }) -} - -/// The request's checks, on the claim lists alone, before any costly work: the coverage, and the published root with -/// the guest's slot for it, a child's or `children.len()` for the node's own, its blobs' or none. -fn plan( - children: &[(AggregateClaims, AggregateProof)], - raw_lean_xmss: &[LeanXmssClaim], - raw_lean_sphincs: &[LeanSphincsClaim], - blobs_root: Option<[u8; 32]>, - output: OutputClaims, -) -> Result<(Coverage, usize, Option<[u8; 32]>), AggregateError> { - let declare = match output { - OutputClaims::All => None, - OutputClaims::Only(claims) => Some(claims), - }; - let own = children.len(); - let (da_pick, da_root) = match (blobs_root, declare.map(|claims| claims.da_commitment)) { - (Some(root), declared) => { - if declared.is_some_and(|declared| declared != Some(root)) { - return Err(AggregateError::InvalidInput("a node with blobs publishes their root")); - } - (own, Some(root)) - } - (None, None) => children - .iter() - .position(|(claims, _)| claims.da_commitment.is_some()) - .map_or((own, None), |i| (i, children[i].0.da_commitment)), - (None, Some(None)) => (own, None), - (None, Some(root)) => children - .iter() - .position(|(claims, _)| claims.da_commitment == root) - .map(|i| (i, root)) - .ok_or(AggregateError::InvalidInput("the requested DA root is not a child's"))?, - }; - if !children.iter().all(|(claims, _)| is_sorted(claims)) { - return Err(AggregateError::InvalidChild); - } - let cover = plan_coverage(raw_lean_xmss, raw_lean_sphincs, children, declare)?; - if cover.lean_xmss.signers.is_empty() && cover.lean_sphincs.signers.is_empty() && da_root.is_none() { - return Err(AggregateError::InvalidInput("nothing to publish")); - } - Ok((cover, da_pick, da_root)) -} - /// Sort raw signatures on their claims and drop repeats: on the whole (pubkey, message) pair, so a signer may appear /// once per message, and one claim needs one signature, whatever leaf index a leanXMSS one carries. Returns the claims. fn sorted_claims(raw: &mut Vec<(K, [u8; 32], S)>) -> Vec<(K, [u8; 32])> { diff --git a/crates/aggregation/src/proof.rs b/crates/aggregation/src/proof.rs index 8dc997c11a..ceea60c7f4 100644 --- a/crates/aggregation/src/proof.rs +++ b/crates/aggregation/src/proof.rs @@ -3,33 +3,10 @@ use super::deferred::{DeferredBytecodeClaim, bytecode_vars}; use super::guest::unified_guest; use super::statement::statement_digest; -use super::{LeanSphincsClaim, LeanXmssClaim}; +use claims::AggregateClaims; use lean_vm_core::cpu::public_input_bytes; use primitives::field::F128; -/// What an [`AggregateProof`] proves: every `(pubkey, message)` claim of two lists, one of leanXMSS and one of -/// leanSPHINCS, had a valid signature, and, optionally, a commitment is the root of a well-formed matrix of blobs (see -/// [`lean_da::BlobMatrix`]). -/// -/// Each list is strictly sorted (so without duplicates), and may be empty. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct AggregateClaims { - pub lean_xmss: Vec, - pub lean_sphincs: Vec, - /// `None` for no blobs. - pub da_commitment: Option<[u8; 32]>, -} - -/// Which covered claims [`aggregate`](super::aggregate) proves. -#[derive(Clone, Copy, Debug)] -pub enum OutputClaims<'a> { - /// Every covered signature claim. The DA root is `blobs`' if any, otherwise the first - /// child's that has one, otherwise none. - All, - /// Exactly these, each covered by the inputs. - Only(&'a AggregateClaims), -} - /// A proof of some [`AggregateClaims`], which travel separately: [`Self::verify`] checks it against /// the claims the receiver expects. #[derive(Clone, Debug)] @@ -65,74 +42,6 @@ pub(super) fn is_sorted(claims: &AggregateClaims) -> bool { claims.lean_xmss.windows(2).all(|w| w[0] < w[1]) && claims.lean_sphincs.windows(2).all(|w| w[0] < w[1]) } -// Wire format of claims: the leanXMSS claims (a u64 count, then each claim's pubkey and message), the leanSPHINCS -// claims alike, then the DA commitment (a byte, 0 for none or 1, then the 32-byte root). Little-endian, and -// canonical: decoding rejects any other flag and trailing bytes. Order is kept, sorted or not. - -const _: () = assert!(lean_xmss::PUB_KEY_SIZE == 32 && lean_sphincs::PUB_KEY_SIZE == 32); -const _: () = assert!(lean_xmss::MESSAGE_LEN == 32 && lean_sphincs::MESSAGE_LEN == 32); - -fn encode_claims(out: &mut Vec, claims: &[(K, [u8; 32])], pk_bytes: impl Fn(&K) -> [u8; 32]) { - out.extend((claims.len() as u64).to_le_bytes()); - for (pk, message) in claims { - out.extend(pk_bytes(pk)); - out.extend(message); - } -} - -/// Reads a count and that many `(pubkey, message)` pairs off the front of `bytes`. A forged count allocates nothing. -fn decode_claims(bytes: &mut &[u8], pk: impl Fn(&[u8; 32]) -> K) -> Option> { - let (count, rest) = bytes.split_first_chunk::<8>()?; - let count = usize::try_from(u64::from_le_bytes(*count)).ok()?; - let (pairs, rest) = rest.split_at_checked(count.checked_mul(64)?)?; - *bytes = rest; - let pairs = pairs.as_chunks::<32>().0.as_chunks::<2>().0; - Some(pairs.iter().map(|[key, message]| (pk(key), *message)).collect()) -} - -impl AggregateClaims { - /// The number of `(pubkey, message)` claims, not of distinct signers: a pubkey may sign - /// several messages. - pub fn num_signature_claims(&self) -> usize { - self.lean_xmss.len() + self.lean_sphincs.len() - } - - /// Encode claims, which [`Self::from_bytes`] reads back. - pub fn to_bytes(&self) -> Vec { - let mut out = Vec::new(); - encode_claims(&mut out, &self.lean_xmss, lean_xmss::LeanXmssPublicKey::to_bytes); - encode_claims( - &mut out, - &self.lean_sphincs, - lean_sphincs::LeanSphincsPublicKey::to_bytes, - ); - match self.da_commitment { - None => out.push(0), - Some(root) => { - out.push(1); - out.extend(root); - } - } - out - } - - /// Decode claims, `None` if the bytes are not exactly one encoding. - pub fn from_bytes(mut bytes: &[u8]) -> Option { - let lean_xmss = decode_claims(&mut bytes, lean_xmss::LeanXmssPublicKey::from_bytes)?; - let lean_sphincs = decode_claims(&mut bytes, lean_sphincs::LeanSphincsPublicKey::from_bytes)?; - let da_commitment = match bytes { - [0] => None, - [1, root @ ..] => Some(<[u8; 32]>::try_from(root).ok()?), - _ => return None, - }; - Some(Self { - lean_xmss, - lean_sphincs, - da_commitment, - }) - } -} - impl AggregateProof { pub fn to_bytes(&self) -> Vec { encode(&self.bytecode_point, &self.proof.to_bytes()) diff --git a/crates/aggregation/src/statement.rs b/crates/aggregation/src/statement.rs index 87daf13854..68c511f8ff 100644 --- a/crates/aggregation/src/statement.rs +++ b/crates/aggregation/src/statement.rs @@ -3,8 +3,7 @@ use super::cells::{cell_bytes, count, pack_16_bytes}; use super::deferred::DeferredBytecodeClaim; use super::guest::fs_seed; -use super::proof::AggregateClaims; -use super::{LeanSphincsClaim, LeanXmssClaim}; +use super::{AggregateClaims, LeanSphincsClaim, LeanXmssClaim}; use fiat_shamir_prover::merkle::hash_to_scalars; use primitives::field::F128; diff --git a/crates/aggregation/src/tests.rs b/crates/aggregation/src/tests.rs index 48b03e5d31..03e3bf6552 100644 --- a/crates/aggregation/src/tests.rs +++ b/crates/aggregation/src/tests.rs @@ -4,7 +4,7 @@ use super::error::AggregateError; use super::guest::{GUEST_SOURCE, compile_guest, unified_guest}; use super::hints::{FORGE, Forgery, Hints}; use super::placeholders::placeholder_map; -use super::proof::{AggregateClaims, AggregateProof, decode, encode}; +use super::proof::{AggregateProof, decode, encode}; use super::statement::{CLAIM_GROUP, claim_list_digest, list_split}; use super::*; use fiat_shamir_prover::merkle::hash_to_scalars; diff --git a/crates/claims/Cargo.toml b/crates/claims/Cargo.toml new file mode 100644 index 0000000000..0f063dc544 --- /dev/null +++ b/crates/claims/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "claims" +version.workspace = true +edition.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +lean_xmss.workspace = true +lean_sphincs.workspace = true diff --git a/crates/claims/src/lib.rs b/crates/claims/src/lib.rs new file mode 100644 index 0000000000..033dbaa440 --- /dev/null +++ b/crates/claims/src/lib.rs @@ -0,0 +1,99 @@ +//! The claims an aggregate proof attests, and their byte form: shared by the library, which proves and verifies +//! them, and the sidecar's client, which builds neither. + +use lean_sphincs::LeanSphincsPublicKey; +use lean_xmss::LeanXmssPublicKey; + +/// One leanXMSS claim: a pubkey, and the message it signed at whatever leaf index its +/// signature carries. +pub type LeanXmssClaim = (LeanXmssPublicKey, lean_xmss::Message); + +/// One leanSPHINCS claim: a pubkey, and the message it signed. +pub type LeanSphincsClaim = (LeanSphincsPublicKey, lean_sphincs::Message); + +/// What an aggregate proof proves: every `(pubkey, message)` claim of two lists, one of leanXMSS and one of +/// leanSPHINCS, had a valid signature, and, optionally, a commitment is the root of a well-formed matrix of blobs +/// (`lean_da::BlobMatrix`). +/// +/// Each list is strictly sorted (so without duplicates), and may be empty. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct AggregateClaims { + pub lean_xmss: Vec, + pub lean_sphincs: Vec, + /// `None` for no blobs. + pub da_commitment: Option<[u8; 32]>, +} + +/// Which covered claims an aggregation proves. +#[derive(Clone, Copy, Debug)] +pub enum OutputClaims<'a> { + /// Every covered signature claim. The DA root is `blobs`' if any, otherwise the first + /// child's that has one, otherwise none. + All, + /// Exactly these, each covered by the inputs. + Only(&'a AggregateClaims), +} + +// Wire format of claims: the leanXMSS claims (a u64 count, then each claim's pubkey and message), the leanSPHINCS +// claims alike, then the DA commitment (a byte, 0 for none or 1, then the 32-byte root). Little-endian, and +// canonical: decoding rejects any other flag and trailing bytes. Order is kept, sorted or not. + +const _: () = assert!(lean_xmss::PUB_KEY_SIZE == 32 && lean_sphincs::PUB_KEY_SIZE == 32); +const _: () = assert!(lean_xmss::MESSAGE_LEN == 32 && lean_sphincs::MESSAGE_LEN == 32); + +fn encode_claims(out: &mut Vec, claims: &[(K, [u8; 32])], pk_bytes: impl Fn(&K) -> [u8; 32]) { + out.extend((claims.len() as u64).to_le_bytes()); + for (pk, message) in claims { + out.extend(pk_bytes(pk)); + out.extend(message); + } +} + +/// Reads a count and that many `(pubkey, message)` pairs off the front of `bytes`. A forged count allocates nothing. +fn decode_claims(bytes: &mut &[u8], pk: impl Fn(&[u8; 32]) -> K) -> Option> { + let (count, rest) = bytes.split_first_chunk::<8>()?; + let count = usize::try_from(u64::from_le_bytes(*count)).ok()?; + let (pairs, rest) = rest.split_at_checked(count.checked_mul(64)?)?; + *bytes = rest; + let pairs = pairs.as_chunks::<32>().0.as_chunks::<2>().0; + Some(pairs.iter().map(|[key, message]| (pk(key), *message)).collect()) +} + +impl AggregateClaims { + /// The number of `(pubkey, message)` claims, not of distinct signers: a pubkey may sign + /// several messages. + pub fn num_signature_claims(&self) -> usize { + self.lean_xmss.len() + self.lean_sphincs.len() + } + + /// Encode claims, which [`Self::from_bytes`] reads back. + pub fn to_bytes(&self) -> Vec { + let mut out = Vec::new(); + encode_claims(&mut out, &self.lean_xmss, LeanXmssPublicKey::to_bytes); + encode_claims(&mut out, &self.lean_sphincs, LeanSphincsPublicKey::to_bytes); + match self.da_commitment { + None => out.push(0), + Some(root) => { + out.push(1); + out.extend(root); + } + } + out + } + + /// Decode claims, `None` if the bytes are not exactly one encoding. + pub fn from_bytes(mut bytes: &[u8]) -> Option { + let lean_xmss = decode_claims(&mut bytes, LeanXmssPublicKey::from_bytes)?; + let lean_sphincs = decode_claims(&mut bytes, LeanSphincsPublicKey::from_bytes)?; + let da_commitment = match bytes { + [0] => None, + [1, root @ ..] => Some(<[u8; 32]>::try_from(root).ok()?), + _ => return None, + }; + Some(Self { + lean_xmss, + lean_sphincs, + da_commitment, + }) + } +} diff --git a/crates/lean_vm/Cargo.toml b/crates/lean_vm/Cargo.toml index c0ced4e277..0d91a2e259 100644 --- a/crates/lean_vm/Cargo.toml +++ b/crates/lean_vm/Cargo.toml @@ -7,12 +7,14 @@ edition.workspace = true workspace = true [dependencies] -isa.workspace = true -aggregation.workspace = true -lean_da.workspace = true +claims.workspace = true +verifier.workspace = true lean_xmss.workspace = true lean_sphincs.workspace = true -stable_lean_vm_core = { path = "../stable_prover/lean_vm_core" } +isa = { workspace = true, optional = true } +aggregation = { workspace = true, optional = true } +lean_da = { workspace = true, optional = true } +stable_lean_vm_core = { path = "../stable_prover/lean_vm_core", optional = true } experimental_lean_vm_core = { path = "../experimental_prover/lean_vm_core", optional = true } # The sidecar speaks over a Unix socket. @@ -20,5 +22,8 @@ experimental_lean_vm_core = { path = "../experimental_prover/lean_vm_core", opti sidecar.workspace = true [features] +default = ["prover"] +# Prove and verify here. Without it, through the sidecar only, and no prover is built. +prover = ["dep:isa", "dep:aggregation", "dep:lean_da", "dep:stable_lean_vm_core"] # Prove with the experimental tree rather than the stable one. -experimental_prover = ["dep:experimental_lean_vm_core"] +experimental_prover = ["prover", "dep:experimental_lean_vm_core"] diff --git a/crates/lean_vm/src/lib.rs b/crates/lean_vm/src/lib.rs index ec32d52fbf..ca0963ab86 100644 --- a/crates/lean_vm/src/lib.rs +++ b/crates/lean_vm/src/lib.rs @@ -3,30 +3,20 @@ //! - leanSPHINCS (stateless, for the execution layer) //! - leanDA (PQ blobs) //! -//! Proving runs the stable prover, or with the `experimental_prover` feature the experimental one. -//! Both give the same proofs, byte for byte, and verifying needs neither. +//! With the default `prover` feature, it proves and verifies here: proving runs the stable prover, or with the +//! `experimental_prover` feature the experimental one, and both give the same proofs, byte for byte. Without it, it +//! builds no prover, and proves and verifies through the sidecar, [`sidecar::Client`]: the same claims and +//! signatures, proofs as their bytes. //! //! Example: [`crates/lean_vm/tests/api.rs`](https://github.com/leanEthereum/leanVM/blob/main/crates/lean_vm/tests/api.rs). -use ::lean_sphincs::{LeanSphincsPublicKey, LeanSphincsSignature}; -use ::lean_xmss::{LeanXmssPublicKey, LeanXmssSignature}; -pub use aggregation::{ - AggregateClaims, AggregateError, AggregateProof, LeanSphincsClaim, LeanXmssClaim, MAX_LOG_INV_RATE, MAX_RECURSIONS, - MIN_LOG_INV_RATE, OutputClaims, -}; -use isa::Prover; +pub use claims::{AggregateClaims, LeanSphincsClaim, LeanXmssClaim, OutputClaims}; +pub use verifier::{MAX_LOG_INV_RATE, MIN_LOG_INV_RATE}; -#[cfg(feature = "experimental_prover")] -type P = experimental_lean_vm_core::ExperimentalProver; -#[cfg(not(feature = "experimental_prover"))] -type P = stable_lean_vm_core::StableProver; - -pub mod lean_da { - pub use ::lean_da::{ - BLOB_SYMBOLS, Blob, BlobCountError, BlobMatrix, CELL_SYMBOLS, CELLS_PER_ROW, Codeword, DA_MAX_ROWS, - MERKLE_PATH_LEN, Symbol, verify_cell, - }; -} +#[cfg(feature = "prover")] +mod prover; +#[cfg(feature = "prover")] +pub use prover::{AggregateError, AggregateProof, MAX_RECURSIONS, aggregate, lean_da, setup}; pub mod lean_xmss { pub use lean_xmss::{ @@ -43,51 +33,10 @@ pub mod lean_sphincs { }; } -/// [`aggregate`] in another process, the sidecar (`bins/leanvm-sidecar`), which keeps the prover's memory, threads -/// and warm state out of this one. Verifying stays here, with [`AggregateProof::verify`]. +/// Proving and verifying in another process, the sidecar (`bins/leanvm-sidecar`), which keeps the prover's memory, +/// threads and warm state out of this one. A client built without the `prover` feature verifies only there, so not +/// while its sidecar is down. #[cfg(unix)] pub mod sidecar { - pub use ::sidecar::{Client, Rejection, SidecarError}; -} - -/// Call this function before generating / verifying proofs: it starts the prover's thread pool, with `threads` threads -/// (0: one per core), and builds the aggregation program. -/// -/// # Panics -/// If the pool already runs with another thread count. -pub fn setup(threads: usize) { - if threads > 0 { - P::set_num_threads(threads); - } - P::init(); - aggregation::unified_guest(); -} - -/// Aggregate leanXMSS and leanSPHINCS signatures, blobs and the claims of child proofs. -/// -/// - `children`: at most [`MAX_RECURSIONS`], each with its claims. -/// - `raw_lean_xmss`, `raw_lean_sphincs`: `(public_key, message, signature)`, in any order. -/// - `blobs`: the encoded blobs, if any ([`lean_da::BlobMatrix::build`]). -/// - `output`: which covered claims the proof attests. -/// - `log_inv_rate`: the code rate's log, in [`MIN_LOG_INV_RATE`]..=[`MAX_LOG_INV_RATE`]; higher gives a smaller proof -/// but slower proving. -/// -/// # Errors -/// -/// [`AggregateError`] says why there is no proof. -/// -/// Calls run one at a time: a second waits for the first. -/// -/// # WARNING -/// - Signatures and children must be valid (children checked with [`AggregateProof::verify`]): otherwise proving fails. -pub fn aggregate( - children: &[(AggregateClaims, AggregateProof)], - raw_lean_xmss: Vec<(LeanXmssPublicKey, lean_xmss::Message, LeanXmssSignature)>, - raw_lean_sphincs: Vec<(LeanSphincsPublicKey, lean_sphincs::Message, LeanSphincsSignature)>, - blobs: Option<&lean_da::BlobMatrix>, - output: OutputClaims, - log_inv_rate: usize, -) -> Result<(AggregateClaims, AggregateProof), AggregateError> { - aggregation::aggregate::

(children, raw_lean_xmss, raw_lean_sphincs, blobs, output, log_inv_rate) - .map(|(claims, proof, _)| (claims, proof)) + pub use ::sidecar::{Blob, Client, Rejection, SidecarError}; } diff --git a/crates/lean_vm/src/prover.rs b/crates/lean_vm/src/prover.rs new file mode 100644 index 0000000000..0b6f0392e2 --- /dev/null +++ b/crates/lean_vm/src/prover.rs @@ -0,0 +1,61 @@ +//! What the `prover` feature brings: proving and verifying here, and LeanDA's encoding. + +use crate::{AggregateClaims, OutputClaims}; +use ::lean_sphincs::{LeanSphincsPublicKey, LeanSphincsSignature}; +use ::lean_xmss::{LeanXmssPublicKey, LeanXmssSignature}; +pub use aggregation::{AggregateError, AggregateProof, MAX_RECURSIONS}; +use isa::Prover; + +#[cfg(feature = "experimental_prover")] +type P = experimental_lean_vm_core::ExperimentalProver; +#[cfg(not(feature = "experimental_prover"))] +type P = stable_lean_vm_core::StableProver; + +pub mod lean_da { + pub use ::lean_da::{ + BLOB_SYMBOLS, Blob, BlobCountError, BlobMatrix, CELL_SYMBOLS, CELLS_PER_ROW, Codeword, DA_MAX_ROWS, + MERKLE_PATH_LEN, Symbol, verify_cell, + }; +} + +/// Call this function before generating / verifying proofs: it starts the prover's thread pool, with `threads` threads +/// (0: one per core), and builds the aggregation program. +/// +/// # Panics +/// If the pool already runs with another thread count. +pub fn setup(threads: usize) { + if threads > 0 { + P::set_num_threads(threads); + } + P::init(); + aggregation::unified_guest(); +} + +/// Aggregate leanXMSS and leanSPHINCS signatures, blobs and the claims of child proofs. +/// +/// - `children`: at most [`MAX_RECURSIONS`], each with its claims. +/// - `raw_lean_xmss`, `raw_lean_sphincs`: `(public_key, message, signature)`, in any order. +/// - `blobs`: the encoded blobs, if any ([`lean_da::BlobMatrix::build`]). +/// - `output`: which covered claims the proof attests. +/// - `log_inv_rate`: the code rate's log, in [`MIN_LOG_INV_RATE`](crate::MIN_LOG_INV_RATE)..= +/// [`MAX_LOG_INV_RATE`](crate::MAX_LOG_INV_RATE); higher gives a smaller proof but slower proving. +/// +/// # Errors +/// +/// [`AggregateError`] says why there is no proof. +/// +/// Calls run one at a time: a second waits for the first. +/// +/// # WARNING +/// - Signatures and children must be valid (children checked with [`AggregateProof::verify`]): otherwise proving fails. +pub fn aggregate( + children: &[(AggregateClaims, AggregateProof)], + raw_lean_xmss: Vec<(LeanXmssPublicKey, ::lean_xmss::Message, LeanXmssSignature)>, + raw_lean_sphincs: Vec<(LeanSphincsPublicKey, ::lean_sphincs::Message, LeanSphincsSignature)>, + blobs: Option<&lean_da::BlobMatrix>, + output: OutputClaims, + log_inv_rate: usize, +) -> Result<(AggregateClaims, AggregateProof), AggregateError> { + aggregation::aggregate::

(children, raw_lean_xmss, raw_lean_sphincs, blobs, output, log_inv_rate) + .map(|(claims, proof, _)| (claims, proof)) +} diff --git a/crates/sidecar/Cargo.toml b/crates/sidecar/Cargo.toml index 06f02cbd1d..2dbeacd81d 100644 --- a/crates/sidecar/Cargo.toml +++ b/crates/sidecar/Cargo.toml @@ -8,10 +8,15 @@ publish = false workspace = true [dependencies] -isa.workspace = true +claims.workspace = true verifier.workspace = true -aggregation.workspace = true -lean_da.workspace = true lean_xmss.workspace = true lean_sphincs.workspace = true -libc.workspace = true +isa = { workspace = true, optional = true } +aggregation = { workspace = true, optional = true } +lean_da = { workspace = true, optional = true } +libc = { workspace = true, optional = true } + +[features] +# The server, which proves: what `bins/leanvm-sidecar` runs. The client builds no prover. +server = ["dep:isa", "dep:aggregation", "dep:lean_da", "dep:libc"] diff --git a/crates/sidecar/src/client.rs b/crates/sidecar/src/client.rs index d02af67619..55e5210491 100644 --- a/crates/sidecar/src/client.rs +++ b/crates/sidecar/src/client.rs @@ -1,9 +1,8 @@ //! The protocol from Rust. use crate::wire::{self, Incoming}; -use crate::{Rejection, SidecarError}; -use aggregation::{AggregateClaims, AggregateProof, OutputClaims}; -use lean_da::Blob; +use crate::{Blob, Rejection, SidecarError}; +use claims::{AggregateClaims, OutputClaims}; use lean_sphincs::{LeanSphincsPublicKey, LeanSphincsSignature}; use lean_xmss::{LeanXmssPublicKey, LeanXmssSignature}; use std::io::Write; @@ -11,7 +10,8 @@ use std::os::unix::net::UnixStream; use std::path::Path; use std::time::Duration; -/// A connection to a sidecar, for one request at a time. +/// A connection to a sidecar, for one request at a time. Proofs are their bytes, as the library's +/// `AggregateProof::to_bytes` writes them and `from_bytes` reads them. /// /// After an error other than [`SidecarError::Rejected`], the connection is in an unknown state: drop it and connect /// again. @@ -23,9 +23,8 @@ pub struct Client { } impl Client { - /// Connects to the sidecar listening on `socket`, and checks it speaks this protocol version and proves this - /// build's aggregation program, which the first connection in a process builds. `timeout` is - /// [`Self::set_timeout`]'s, already for reading the sidecar's hello. + /// Connects to the sidecar listening on `socket`, and checks it speaks this protocol version and proves for this + /// verifier. `timeout` is [`Self::set_timeout`]'s, already for reading the sidecar's hello. pub fn connect(socket: impl AsRef, timeout: Option) -> Result { let mut stream = UnixStream::connect(socket)?; stream.set_read_timeout(timeout)?; @@ -53,11 +52,6 @@ impl Client { String::from_utf8_lossy(verifier::TAG) ))); } - if hello.program != aggregation::unified_guest().bytecode().fiat_shamir_iv() { - return Err(SidecarError::Incompatible( - "proves another aggregation program: another leanVM build".into(), - )); - } Ok(Self { stream, prover: hello.prover, @@ -78,45 +72,32 @@ impl Client { &self.prover } - /// The library's `aggregate`, proven by the sidecar. The arguments are the library's but for the blobs, which - /// go as they are (none for no blobs) and which the sidecar builds the matrix of. + /// The library's `aggregate`, proven by the sidecar: its arguments, but for the blobs, which go as they are (none + /// for no blobs) and which the sidecar builds the matrix of. /// - /// As with the library, children must be valid (checked with [`AggregateProof::verify`]), or proving fails. The - /// claims returned are checked to be those asked for, but for the root of `blobs`, only checked to be there. + /// As with the library, children must be valid (checked with [`Self::verify`]), or proving fails. pub fn aggregate( &mut self, - children: &[(AggregateClaims, AggregateProof)], + children: &[(AggregateClaims, Vec)], raw_lean_xmss: &[(LeanXmssPublicKey, lean_xmss::Message, LeanXmssSignature)], raw_lean_sphincs: &[(LeanSphincsPublicKey, lean_sphincs::Message, LeanSphincsSignature)], blobs: &[Blob], output: OutputClaims, log_inv_rate: usize, - ) -> Result<(AggregateClaims, AggregateProof), SidecarError> { + ) -> Result<(AggregateClaims, Vec), SidecarError> { let request = wire::encode_request(children, raw_lean_xmss, raw_lean_sphincs, blobs, output, log_inv_rate); // The claims a response carries are no longer than the request. let body = self.call(&request, request.len() as u64 + PROOF_ROOM)?; - let (claims, proof) = match wire::decode_response(&body) { - Some(Ok(proven)) => proven, - Some(Err((rejection, message))) => return Err(SidecarError::Rejected(rejection, message)), - None => return Err(SidecarError::Protocol("a response that does not decode".into())), - }; - // The root of blobs is the sidecar's to compute: it is taken as given, but must be there. - let blobs_root = (!blobs.is_empty()).then_some(claims.da_commitment); - let asked = aggregation::published_claims( - children, - &claims_of(raw_lean_xmss), - &claims_of(raw_lean_sphincs), - blobs_root.flatten(), - output, - ); - if blobs_root == Some(None) || asked.as_ref() != Ok(&claims) { - return Err(SidecarError::Protocol("a proof of other claims than asked for".into())); + match wire::decode_response(&body) { + Some(Ok(proven)) => Ok(proven), + Some(Err((rejection, message))) => Err(SidecarError::Rejected(rejection, message)), + None => Err(SidecarError::Protocol("a response that does not decode".into())), } - Ok((claims, proof)) } - /// The sidecar's verify operation, which [`AggregateProof::verify`] answers faster in this process. - pub fn verify(&mut self, claims: &AggregateClaims, proof: &AggregateProof) -> Result { + /// The library's `AggregateProof::verify`, run by the sidecar: whether `proof` proves `claims`. Bytes that do not + /// decode prove nothing. + pub fn verify(&mut self, claims: &AggregateClaims, proof: &[u8]) -> Result { let body = self.call(&wire::encode_verify(claims, proof), 1 << 16)?; match wire::decode_verdict(&body) { Some(Ok(valid)) => Ok(valid), @@ -141,14 +122,12 @@ impl Client { match wire::read_frame(&mut self.stream, max_response)? { Incoming::Body(body) => Ok(body), Incoming::End => Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof).into()), - Incoming::TooLarge(_) => Err(SidecarError::Protocol("a response longer than any".into())), + Incoming::TooLarge(len) => Err(SidecarError::Protocol(format!( + "a response of {len} bytes, longer than any" + ))), } } } /// Room for a proof in a response, far more than one takes. const PROOF_ROOM: u64 = 1 << 28; - -fn claims_of(raw: &[(K, [u8; 32], S)]) -> Vec<(K, [u8; 32])> { - raw.iter().map(|(pk, message, _)| (*pk, *message)).collect() -} diff --git a/crates/sidecar/src/lib.rs b/crates/sidecar/src/lib.rs index 40110ea7e2..67df334f52 100644 --- a/crates/sidecar/src/lib.rs +++ b/crates/sidecar/src/lib.rs @@ -1,28 +1,33 @@ -//! The leanVM sidecar: a process that proves for clients on the same machine. +//! The leanVM sidecar: a process that proves and verifies for clients on the same machine. //! -//! A client calls the library's `aggregate` over a Unix stream socket, and verifies proofs with the library, or with -//! the sidecar if it has no binding to it. The sidecar keeps the prover's allocator, threads and warm state out of -//! the client's process. `bins/leanvm-sidecar` runs [`serve`]; its `README.md` specifies the protocol, which any -//! language can speak, and [`Client`] speaks it from Rust. +//! A client calls the library's `aggregate` and `verify` over a Unix stream socket, so it builds no prover: the +//! sidecar keeps the prover's allocator, threads and warm state out of the client's process. `bins/leanvm-sidecar` +//! runs `serve`, which the `server` feature brings; its `README.md` specifies the protocol, which any language can +//! speak, and [`Client`] speaks it from Rust. mod client; +#[cfg(feature = "server")] mod server; mod wire; pub use client::Client; +#[cfg(feature = "server")] pub use server::{Config, serve}; +/// One blob, as `lean_da` takes it: its symbols, each 128 bits. +pub type Blob = [u128; 1 << verifier::da::LOG_BLOB_SYMBOLS]; + /// Why the sidecar made no proof. #[derive(Clone, Copy, Debug, PartialEq, Eq)] #[repr(u8)] pub enum Rejection { - /// [`aggregation::AggregateError::InvalidInput`], or more blobs than one proof takes. + /// `AggregateError::InvalidInput`, or more blobs than one proof takes. InvalidInput = 1, - /// [`aggregation::AggregateError::InvalidSignature`]. + /// `AggregateError::InvalidSignature`. InvalidSignature = 2, - /// [`aggregation::AggregateError::InvalidChild`], or a child proof's bytes that do not decode. + /// `AggregateError::InvalidChild`, or a child proof's bytes that do not decode. InvalidChild = 3, - /// [`aggregation::AggregateError::ProvingFailed`]. + /// `AggregateError::ProvingFailed`. ProvingFailed = 4, /// The request is not one: its bytes do not decode, or it is longer than the sidecar takes. Malformed = 5, @@ -47,9 +52,9 @@ impl Rejection { pub enum SidecarError { /// The socket failed, or the sidecar closed it: it stopped, or crashed. Connect again. Io(std::io::Error), - /// The sidecar speaks another protocol version, or proves for another verifier or aggregation program. + /// The sidecar speaks another protocol version, or proves for another verifier. Incompatible(String), - /// The sidecar's bytes are not the protocol, or prove other claims than asked for. Connect again. + /// The sidecar's bytes are not the protocol. Connect again. Protocol(String), /// The sidecar made no proof, and says why. The connection stays usable. Rejected(Rejection, String), diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs index c7a513f9ab..ab9e8d8997 100644 --- a/crates/sidecar/src/server.rs +++ b/crates/sidecar/src/server.rs @@ -1,10 +1,15 @@ //! The server: a thread per connection reads and decodes requests and verifies proofs, and the thread that calls -//! [`serve`], the one that started the prover's pool, proves them one at a time. +//! [`serve`], the one that started the prover's pool, proves them one at a time. Also the server's half of the +//! protocol: requests in, the hello and responses out. -use crate::Rejection; -use crate::wire::{self, Call, Hello, Incoming, Outcome, Request}; -use aggregation::{AggregateError, OutputClaims}; +use crate::wire::{ + self, Cursor, Frame, Hello, Incoming, OK, OP_AGGREGATE, OP_VERIFY, Outcome, Proven, RawLeanSphincs, RawLeanXmss, +}; +use crate::{Blob, Rejection}; +use aggregation::{AggregateClaims, AggregateError, AggregateProof, OutputClaims}; use isa::Prover; +use lean_sphincs::{LeanSphincsPublicKey, LeanSphincsSignature}; +use lean_xmss::{LeanXmssPublicKey, LeanXmssSignature}; use std::convert::Infallible; use std::fs::{File, TryLockError}; use std::io::{self, Write}; @@ -47,7 +52,7 @@ pub fn serve(config: &Config) -> io::Result { let program = aggregation::unified_guest().bytecode().fiat_shamir_iv(); warm_up::

(); let listener = UnixListener::bind(&config.socket)?; - let hello = wire::encode_hello(&Hello { + let hello = encode_hello(&Hello { max_request: config.max_request, tag: verifier::TAG.to_vec(), program, @@ -168,6 +173,15 @@ fn hung_up(client: &UnixStream) -> bool { } fn aggregate(request: Request) -> Outcome { + let children = request + .children + .into_iter() + .enumerate() + .map(|(i, (claims, proof))| match AggregateProof::from_bytes(&proof) { + Some(proof) => Ok((claims, proof)), + None => Err((Rejection::InvalidChild, format!("child {i}'s proof does not decode"))), + }) + .collect::, _>>()?; let blobs = match request.blobs.len() { 0 => None, _ => Some(lean_da::BlobMatrix::build(&request.blobs).map_err(|e| (Rejection::InvalidInput, e.to_string()))?), @@ -175,14 +189,14 @@ fn aggregate(request: Request) -> Outcome { drop(request.blobs); let output = request.output.as_ref().map_or(OutputClaims::All, OutputClaims::Only); aggregation::aggregate::

( - &request.children, + &children, request.raw_lean_xmss, request.raw_lean_sphincs, blobs.as_ref(), output, request.log_inv_rate, ) - .map(|(claims, proof, _)| (claims, proof)) + .map(|(claims, proof, _)| (claims, proof.to_bytes())) .map_err(|e| match e { AggregateError::InvalidInput(reason) => (Rejection::InvalidInput, reason.into()), AggregateError::InvalidSignature => (Rejection::InvalidSignature, e.to_string()), @@ -224,12 +238,12 @@ fn connection(mut stream: UnixStream, jobs: &Sender, hello: &[u8], max_requ Ok(Incoming::TooLarge(len)) => { let message = format!("a request of {len} bytes, the sidecar takes at most {max_request}"); eprintln!("a request refused (Malformed): {message}"); - let _ = stream.write_all(&wire::encode_response(&Err((Rejection::Malformed, message)))); + let _ = stream.write_all(&encode_response(&Err((Rejection::Malformed, message)))); return; } Ok(Incoming::End) | Err(_) => return, }; - let response = match wire::decode_request(&body) { + let response = match decode_request(&body) { Ok(Call::Aggregate(request)) => { drop(body); let Ok(client) = stream.try_clone() else { return }; @@ -238,13 +252,15 @@ fn connection(mut stream: UnixStream, jobs: &Sender, hello: &[u8], max_requ .expect("the prover outlives every connection"); // No outcome: the prover dropped the request, its client having hung up. let Ok(outcome) = outcome.recv() else { return }; - wire::encode_response(&outcome) + encode_response(&outcome) } // Here rather than behind the proofs: it is fast, and on one thread. - Ok(Call::Verify(claims, proof)) => wire::encode_verdict(proof.is_some_and(|proof| proof.verify(&claims))), + Ok(Call::Verify(claims, proof)) => { + encode_verdict(AggregateProof::from_bytes(&proof).is_some_and(|proof| proof.verify(&claims))) + } Err((rejection, message)) => { eprintln!("a request refused ({rejection:?}): {message}"); - wire::encode_response(&Err((rejection, message))) + encode_response(&Err((rejection, message))) } }; if stream.write_all(&response).is_err() { @@ -252,3 +268,242 @@ fn connection(mut stream: UnixStream, jobs: &Sender, hello: &[u8], max_requ } } } + +const XMSS_RECORD: usize = lean_xmss::PUB_KEY_SIZE + lean_xmss::MESSAGE_LEN + lean_xmss::SIG_SIZE; +const SPHINCS_RECORD: usize = lean_sphincs::PUB_KEY_SIZE + lean_sphincs::MESSAGE_LEN + lean_sphincs::SIG_SIZE; +const BLOB_BYTES: usize = 16 * lean_da::BLOB_SYMBOLS; + +fn encode_hello(hello: &Hello) -> Vec { + let mut f = Frame::new(); + f.raw(&wire::MAGIC); + f.raw(&wire::VERSION.to_le_bytes()); + f.u64(hello.max_request); + f.bytes(&hello.tag); + f.raw(&hello.program); + f.bytes(hello.prover.as_bytes()); + f.finish() +} + +/// A request, as the server decodes it. +enum Call { + Aggregate(Request), + /// Claims, and their proof's bytes. + Verify(AggregateClaims, Vec), +} + +/// An `aggregate` call, its children's proofs still bytes. +struct Request { + children: Vec, + raw_lean_xmss: Vec, + raw_lean_sphincs: Vec, + blobs: Vec, + /// `None` for [`OutputClaims::All`]. + output: Option, + log_inv_rate: usize, +} + +fn decode_request(body: &[u8]) -> Result { + let malformed = |what: &str| (Rejection::Malformed, format!("malformed request: {what}")); + let mut c = Cursor(body); + let call = match c.u8() { + Some(OP_AGGREGATE) => Call::Aggregate(decode_aggregate(&mut c, malformed)?), + Some(OP_VERIFY) => { + let claims = c.claims().ok_or_else(|| malformed("claims"))?; + let proof = c.bytes().ok_or_else(|| malformed("proof"))?; + Call::Verify(claims, proof.to_vec()) + } + Some(op) => return Err((Rejection::Malformed, format!("unknown operation {op}"))), + None => return Err(malformed("empty")), + }; + if !c.is_empty() { + return Err(malformed("trailing bytes")); + } + Ok(call) +} + +/// A count of records of `size` bytes each, all of which must follow: a forged count allocates nothing. +fn count(c: &mut Cursor, size: usize) -> Option { + let n = usize::try_from(c.u64()?).ok()?; + (n.checked_mul(size)? <= c.0.len()).then_some(n) +} + +fn decode_aggregate( + c: &mut Cursor, + malformed: impl Fn(&str) -> (Rejection, String), +) -> Result { + let log_inv_rate = c.u8().ok_or_else(|| malformed("no rate"))?.into(); + + // Each child is at least its two lengths. Nothing is reserved: `aggregate` bounds the count, not this. + let n = count(c, 16).ok_or_else(|| malformed("children"))?; + let mut children = Vec::new(); + for i in 0..n { + let claims = c.claims().ok_or_else(|| malformed(&format!("child {i}'s claims")))?; + let proof = c.bytes().ok_or_else(|| malformed(&format!("child {i}'s proof")))?; + children.push((claims, proof.to_vec())); + } + + let n = count(c, XMSS_RECORD).ok_or_else(|| malformed("leanXMSS signatures"))?; + let raw_lean_xmss = (0..n) + .map(|_| { + let pk = LeanXmssPublicKey::from_bytes(c.array().unwrap()); + ( + pk, + *c.array().unwrap(), + LeanXmssSignature::from_bytes(c.array().unwrap()), + ) + }) + .collect(); + let n = count(c, SPHINCS_RECORD).ok_or_else(|| malformed("leanSPHINCS signatures"))?; + let raw_lean_sphincs = (0..n) + .map(|_| { + let pk = LeanSphincsPublicKey::from_bytes(c.array().unwrap()); + ( + pk, + *c.array().unwrap(), + LeanSphincsSignature::from_bytes(c.array().unwrap()), + ) + }) + .collect(); + + let n = count(c, BLOB_BYTES).ok_or_else(|| malformed("blobs"))?; + let blobs = (0..n) + .map(|_| { + let symbols = c.take(BLOB_BYTES).unwrap().as_chunks::<16>().0; + std::array::from_fn(|i| u128::from_le_bytes(symbols[i])) + }) + .collect(); + + let output = match c.u8() { + Some(0) => None, + Some(1) => Some(c.claims().ok_or_else(|| malformed("output claims"))?), + _ => return Err(malformed("output")), + }; + Ok(Request { + children, + raw_lean_xmss, + raw_lean_sphincs, + blobs, + output, + log_inv_rate, + }) +} + +fn encode_response(result: &Outcome) -> Vec { + let mut f = Frame::new(); + match result { + Ok((claims, proof)) => { + f.u8(OK); + f.bytes(&claims.to_bytes()); + f.bytes(proof); + } + Err((rejection, message)) => { + f.u8(*rejection as u8); + f.bytes(message.as_bytes()); + } + } + f.finish() +} + +/// A verify request's response: whether the proof is valid. +fn encode_verdict(valid: bool) -> Vec { + let mut f = Frame::new(); + f.u8(OK); + f.u8(valid.into()); + f.finish() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::wire::{decode_hello, decode_response, encode_request, hello_version}; + + fn body(frame: Vec) -> Vec { + assert_eq!( + u64::from_le_bytes(frame[..8].try_into().unwrap()), + frame.len() as u64 - 8 + ); + frame[8..].to_vec() + } + + #[test] + fn hello_round_trips() { + let hello = Hello { + max_request: 1 << 30, + tag: verifier::TAG.to_vec(), + program: [7; 32], + prover: "stable".into(), + }; + let bytes = body(encode_hello(&hello)); + assert_eq!(hello_version(&bytes), Some(wire::VERSION)); + assert_eq!(decode_hello(&bytes), Some(hello)); + assert_eq!(decode_hello(&bytes[..bytes.len() - 1]), None); + assert_eq!(decode_hello(&[&bytes[..], &[0]].concat()), None); + let mut other = bytes.clone(); + other[0] ^= 1; + assert_eq!(hello_version(&other), None); + } + + /// A request decodes to what was encoded; every truncation, and a trailing byte, is refused, never a panic. + #[test] + fn requests_round_trip_and_truncations_are_refused() { + let xmss = ( + LeanXmssPublicKey::from_bytes(&[1; 32]), + [2; 32], + LeanXmssSignature::from_bytes(&std::array::from_fn(|i| i as u8)), + ); + let sphincs = ( + LeanSphincsPublicKey::from_bytes(&[3; 32]), + [4; 32], + LeanSphincsSignature::from_bytes(&std::array::from_fn(|i| (i * 7) as u8)), + ); + let blob: Blob = std::array::from_fn(|i| ((i as u128) << 100) | i as u128); + let only = AggregateClaims { + lean_xmss: vec![(xmss.0, xmss.1)], + lean_sphincs: vec![(sphincs.0, sphincs.1)], + da_commitment: Some([5; 32]), + }; + let child = (only.clone(), vec![6, 7, 8]); + let frame = encode_request( + std::slice::from_ref(&child), + std::slice::from_ref(&xmss), + std::slice::from_ref(&sphincs), + &[blob], + OutputClaims::Only(&only), + 3, + ); + let bytes = body(frame); + let Ok(Call::Aggregate(request)) = decode_request(&bytes) else { + panic!("not an aggregate request") + }; + assert_eq!(request.children, vec![child]); + assert_eq!(request.raw_lean_xmss, vec![xmss]); + assert_eq!(request.raw_lean_sphincs, vec![sphincs]); + assert_eq!(request.blobs, vec![blob]); + assert_eq!(request.output, Some(only)); + assert_eq!(request.log_inv_rate, 3); + for len in 0..bytes.len() { + assert!( + decode_request(&bytes[..len]).is_err(), + "a prefix of {len} bytes decoded" + ); + } + assert!(decode_request(&[&bytes[..], &[0]].concat()).is_err()); + // A count no body could hold. + let mut huge = vec![OP_AGGREGATE, 1]; + huge.extend(u64::MAX.to_le_bytes()); + assert!(decode_request(&huge).is_err()); + } + + #[test] + fn responses_round_trip() { + let proven = (AggregateClaims::default(), vec![1, 2, 3]); + assert_eq!( + decode_response(&body(encode_response(&Ok(proven.clone())))), + Some(Ok(proven)) + ); + let rejection = (Rejection::InvalidInput, "log_inv_rate out of range".to_string()); + let bytes = body(encode_response(&Err(rejection.clone()))); + assert_eq!(decode_response(&bytes).unwrap().unwrap_err(), rejection); + assert!(decode_response(&[42, 0, 0, 0, 0, 0, 0, 0, 0]).is_none()); + } +} diff --git a/crates/sidecar/src/wire.rs b/crates/sidecar/src/wire.rs index 7e3375511f..8849fe5024 100644 --- a/crates/sidecar/src/wire.rs +++ b/crates/sidecar/src/wire.rs @@ -1,9 +1,9 @@ -//! The protocol's bytes, as `bins/leanvm-sidecar/README.md` specifies them: frames, the hello, requests and -//! responses. Every integer is little-endian; every variable-length field is a `u64` length, then its bytes. +//! The protocol's bytes, as `bins/leanvm-sidecar/README.md` specifies them: frames, the hello, and the client's half, +//! requests out and responses in (the server's half is beside the server). Every integer is little-endian; every +//! variable-length field is a `u64` length, then its bytes. -use crate::Rejection; -use aggregation::{AggregateClaims, AggregateProof, OutputClaims}; -use lean_da::Blob; +use crate::{Blob, Rejection}; +use claims::{AggregateClaims, OutputClaims}; use lean_sphincs::{LeanSphincsPublicKey, LeanSphincsSignature}; use lean_xmss::{LeanXmssPublicKey, LeanXmssSignature}; use std::io::{self, Read}; @@ -14,46 +14,43 @@ pub(crate) const MAGIC: [u8; 8] = *b"leanvmsc"; /// The protocol's version, which the hello announces right after [`MAGIC`], where every version keeps it. pub(crate) const VERSION: u32 = 1; -const OP_AGGREGATE: u8 = 1; -const OP_VERIFY: u8 = 2; -const OK: u8 = 0; - -const XMSS_RECORD: usize = lean_xmss::PUB_KEY_SIZE + lean_xmss::MESSAGE_LEN + lean_xmss::SIG_SIZE; -const SPHINCS_RECORD: usize = lean_sphincs::PUB_KEY_SIZE + lean_sphincs::MESSAGE_LEN + lean_sphincs::SIG_SIZE; -const BLOB_BYTES: usize = 16 * lean_da::BLOB_SYMBOLS; +pub(crate) const OP_AGGREGATE: u8 = 1; +pub(crate) const OP_VERIFY: u8 = 2; +pub(crate) const OK: u8 = 0; pub(crate) type RawLeanXmss = (LeanXmssPublicKey, lean_xmss::Message, LeanXmssSignature); pub(crate) type RawLeanSphincs = (LeanSphincsPublicKey, lean_sphincs::Message, LeanSphincsSignature); -pub(crate) type Proven = (AggregateClaims, AggregateProof); +/// Claims, and the bytes of their proof. +pub(crate) type Proven = (AggregateClaims, Vec); /// A request's proof, or why there is none. pub(crate) type Outcome = Result; /// A frame being built: its length is patched in by [`Self::finish`]. -struct Frame(Vec); +pub(crate) struct Frame(Vec); impl Frame { - fn new() -> Self { + pub(crate) fn new() -> Self { Self(vec![0; 8]) } - fn u8(&mut self, x: u8) { + pub(crate) fn u8(&mut self, x: u8) { self.0.push(x); } - fn u64(&mut self, x: u64) { + pub(crate) fn u64(&mut self, x: u64) { self.0.extend(x.to_le_bytes()); } - fn raw(&mut self, bytes: &[u8]) { + pub(crate) fn raw(&mut self, bytes: &[u8]) { self.0.extend(bytes); } - fn bytes(&mut self, bytes: &[u8]) { + pub(crate) fn bytes(&mut self, bytes: &[u8]) { self.u64(bytes.len() as u64); self.raw(bytes); } - fn finish(mut self) -> Vec { + pub(crate) fn finish(mut self) -> Vec { let len = (self.0.len() as u64 - 8).to_le_bytes(); self.0[..8].copy_from_slice(&len); self.0 @@ -95,20 +92,20 @@ pub(crate) fn read_frame(r: &mut impl Read, max_len: u64) -> io::Result(&'a [u8]); +pub(crate) struct Cursor<'a>(pub(crate) &'a [u8]); impl<'a> Cursor<'a> { - fn take(&mut self, n: usize) -> Option<&'a [u8]> { + pub(crate) fn take(&mut self, n: usize) -> Option<&'a [u8]> { let (head, rest) = self.0.split_at_checked(n)?; self.0 = rest; Some(head) } - fn array(&mut self) -> Option<&'a [u8; N]> { + pub(crate) fn array(&mut self) -> Option<&'a [u8; N]> { self.take(N).map(|bytes| bytes.try_into().unwrap()) } - fn u8(&mut self) -> Option { + pub(crate) fn u8(&mut self) -> Option { self.array::<1>().map(|[x]| *x) } @@ -116,28 +113,16 @@ impl<'a> Cursor<'a> { self.array().map(|x| u32::from_le_bytes(*x)) } - fn u64(&mut self) -> Option { + pub(crate) fn u64(&mut self) -> Option { self.array().map(|x| u64::from_le_bytes(*x)) } - fn bytes(&mut self) -> Option<&'a [u8]> { + pub(crate) fn bytes(&mut self) -> Option<&'a [u8]> { let len = usize::try_from(self.u64()?).ok()?; self.take(len) } - /// A count of records of `size` bytes each, all of which must follow: a forged count allocates nothing. - fn count(&mut self, size: usize) -> Option { - let n = usize::try_from(self.u64()?).ok()?; - (n.checked_mul(size)? <= self.0.len()).then_some(n) - } - - /// A count of records of `size` bytes each, then the records, each read by `record`. - fn records(&mut self, size: usize, record: impl Fn(&mut Self) -> T) -> Option> { - let n = self.count(size)?; - Some((0..n).map(|_| record(self)).collect()) - } - - fn claims(&mut self) -> Option { + pub(crate) fn claims(&mut self) -> Option { AggregateClaims::from_bytes(self.bytes()?) } @@ -145,7 +130,7 @@ impl<'a> Cursor<'a> { String::from_utf8(self.bytes()?.to_vec()).ok() } - fn is_empty(&self) -> bool { + pub(crate) fn is_empty(&self) -> bool { self.0.is_empty() } } @@ -160,17 +145,6 @@ pub(crate) struct Hello { pub prover: String, } -pub(crate) fn encode_hello(hello: &Hello) -> Vec { - let mut f = Frame::new(); - f.raw(&MAGIC); - f.raw(&VERSION.to_le_bytes()); - f.u64(hello.max_request); - f.bytes(&hello.tag); - f.raw(&hello.program); - f.bytes(hello.prover.as_bytes()); - f.finish() -} - /// The version a hello announces, `None` unless the body starts with [`MAGIC`]. pub(crate) fn hello_version(body: &[u8]) -> Option { let mut c = Cursor(body); @@ -190,24 +164,6 @@ pub(crate) fn decode_hello(body: &[u8]) -> Option { c.is_empty().then_some(hello) } -/// A request, as the server decodes it. -pub(crate) enum Call { - Aggregate(Request), - /// Claims, and their proof unless its bytes do not decode, which makes it invalid. - Verify(AggregateClaims, Option), -} - -/// An `aggregate` call. -pub(crate) struct Request { - pub children: Vec, - pub raw_lean_xmss: Vec, - pub raw_lean_sphincs: Vec, - pub blobs: Vec, - /// `None` for [`OutputClaims::All`]. - pub output: Option, - pub log_inv_rate: usize, -} - pub(crate) fn encode_request( children: &[Proven], raw_lean_xmss: &[RawLeanXmss], @@ -223,7 +179,7 @@ pub(crate) fn encode_request( f.u64(children.len() as u64); for (claims, proof) in children { f.bytes(&claims.to_bytes()); - f.bytes(&proof.to_bytes()); + f.bytes(proof); } f.u64(raw_lean_xmss.len() as u64); for (pk, message, signature) in raw_lean_xmss { @@ -253,116 +209,11 @@ pub(crate) fn encode_request( f.finish() } -pub(crate) fn encode_verify(claims: &AggregateClaims, proof: &AggregateProof) -> Vec { +pub(crate) fn encode_verify(claims: &AggregateClaims, proof: &[u8]) -> Vec { let mut f = Frame::new(); f.u8(OP_VERIFY); f.bytes(&claims.to_bytes()); - f.bytes(&proof.to_bytes()); - f.finish() -} - -pub(crate) fn decode_request(body: &[u8]) -> Result { - let malformed = |what: &str| (Rejection::Malformed, format!("malformed request: {what}")); - let mut c = Cursor(body); - let call = match c.u8() { - Some(OP_AGGREGATE) => Call::Aggregate(decode_aggregate(&mut c, malformed)?), - Some(OP_VERIFY) => { - let claims = c.claims().ok_or_else(|| malformed("claims"))?; - let proof = c.bytes().ok_or_else(|| malformed("proof"))?; - Call::Verify(claims, AggregateProof::from_bytes(proof)) - } - Some(op) => return Err((Rejection::Malformed, format!("unknown operation {op}"))), - None => return Err(malformed("empty")), - }; - if !c.is_empty() { - return Err(malformed("trailing bytes")); - } - Ok(call) -} - -fn decode_aggregate( - c: &mut Cursor, - malformed: impl Fn(&str) -> (Rejection, String), -) -> Result { - let log_inv_rate = c.u8().ok_or_else(|| malformed("no rate"))?.into(); - - // Each child is at least its two lengths. Nothing is reserved: `aggregate` bounds the count, not this. - let n = c.count(16).ok_or_else(|| malformed("children"))?; - let mut children = Vec::new(); - for i in 0..n { - let claims = c.claims().ok_or_else(|| malformed(&format!("child {i}'s claims")))?; - let proof = c.bytes().ok_or_else(|| malformed(&format!("child {i}'s proof")))?; - let proof = AggregateProof::from_bytes(proof) - .ok_or_else(|| (Rejection::InvalidChild, format!("child {i}'s proof does not decode")))?; - children.push((claims, proof)); - } - - let raw_lean_xmss = c - .records(XMSS_RECORD, |c| { - let pk = LeanXmssPublicKey::from_bytes(c.array().unwrap()); - ( - pk, - *c.array().unwrap(), - LeanXmssSignature::from_bytes(c.array().unwrap()), - ) - }) - .ok_or_else(|| malformed("leanXMSS signatures"))?; - let raw_lean_sphincs = c - .records(SPHINCS_RECORD, |c| { - let pk = LeanSphincsPublicKey::from_bytes(c.array().unwrap()); - ( - pk, - *c.array().unwrap(), - LeanSphincsSignature::from_bytes(c.array().unwrap()), - ) - }) - .ok_or_else(|| malformed("leanSPHINCS signatures"))?; - - let n = c.count(BLOB_BYTES).ok_or_else(|| malformed("blobs"))?; - let mut blobs = vec![[0; lean_da::BLOB_SYMBOLS]; n]; - for blob in &mut blobs { - let bytes = c.take(BLOB_BYTES).unwrap(); - for (symbol, bytes) in blob.iter_mut().zip(bytes.as_chunks::<16>().0) { - *symbol = u128::from_le_bytes(*bytes); - } - } - - let output = match c.u8() { - Some(0) => None, - Some(1) => Some(c.claims().ok_or_else(|| malformed("output claims"))?), - _ => return Err(malformed("output")), - }; - Ok(Request { - children, - raw_lean_xmss, - raw_lean_sphincs, - blobs, - output, - log_inv_rate, - }) -} - -pub(crate) fn encode_response(result: &Outcome) -> Vec { - let mut f = Frame::new(); - match result { - Ok((claims, proof)) => { - f.u8(OK); - f.bytes(&claims.to_bytes()); - f.bytes(&proof.to_bytes()); - } - Err((rejection, message)) => { - f.u8(*rejection as u8); - f.bytes(message.as_bytes()); - } - } - f.finish() -} - -/// A verify request's response: whether the proof is valid. -pub(crate) fn encode_verdict(valid: bool) -> Vec { - let mut f = Frame::new(); - f.u8(OK); - f.u8(valid.into()); + f.bytes(proof); f.finish() } @@ -384,97 +235,8 @@ pub(crate) fn decode_verdict(body: &[u8]) -> Option Option { let mut c = Cursor(body); let result = match c.u8()? { - OK => Ok((c.claims()?, AggregateProof::from_bytes(c.bytes()?)?)), + OK => Ok((c.claims()?, c.bytes()?.to_vec())), status => Err((Rejection::from_code(status)?, c.string()?)), }; c.is_empty().then_some(result) } - -#[cfg(test)] -mod tests { - use super::*; - - fn body(frame: Vec) -> Vec { - assert_eq!( - u64::from_le_bytes(frame[..8].try_into().unwrap()), - frame.len() as u64 - 8 - ); - frame[8..].to_vec() - } - - #[test] - fn hello_round_trips() { - let hello = Hello { - max_request: 1 << 30, - tag: verifier::TAG.to_vec(), - program: [7; 32], - prover: "stable".into(), - }; - let bytes = body(encode_hello(&hello)); - assert_eq!(hello_version(&bytes), Some(VERSION)); - assert_eq!(decode_hello(&bytes), Some(hello)); - assert_eq!(decode_hello(&bytes[..bytes.len() - 1]), None); - assert_eq!(decode_hello(&[&bytes[..], &[0]].concat()), None); - let mut other = bytes.clone(); - other[0] ^= 1; - assert_eq!(hello_version(&other), None); - } - - /// A request decodes to what was encoded; every truncation, and a trailing byte, is refused, never a panic. - #[test] - fn requests_round_trip_and_truncations_are_refused() { - let xmss = ( - LeanXmssPublicKey::from_bytes(&[1; 32]), - [2; 32], - LeanXmssSignature::from_bytes(&std::array::from_fn(|i| i as u8)), - ); - let sphincs = ( - LeanSphincsPublicKey::from_bytes(&[3; 32]), - [4; 32], - LeanSphincsSignature::from_bytes(&std::array::from_fn(|i| (i * 7) as u8)), - ); - let blob: Blob = std::array::from_fn(|i| ((i as u128) << 100) | i as u128); - let only = AggregateClaims { - lean_xmss: vec![(xmss.0, xmss.1)], - lean_sphincs: vec![(sphincs.0, sphincs.1)], - da_commitment: Some([5; 32]), - }; - let frame = encode_request( - &[], - std::slice::from_ref(&xmss), - std::slice::from_ref(&sphincs), - &[blob], - OutputClaims::Only(&only), - 3, - ); - let bytes = body(frame); - let Ok(Call::Aggregate(request)) = decode_request(&bytes) else { - panic!("not an aggregate request") - }; - assert!(request.children.is_empty()); - assert_eq!(request.raw_lean_xmss, vec![xmss]); - assert_eq!(request.raw_lean_sphincs, vec![sphincs]); - assert_eq!(request.blobs, vec![blob]); - assert_eq!(request.output, Some(only)); - assert_eq!(request.log_inv_rate, 3); - for len in 0..bytes.len() { - assert!( - decode_request(&bytes[..len]).is_err(), - "a prefix of {len} bytes decoded" - ); - } - assert!(decode_request(&[&bytes[..], &[0]].concat()).is_err()); - // A count no body could hold. - let mut huge = vec![OP_AGGREGATE, 1]; - huge.extend(u64::MAX.to_le_bytes()); - assert!(decode_request(&huge).is_err()); - } - - #[test] - fn rejections_round_trip() { - let rejection = (Rejection::InvalidInput, "log_inv_rate out of range".to_string()); - let bytes = body(encode_response(&Err(rejection.clone()))); - assert_eq!(decode_response(&bytes).unwrap().unwrap_err(), rejection); - assert!(decode_response(&[42, 0, 0, 0, 0, 0, 0, 0, 0]).is_none()); - } -} diff --git a/formal/lean_vm/scripts/guest-export/Cargo.lock b/formal/lean_vm/scripts/guest-export/Cargo.lock index ce924b30ef..f4344c915a 100644 --- a/formal/lean_vm/scripts/guest-export/Cargo.lock +++ b/formal/lean_vm/scripts/guest-export/Cargo.lock @@ -8,6 +8,14 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "claims" +version = "0.1.0" +dependencies = [ + "lean_sphincs", + "lean_xmss", +] + [[package]] name = "fiat_shamir_prover" version = "0.1.0" @@ -68,6 +76,7 @@ dependencies = [ name = "leanvm-guest-export" version = "0.0.0" dependencies = [ + "claims", "fiat_shamir_prover", "isa", "keccak256", diff --git a/formal/lean_vm/scripts/guest-export/Cargo.toml b/formal/lean_vm/scripts/guest-export/Cargo.toml index 6f12380eb1..33c7ff21a6 100644 --- a/formal/lean_vm/scripts/guest-export/Cargo.toml +++ b/formal/lean_vm/scripts/guest-export/Cargo.toml @@ -37,6 +37,7 @@ verifier = { path = "../../../../crates/verifier" } keccak256 = { path = "../../../../crates/keccak256" } fiat_shamir_prover = { path = "../../../../crates/fiat_shamir_prover" } zkdsl_compiler = { path = "../../../../crates/zkdsl_compiler" } +claims = { path = "../../../../crates/claims" } lean_xmss = { path = "../../../../crates/lean_xmss" } lean_sphincs = { path = "../../../../crates/lean_sphincs" } lean_da = { path = "../../../../crates/lean_da" } diff --git a/formal/lean_vm/scripts/guest-export/src/export/guest_diff.rs b/formal/lean_vm/scripts/guest-export/src/export/guest_diff.rs index 75018339c8..5e4a6a2197 100644 --- a/formal/lean_vm/scripts/guest-export/src/export/guest_diff.rs +++ b/formal/lean_vm/scripts/guest-export/src/export/guest_diff.rs @@ -17,8 +17,8 @@ use isa::Prover; use lean_vm_core::StableProver; use lean_vm_core::cpu::{from_scalar, public_input_bytes}; use primitives::field::F128; -use lean_sphincs::LeanSphincsSignature; -use lean_xmss::LeanXmssSignature; +use lean_sphincs::{LeanSphincsPublicKey, LeanSphincsSignature}; +use lean_xmss::{LeanXmssPublicKey, LeanXmssSignature}; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; use std::fmt::Write as _; diff --git a/formal/lean_vm/scripts/guest-export/src/export/node_vectors.rs b/formal/lean_vm/scripts/guest-export/src/export/node_vectors.rs index a8f2094c4e..bd86db060e 100644 --- a/formal/lean_vm/scripts/guest-export/src/export/node_vectors.rs +++ b/formal/lean_vm/scripts/guest-export/src/export/node_vectors.rs @@ -7,6 +7,8 @@ //! are hand-written in `tests/LeanVMTests/Spec/Node.lean`. use super::*; +use lean_sphincs::LeanSphincsPublicKey; +use lean_xmss::LeanXmssPublicKey; use lean_vm_core::StableProver; use lean_vm_core::cpu::{from_scalar, to_scalar}; use primitives::field::F128; From 0f4bd54858fb344dcf9b825d754199126a9e0636 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Tue, 17 Nov 2026 19:30:28 -0500 Subject: [PATCH 09/34] sidecar: warm up at rate 1/4, the README's hello as the client checks it, the site as main has it The warm-up proved at the highest rate, whose memory the process then keeps: on the stable prover its peak RSS was 11.9 GB, now 4.8 GB at rate 1/4 (M4 Max), ready in about 2 s either way. A higher rate pays for its extra memory on its first proof. The client checks the hello's magic, version and verifier tag; the program identifies the build, which only a client that also verifies with the library checks. The site's reproduce command describes the benchmarked commit it pins, from before the rename, so it stays as on main. Co-Authored-By: Claude Opus 5.5 --- bins/leanvm-sidecar/README.md | 2 +- crates/sidecar/src/server.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index a33b3ff984..c4b64dae72 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -57,7 +57,7 @@ On each connection the sidecar first sends a hello frame. The client then sends | program | 32 bytes | the aggregation program's Fiat-Shamir IV, `Bytecode::fiat_shamir_iv` | | prover | bytes | `stable` or `experimental` | -Every version keeps the magic and the version first. A client checks them, the verifier tag and the program, and disconnects if any differs from what it expects: a proof for another verifier or another build's program does not verify with its library. A client must not send a request body longer than the max request: the sidecar would close the connection without reading it, and the client could fail mid-write. +Every version keeps the magic and the version first. A client checks them and the verifier tag, and disconnects if any differs from what it expects: a proof for another verifier does not verify. The program identifies the leanVM build: a client that also verifies with the library checks it too, since another build's proofs do not verify there. A client must not send a request body longer than the max request: the sidecar would close the connection without reading it, and the client could fail mid-write. ### Requests diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs index ab9e8d8997..fbfb625552 100644 --- a/crates/sidecar/src/server.rs +++ b/crates/sidecar/src/server.rs @@ -72,11 +72,11 @@ pub fn serve(config: &Config) -> io::Result { } /// A fresh process's first proofs are slower, touching memory the allocator then keeps: this pays for it before the -/// first request, with a discarded leaf and a full recursion over it, at the highest rate, whose memory is the -/// largest. A blob is the one input that needs no key. +/// first request, with a discarded leaf and a full recursion over it, at rate 1/4, the one used in practice (a higher +/// rate still pays for its extra memory on its first proof). A blob is the one input that needs no key. fn warm_up() { let blobs = lean_da::BlobMatrix::build(&[[0; lean_da::BLOB_SYMBOLS]]).expect("one blob"); - let rate = aggregation::MAX_LOG_INV_RATE; + let rate = 2; let leaf = aggregation::aggregate::

(&[], vec![], vec![], Some(&blobs), OutputClaims::All, rate) .expect("the warm-up's leaf proves"); let children = vec![(leaf.0, leaf.1); aggregation::MAX_RECURSIONS]; From c6e8de3218f59a22b4961dce22b3665668ff0593 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Tue, 17 Nov 2026 19:32:45 -0500 Subject: [PATCH 10/34] review fixes: the no-prover lint over every target, the thread count on experimental, the leaves' file, the socket's directory - lean_vm's `api` test needs the `prover` feature, so CI lints the build without it with `--all-targets`. - `--threads` and `setup` say that on experimental they count the performance cores' workers. - `write_leaves` describes the file as it is: the claims carry their length too. - The sidecar's README: the socket goes in a directory only its user can write, and a connection may hold about twice the max request. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/rust.yml | 2 +- bench/src/benchmark.rs | 4 ++-- bench/src/main.rs | 3 ++- bins/leanvm-sidecar/README.md | 3 ++- bins/leanvm-sidecar/src/main.rs | 3 ++- crates/lean_vm/Cargo.toml | 4 ++++ crates/lean_vm/src/prover.rs | 3 ++- 7 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8a61996d48..008330d8eb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -67,7 +67,7 @@ jobs: - name: Clippy (the API on the experimental prover) run: cargo clippy --release -p lean_vm --all-targets --features experimental_prover -- -D warnings - name: Clippy (the API without the prover, the sidecar's client alone) - run: cargo clippy --release -p lean_vm --no-default-features -- -D warnings + run: cargo clippy --release -p lean_vm --all-targets --no-default-features -- -D warnings # Clippy type-checks every arm the ISA selects, so it also does `cargo check`'s work. # Nothing runs: GitHub's x86-64 runners do not reliably have AVX-512. diff --git a/bench/src/benchmark.rs b/bench/src/benchmark.rs index 4ad7b224bc..cf5af42242 100644 --- a/bench/src/benchmark.rs +++ b/bench/src/benchmark.rs @@ -232,8 +232,8 @@ pub fn run_recursion( ); } -/// Prove a recursion step's `n` leaves and write them to `out`: each leaf's claims, then its proof's -/// bytes after their length. +/// Prove a recursion step's `n` leaves and write them to `out`: each leaf's claims and its proof's bytes, each after +/// its length. fn write_leaves( n: usize, per_leaf: usize, diff --git a/bench/src/main.rs b/bench/src/main.rs index 6fd3a46e43..dd189b1788 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -81,7 +81,8 @@ struct Cli { )] repeat: usize, - /// Thread count (0: one per core). + /// Thread count (0: one per core). With the experimental prover, the performance cores' workers, beside which + /// it adds the efficiency cores'. #[arg(long, global = true, default_value_t = 0)] threads: usize, diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index c4b64dae72..d902124a82 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -20,7 +20,8 @@ target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar - The sidecar starts the prover's thread pool, compiles the aggregation program and warms up before it listens, so the socket appears once it is ready. It says so on standard error, then logs one line per `aggregate` request there. - The warm-up, two discarded proofs, makes the first proofs as fast as the next ones: without it they pay for touching the memory the prover then keeps. A proof larger than any before it still pays for its extra memory. - A sidecar holds a lock on the file beside its socket, the socket's path with `.lock` appended, while it runs. A second sidecar on the same socket fails at once. A socket left by a sidecar that stopped is replaced; a path that is some other file, or a socket another program listens on, is not. -- The socket takes the process's umask, and connecting needs write permission on it. Restrict it to the clients the sidecar serves: it trusts them with its resources. Any of them can keep it proving, and every connection may hold a request up to the max request in memory. +- Put the socket in a directory only the sidecar's user can write, such as `/run/leanvm`, not `/tmp`: there another user could take its path, or its lock's, first. +- The socket takes the process's umask, and connecting needs write permission on it. Restrict it to the clients the sidecar serves: it trusts them with its resources. Any of them can keep it proving, and every connection may hold about twice the max request in memory. - Requests are proven one at a time, across all connections, in the order they are received in full. A request whose client disconnects before its proof starts is dropped; one already proving finishes, and its proof is dropped. A client that gives up on a request should therefore close its connection, and send the request again, if at all, on a new one. - Stop the sidecar with a signal: it holds nothing to save. A request in flight is lost, and its client sees the connection close. The same goes if the prover panics, which exits the process: run the sidecar under a supervisor that restarts it. diff --git a/bins/leanvm-sidecar/src/main.rs b/bins/leanvm-sidecar/src/main.rs index 2ed0bf4f00..6b858be49a 100644 --- a/bins/leanvm-sidecar/src/main.rs +++ b/bins/leanvm-sidecar/src/main.rs @@ -38,7 +38,8 @@ struct Cli { )] max_request_mib: u64, - /// Thread count (0: one per core). + /// Thread count (0: one per core). With the experimental prover, the performance cores' workers, beside which + /// it adds the efficiency cores'. #[arg(long, default_value_t = 0)] threads: usize, diff --git a/crates/lean_vm/Cargo.toml b/crates/lean_vm/Cargo.toml index 0d91a2e259..b459580175 100644 --- a/crates/lean_vm/Cargo.toml +++ b/crates/lean_vm/Cargo.toml @@ -27,3 +27,7 @@ default = ["prover"] prover = ["dep:isa", "dep:aggregation", "dep:lean_da", "dep:stable_lean_vm_core"] # Prove with the experimental tree rather than the stable one. experimental_prover = ["prover", "dep:experimental_lean_vm_core"] + +[[test]] +name = "api" +required-features = ["prover"] diff --git a/crates/lean_vm/src/prover.rs b/crates/lean_vm/src/prover.rs index 0b6f0392e2..e11cee296c 100644 --- a/crates/lean_vm/src/prover.rs +++ b/crates/lean_vm/src/prover.rs @@ -19,7 +19,8 @@ pub mod lean_da { } /// Call this function before generating / verifying proofs: it starts the prover's thread pool, with `threads` threads -/// (0: one per core), and builds the aggregation program. +/// (0: one per core), and builds the aggregation program. With `experimental_prover`, `threads` counts the performance +/// cores' workers, beside which the pool adds the efficiency cores'. /// /// # Panics /// If the pool already runs with another thread count. From a435328d03f24e849df2ad1d09e6e725cdf248dd Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Tue, 17 Nov 2026 19:46:06 -0500 Subject: [PATCH 11/34] final review fixes: the child count refused before decoding, AGENTS.md names the sidecar, and cleanups - The server refuses more than MAX_RECURSIONS children right after the count, as `aggregate` would, so a request's decoded children never outgrow its bytes: a request of empty children otherwise held about 4x the max request. - The README's hello: the program identifies the build, and no longer promises a check the Rust client does not make. - AGENTS.md: the sidecar takes `--threads` and `--tracing` and installs jemalloc too. - Cleanups: the library's log lines lose their `leanvm-sidecar:` prefix, the client's limits are named constants, `server.rs` imports the claims from `claims`, experimental's `set_num_threads` counts the efficiency cores only when it sizes the pool, and the end-to-end test checks a refusal keeps the connection usable. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 6 ++--- bins/leanvm-sidecar/README.md | 2 +- bins/leanvm-sidecar/tests/sidecar.rs | 9 +++++--- crates/sidecar/src/client.rs | 12 +++++----- crates/sidecar/src/server.rs | 33 ++++++++++++++++------------ 5 files changed, 36 insertions(+), 26 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index aae43b3249..1ca2ae0dbf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,7 +82,7 @@ To confirm an arm is reached rather than silently skipped, put a `compile_error! Heavy benches are `benches/` targets, run with `cargo bench -p --bench `: `experimental_keccak_gkr`'s `hash_batch` and `experimental_pcs`'s `throughput` (and their `stable_*` copies), and `keccak256`'s `hash_throughput`. Slow checks are `#[ignore]`d tests, run by name with `-- --ignored --nocapture`: `print_whir_query_counts`, `print_whir_query_table`, `encoding_grinding_bits`. `aggregate_two_to_one` is the fast end-to-end check. -Where a proof's time goes is the `tracing` span tree, and only that: `--tracing` on the CLI, `BENCH_TRACING=1` on the benches, `RUST_LOG` for the level. A new stage worth timing gets an `info_span!` (`stage!` in `lean_vm_core`), never an `Instant` behind an env var. +Where a proof's time goes is the `tracing` span tree, and only that: `--tracing` on the benchmark CLI and the sidecar, `BENCH_TRACING=1` on the benches, `RUST_LOG` for the level. A new stage worth timing gets an `info_span!` (`stage!` in `lean_vm_core`), never an `Instant` behind an env var. ## Benchmarking @@ -99,7 +99,7 @@ CI compares every PR with its base, never with a history: `bench.yml` runs both ## The allocator -The prover's buffers are plain `Vec`s and the libraries impose no allocator. A proof allocates and frees gigabytes, so the CLI and the benches install jemalloc with `dirty_decay_ms:-1` (`bench_harness::Jemalloc`), which keeps freed pages mapped for the next proof at the cost of resident memory, and on Linux with `thp:always`, which backs its extents with transparent huge pages. Experimental skips a buffer's zero-fill with `primitives::uninit_vec` or `filled_vec` (`T: Plain`, every element written before it is read); stable never reads uninitialized memory, and writes a large buffer's first pass with the pool (`parallel::filled`, or `parallel::map_collect` when every element is computed anyway), since a zero-fill on one thread costs a whole-proof factor on many cores. Both drop large buffers where their last use ends. +The prover's buffers are plain `Vec`s and the libraries impose no allocator. A proof allocates and frees gigabytes, so the benchmark CLI, the sidecar and the benches install jemalloc with `dirty_decay_ms:-1` (`bench_harness::Jemalloc`), which keeps freed pages mapped for the next proof at the cost of resident memory, and on Linux with `thp:always`, which backs its extents with transparent huge pages. Experimental skips a buffer's zero-fill with `primitives::uninit_vec` or `filled_vec` (`T: Plain`, every element written before it is read); stable never reads uninitialized memory, and writes a large buffer's first pass with the pool (`parallel::filled`, or `parallel::map_collect` when every element is computed anyway), since a zero-fill on one thread costs a whole-proof factor on many cores. Both drop large buffers where their last use ends. ## The thread pool (`parallel`) @@ -107,7 +107,7 @@ No rayon. Every parallel site is "N independent items, each writing its own disj - Nested dispatch panics, because it would deadlock the dispatch lock. Each pool only sees its own nesting, so a task of either pool must not dispatch on either: inside a `P::map_collect` task, the shared crates call no dispatching helper of the stable tree. - One experimental site runs beside the pool: `keccak::while_executing` spawns half the pool's count of plain threads while a program executes, beside the pool's dispatches of a signature loop's segments and of their Keccak blocks, and gone when it ends. Stable builds the Keccak witness after execution, in the pool. -- The thread count is set in code, before a pool starts (`isa::Prover::set_num_threads`, which `lean_vm::setup` and the CLI's `--threads` call); `1` is strictly sequential. In experimental, workers fill the performance cores before the efficiency cores, and the default is one worker per logical CPU (on Linux, per physical core the process may use), holding back one performance worker when efficiency workers exist. In stable the default is `available_parallelism()`, halved where Linux reports SMT active. +- The thread count is set in code, before a pool starts (`isa::Prover::set_num_threads`, which `lean_vm::setup` and the benchmark CLI's and the sidecar's `--threads` call); `1` is strictly sequential. In experimental, workers fill the performance cores before the efficiency cores, and the default is one worker per logical CPU (on Linux, per physical core the process may use), holding back one performance worker when efficiency workers exist. In stable the default is `available_parallelism()`, halved where Linux reports SMT active. ## Three verifiers, one protocol diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index d902124a82..2a686cb33c 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -58,7 +58,7 @@ On each connection the sidecar first sends a hello frame. The client then sends | program | 32 bytes | the aggregation program's Fiat-Shamir IV, `Bytecode::fiat_shamir_iv` | | prover | bytes | `stable` or `experimental` | -Every version keeps the magic and the version first. A client checks them and the verifier tag, and disconnects if any differs from what it expects: a proof for another verifier does not verify. The program identifies the leanVM build: a client that also verifies with the library checks it too, since another build's proofs do not verify there. A client must not send a request body longer than the max request: the sidecar would close the connection without reading it, and the client could fail mid-write. +Every version keeps the magic and the version first. A client checks them and the verifier tag, and disconnects if any differs from what it expects: a proof for another verifier does not verify. The program identifies the leanVM build, whose proofs only that build's library verifies. A client must not send a request body longer than the max request: the sidecar would close the connection without reading it, and the client could fail mid-write. ### Requests diff --git a/bins/leanvm-sidecar/tests/sidecar.rs b/bins/leanvm-sidecar/tests/sidecar.rs index be22806a6b..4784d4d963 100644 --- a/bins/leanvm-sidecar/tests/sidecar.rs +++ b/bins/leanvm-sidecar/tests/sidecar.rs @@ -109,11 +109,14 @@ fn sidecar_end_to_end() { } } - // A client in another language sends an operation this version does not have. + // A client in another language sends an operation this version does not have, twice: a refusal keeps the + // connection usable. let mut raw = UnixStream::connect(socket).unwrap(); assert!(read_frame(&mut raw).starts_with(b"leanvmsc")); - raw.write_all(&[&1u64.to_le_bytes()[..], &[7]].concat()).unwrap(); - assert_eq!(read_frame(&mut raw)[0], Rejection::Malformed as u8); + for _ in 0..2 { + raw.write_all(&[&1u64.to_le_bytes()[..], &[7]].concat()).unwrap(); + assert_eq!(read_frame(&mut raw)[0], Rejection::Malformed as u8); + } // A second sidecar leaves a live one's socket alone; a restart replaces the stale socket a killed one leaves. let refused = command(socket).output().unwrap(); diff --git a/crates/sidecar/src/client.rs b/crates/sidecar/src/client.rs index 55e5210491..966e21da4c 100644 --- a/crates/sidecar/src/client.rs +++ b/crates/sidecar/src/client.rs @@ -10,6 +10,11 @@ use std::os::unix::net::UnixStream; use std::path::Path; use std::time::Duration; +/// Room for a hello or a verdict, far more than either takes. +const SHORT_ROOM: u64 = 1 << 16; +/// Room for a proof in a response, far more than one takes. +const PROOF_ROOM: u64 = 1 << 28; + /// A connection to a sidecar, for one request at a time. Proofs are their bytes, as the library's /// `AggregateProof::to_bytes` writes them and `from_bytes` reads them. /// @@ -29,7 +34,7 @@ impl Client { let mut stream = UnixStream::connect(socket)?; stream.set_read_timeout(timeout)?; stream.set_write_timeout(timeout)?; - let body = match wire::read_frame(&mut stream, 1 << 16)? { + let body = match wire::read_frame(&mut stream, SHORT_ROOM)? { Incoming::Body(body) => body, Incoming::End => return Err(SidecarError::Protocol("closed before its hello".into())), // Not a hello, as `hello_version` then says. @@ -98,7 +103,7 @@ impl Client { /// The library's `AggregateProof::verify`, run by the sidecar: whether `proof` proves `claims`. Bytes that do not /// decode prove nothing. pub fn verify(&mut self, claims: &AggregateClaims, proof: &[u8]) -> Result { - let body = self.call(&wire::encode_verify(claims, proof), 1 << 16)?; + let body = self.call(&wire::encode_verify(claims, proof), SHORT_ROOM)?; match wire::decode_verdict(&body) { Some(Ok(valid)) => Ok(valid), Some(Err((rejection, message))) => Err(SidecarError::Rejected(rejection, message)), @@ -128,6 +133,3 @@ impl Client { } } } - -/// Room for a proof in a response, far more than one takes. -const PROOF_ROOM: u64 = 1 << 28; diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs index fbfb625552..499d18ac83 100644 --- a/crates/sidecar/src/server.rs +++ b/crates/sidecar/src/server.rs @@ -6,7 +6,8 @@ use crate::wire::{ self, Cursor, Frame, Hello, Incoming, OK, OP_AGGREGATE, OP_VERIFY, Outcome, Proven, RawLeanSphincs, RawLeanXmss, }; use crate::{Blob, Rejection}; -use aggregation::{AggregateClaims, AggregateError, AggregateProof, OutputClaims}; +use aggregation::{AggregateError, AggregateProof}; +use claims::{AggregateClaims, OutputClaims}; use isa::Prover; use lean_sphincs::{LeanSphincsPublicKey, LeanSphincsSignature}; use lean_xmss::{LeanXmssPublicKey, LeanXmssSignature}; @@ -63,11 +64,7 @@ pub fn serve(config: &Config) -> io::Result { thread::Builder::new() .name("accept".into()) .spawn(move || accept(&listener, &jobs, hello.into(), max_request))?; - eprintln!( - "leanvm-sidecar: serving {} with the {} prover", - config.socket.display(), - P::NAME - ); + eprintln!("serving {} with the {} prover", config.socket.display(), P::NAME); prove_forever::

(&queue) } @@ -214,12 +211,12 @@ fn accept(listener: &UnixListener, jobs: &Sender, hello: Arc<[u8]>, max_req .name("connection".into()) .spawn(move || connection(stream, &jobs, &hello, max_request)); if let Err(e) = spawned { - eprintln!("leanvm-sidecar: a connection dropped: {e}"); + eprintln!("a connection dropped: {e}"); } } Err(e) => { // Out of file descriptors, say: wait for some to close rather than spin. - eprintln!("leanvm-sidecar: accept: {e}"); + eprintln!("accept: {e}"); thread::sleep(Duration::from_millis(100)); } } @@ -333,9 +330,13 @@ fn decode_aggregate( ) -> Result { let log_inv_rate = c.u8().ok_or_else(|| malformed("no rate"))?.into(); - // Each child is at least its two lengths. Nothing is reserved: `aggregate` bounds the count, not this. + // Each child is at least its two lengths. let n = count(c, 16).ok_or_else(|| malformed("children"))?; - let mut children = Vec::new(); + // `aggregate`'s bound, checked before decoding: a decoded child takes more memory than its bytes. + if n > aggregation::MAX_RECURSIONS { + return Err((Rejection::InvalidInput, "more than MAX_RECURSIONS children".into())); + } + let mut children = Vec::with_capacity(n); for i in 0..n { let claims = c.claims().ok_or_else(|| malformed(&format!("child {i}'s claims")))?; let proof = c.bytes().ok_or_else(|| malformed(&format!("child {i}'s proof")))?; @@ -345,9 +346,8 @@ fn decode_aggregate( let n = count(c, XMSS_RECORD).ok_or_else(|| malformed("leanXMSS signatures"))?; let raw_lean_xmss = (0..n) .map(|_| { - let pk = LeanXmssPublicKey::from_bytes(c.array().unwrap()); ( - pk, + LeanXmssPublicKey::from_bytes(c.array().unwrap()), *c.array().unwrap(), LeanXmssSignature::from_bytes(c.array().unwrap()), ) @@ -356,9 +356,8 @@ fn decode_aggregate( let n = count(c, SPHINCS_RECORD).ok_or_else(|| malformed("leanSPHINCS signatures"))?; let raw_lean_sphincs = (0..n) .map(|_| { - let pk = LeanSphincsPublicKey::from_bytes(c.array().unwrap()); ( - pk, + LeanSphincsPublicKey::from_bytes(c.array().unwrap()), *c.array().unwrap(), LeanSphincsSignature::from_bytes(c.array().unwrap()), ) @@ -492,6 +491,12 @@ mod tests { let mut huge = vec![OP_AGGREGATE, 1]; huge.extend(u64::MAX.to_le_bytes()); assert!(decode_request(&huge).is_err()); + let many = vec![(AggregateClaims::default(), Vec::new()); aggregation::MAX_RECURSIONS + 1]; + let frame = encode_request(&many, &[], &[], &[], OutputClaims::All, 1); + assert!(matches!( + decode_request(&body(frame)), + Err((Rejection::InvalidInput, _)) + )); } #[test] From 3fa4f3b7d1d4f5c78955939a2259ef9376151a26 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 04:51:38 -0500 Subject: [PATCH 12/34] thread count: the help texts lose the experimental caveat, now that both provers count every worker Co-Authored-By: Claude Opus 5.5 --- bench/src/main.rs | 3 +-- bins/leanvm-sidecar/src/main.rs | 3 +-- crates/lean_vm/src/prover.rs | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/bench/src/main.rs b/bench/src/main.rs index dd189b1788..6fd3a46e43 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -81,8 +81,7 @@ struct Cli { )] repeat: usize, - /// Thread count (0: one per core). With the experimental prover, the performance cores' workers, beside which - /// it adds the efficiency cores'. + /// Thread count (0: one per core). #[arg(long, global = true, default_value_t = 0)] threads: usize, diff --git a/bins/leanvm-sidecar/src/main.rs b/bins/leanvm-sidecar/src/main.rs index 6b858be49a..2ed0bf4f00 100644 --- a/bins/leanvm-sidecar/src/main.rs +++ b/bins/leanvm-sidecar/src/main.rs @@ -38,8 +38,7 @@ struct Cli { )] max_request_mib: u64, - /// Thread count (0: one per core). With the experimental prover, the performance cores' workers, beside which - /// it adds the efficiency cores'. + /// Thread count (0: one per core). #[arg(long, default_value_t = 0)] threads: usize, diff --git a/crates/lean_vm/src/prover.rs b/crates/lean_vm/src/prover.rs index e11cee296c..0b6f0392e2 100644 --- a/crates/lean_vm/src/prover.rs +++ b/crates/lean_vm/src/prover.rs @@ -19,8 +19,7 @@ pub mod lean_da { } /// Call this function before generating / verifying proofs: it starts the prover's thread pool, with `threads` threads -/// (0: one per core), and builds the aggregation program. With `experimental_prover`, `threads` counts the performance -/// cores' workers, beside which the pool adds the efficiency cores'. +/// (0: one per core), and builds the aggregation program. /// /// # Panics /// If the pool already runs with another thread count. From 64f05f4791c2ca80e8f00ec325f0bd90cbf47358 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 05:07:24 -0500 Subject: [PATCH 13/34] sidecar: the max request is a constant, 64 MiB, and --max-request-mib goes A request holds at most 32 blobs (4 MiB) and 16 children (a few MiB), and 64 MiB leaves room for about 25,000 leanXMSS or 8,000 leanSPHINCS signatures besides, so the limit needs no knob; at 64 MiB rather than 1 GiB, a connection holds at most about 128 MiB. The hello still announces it, so a later build can change it without a protocol version. Co-Authored-By: Claude Opus 5.5 --- bins/leanvm-sidecar/README.md | 3 +-- bins/leanvm-sidecar/src/main.rs | 9 --------- crates/sidecar/src/server.rs | 21 +++++++++++---------- 3 files changed, 12 insertions(+), 21 deletions(-) diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index 2a686cb33c..fd1240745a 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -13,7 +13,6 @@ target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar | --- | --- | --- | | `--prover` | required | `stable` or `experimental`: both give the same proofs, experimental faster | | `--socket` | required | the socket to listen on | -| `--max-request-mib` | 1024 | the longest request taken, in MiB | | `--threads` | 0 | the prover's thread count, 0 for one per core | | `--tracing` | off | print each proof's timing tree (`RUST_LOG` sets the level) | @@ -53,7 +52,7 @@ On each connection the sidecar first sends a hello frame. The client then sends | --- | --- | --- | | magic | 8 bytes | `leanvmsc` | | version | `u32` | 1 | -| max request | `u64` | the longest request body the sidecar takes | +| max request | `u64` | the longest request body the sidecar takes, 64 MiB | | verifier | bytes | the verifier's version tag, `leanvm-v1` | | program | 32 bytes | the aggregation program's Fiat-Shamir IV, `Bytecode::fiat_shamir_iv` | | prover | bytes | `stable` or `experimental` | diff --git a/bins/leanvm-sidecar/src/main.rs b/bins/leanvm-sidecar/src/main.rs index 2ed0bf4f00..bf8dd09959 100644 --- a/bins/leanvm-sidecar/src/main.rs +++ b/bins/leanvm-sidecar/src/main.rs @@ -30,14 +30,6 @@ struct Cli { #[arg(long)] socket: PathBuf, - /// The longest request taken, in MiB. - #[arg( - long, - default_value_t = 1024, - value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1 << 20) - )] - max_request_mib: u64, - /// Thread count (0: one per core). #[arg(long, default_value_t = 0)] threads: usize, @@ -54,7 +46,6 @@ fn main() -> ExitCode { } let config = sidecar::Config { socket: cli.socket, - max_request: cli.max_request_mib << 20, threads: cli.threads, }; let Err(e) = match cli.prover { diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs index 499d18ac83..d22a0b4f4e 100644 --- a/crates/sidecar/src/server.rs +++ b/crates/sidecar/src/server.rs @@ -28,12 +28,14 @@ use std::time::Duration; pub struct Config { /// The socket to listen on. A stale one, left by a sidecar that stopped, is replaced. pub socket: PathBuf, - /// The longest request body taken, in bytes. - pub max_request: u64, /// The prover's thread count (0: one per core). pub threads: usize, } +/// The longest request body taken, in bytes: room for every blob and child a request may have and thousands of +/// signatures, while bounding the memory a connection holds. +const MAX_REQUEST: u64 = 64 << 20; + /// A request, its client's connection, and where its outcome goes. type Job = (Request, UnixStream, Sender); @@ -54,16 +56,15 @@ pub fn serve(config: &Config) -> io::Result { warm_up::

(); let listener = UnixListener::bind(&config.socket)?; let hello = encode_hello(&Hello { - max_request: config.max_request, + max_request: MAX_REQUEST, tag: verifier::TAG.to_vec(), program, prover: P::NAME.into(), }); let (jobs, queue) = mpsc::channel(); - let max_request = config.max_request; thread::Builder::new() .name("accept".into()) - .spawn(move || accept(&listener, &jobs, hello.into(), max_request))?; + .spawn(move || accept(&listener, &jobs, hello.into()))?; eprintln!("serving {} with the {} prover", config.socket.display(), P::NAME); prove_forever::

(&queue) } @@ -202,14 +203,14 @@ fn aggregate(request: Request) -> Outcome { }) } -fn accept(listener: &UnixListener, jobs: &Sender, hello: Arc<[u8]>, max_request: u64) { +fn accept(listener: &UnixListener, jobs: &Sender, hello: Arc<[u8]>) { for stream in listener.incoming() { match stream { Ok(stream) => { let (jobs, hello) = (jobs.clone(), hello.clone()); let spawned = thread::Builder::new() .name("connection".into()) - .spawn(move || connection(stream, &jobs, &hello, max_request)); + .spawn(move || connection(stream, &jobs, &hello)); if let Err(e) = spawned { eprintln!("a connection dropped: {e}"); } @@ -224,16 +225,16 @@ fn accept(listener: &UnixListener, jobs: &Sender, hello: Arc<[u8]>, max_req } /// Serves one connection: its hello, then each request in turn, until the client closes it. -fn connection(mut stream: UnixStream, jobs: &Sender, hello: &[u8], max_request: u64) { +fn connection(mut stream: UnixStream, jobs: &Sender, hello: &[u8]) { if stream.write_all(hello).is_err() { return; } loop { - let body = match wire::read_frame(&mut stream, max_request) { + let body = match wire::read_frame(&mut stream, MAX_REQUEST) { Ok(Incoming::Body(body)) => body, // The body is left unread, so the stream is lost: say why, and close. Ok(Incoming::TooLarge(len)) => { - let message = format!("a request of {len} bytes, the sidecar takes at most {max_request}"); + let message = format!("a request of {len} bytes, the sidecar takes at most {MAX_REQUEST}"); eprintln!("a request refused (Malformed): {message}"); let _ = stream.write_all(&encode_response(&Err((Rejection::Malformed, message)))); return; From 79133ac9f94d98668ae5985c8d8953e985bfba0f Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 05:12:57 -0500 Subject: [PATCH 14/34] sidecar: no --tracing, which the benchmark CLI covers Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- bins/leanvm-sidecar/README.md | 1 - bins/leanvm-sidecar/src/main.rs | 7 ------- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 1ca2ae0dbf..bcb33e21e5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,7 +82,7 @@ To confirm an arm is reached rather than silently skipped, put a `compile_error! Heavy benches are `benches/` targets, run with `cargo bench -p --bench `: `experimental_keccak_gkr`'s `hash_batch` and `experimental_pcs`'s `throughput` (and their `stable_*` copies), and `keccak256`'s `hash_throughput`. Slow checks are `#[ignore]`d tests, run by name with `-- --ignored --nocapture`: `print_whir_query_counts`, `print_whir_query_table`, `encoding_grinding_bits`. `aggregate_two_to_one` is the fast end-to-end check. -Where a proof's time goes is the `tracing` span tree, and only that: `--tracing` on the benchmark CLI and the sidecar, `BENCH_TRACING=1` on the benches, `RUST_LOG` for the level. A new stage worth timing gets an `info_span!` (`stage!` in `lean_vm_core`), never an `Instant` behind an env var. +Where a proof's time goes is the `tracing` span tree, and only that: `--tracing` on the benchmark CLI, `BENCH_TRACING=1` on the benches, `RUST_LOG` for the level. A new stage worth timing gets an `info_span!` (`stage!` in `lean_vm_core`), never an `Instant` behind an env var. ## Benchmarking diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index fd1240745a..e2713d2dd7 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -14,7 +14,6 @@ target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar | `--prover` | required | `stable` or `experimental`: both give the same proofs, experimental faster | | `--socket` | required | the socket to listen on | | `--threads` | 0 | the prover's thread count, 0 for one per core | -| `--tracing` | off | print each proof's timing tree (`RUST_LOG` sets the level) | - The sidecar starts the prover's thread pool, compiles the aggregation program and warms up before it listens, so the socket appears once it is ready. It says so on standard error, then logs one line per `aggregate` request there. - The warm-up, two discarded proofs, makes the first proofs as fast as the next ones: without it they pay for touching the memory the prover then keeps. A proof larger than any before it still pays for its extra memory. diff --git a/bins/leanvm-sidecar/src/main.rs b/bins/leanvm-sidecar/src/main.rs index bf8dd09959..81ff209655 100644 --- a/bins/leanvm-sidecar/src/main.rs +++ b/bins/leanvm-sidecar/src/main.rs @@ -33,17 +33,10 @@ struct Cli { /// Thread count (0: one per core). #[arg(long, default_value_t = 0)] threads: usize, - - /// Print each proof's timing tree. Use RUST_LOG to adjust verbosity. - #[arg(long)] - tracing: bool, } fn main() -> ExitCode { let cli = Cli::parse(); - if cli.tracing { - bench_harness::init_tracing(); - } let config = sidecar::Config { socket: cli.socket, threads: cli.threads, From a6e83f301af2e882b3fa27a4e5fadd3ff6d9a311 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 05:14:37 -0500 Subject: [PATCH 15/34] allocator: jemalloc and its configuration in a crate of their own, so the sidecar no longer links bench_harness bench_harness re-exports it, so the benchmark CLI and the benches are unchanged. jemalloc still reads the configuration in both binaries (`_RJEM_MALLOC_CONF=stats_print:true` shows `opt.dirty_decay_ms: -1`), though nothing calls into the crate. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 9 +++++---- Cargo.lock | 11 +++++++++-- Cargo.toml | 2 ++ bins/leanvm-sidecar/Cargo.toml | 2 +- bins/leanvm-sidecar/src/main.rs | 2 +- crates/allocator/Cargo.toml | 11 +++++++++++ .../src/allocator.rs => allocator/src/lib.rs} | 2 +- crates/bench_harness/Cargo.toml | 2 +- crates/bench_harness/src/lib.rs | 1 - 9 files changed, 31 insertions(+), 11 deletions(-) create mode 100644 crates/allocator/Cargo.toml rename crates/{bench_harness/src/allocator.rs => allocator/src/lib.rs} (94%) diff --git a/AGENTS.md b/AGENTS.md index bcb33e21e5..ffd06bc1c4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,7 +34,8 @@ Both prove the same protocol and give the same proof bytes, pinned by `bench/tes | `zkdsl_compiler` | zkDSL → ISA; its tests run the stable prover | | `lean_xmss` | leanXMSS over Keccak-256 | | `lean_sphincs` | leanSPHINCS over Keccak-256 | -| `bench_harness` | benchmark harness (`Plan`, `Timing`, `--tracing`, Bencher output, the jemalloc allocator); never linked by a prover | +| `allocator` | the jemalloc global allocator and its configuration (below), for the benchmark CLI, the benches and the sidecar; never linked by a library | +| `bench_harness` | benchmark harness (`Plan`, `Timing`, `--tracing`, Bencher output; it re-exports `allocator::Jemalloc`); never linked by a prover | | `lean_da` | additive Reed-Solomon blob encoding (the stable tree's NTT), commitments, membership vectors | | `claims` | `AggregateClaims` and `OutputClaims`, and the claims' byte form: what both the library and the sidecar's client name | | `aggregation` | recursive signature and DA aggregation, generic over the prover: the guest's constants and witness, and the entry points | @@ -69,11 +70,11 @@ cargo fmt --all # max_width = 120 cargo machete --with-metadata # unused dependencies, dev ones included ``` -An x86-only `cfg` arm never compiles on an Apple machine, so type-check the other target before pushing anything `cfg`-gated (needs only `rustup target add x86_64-unknown-linux-gnu`; `bench_harness`, the benchmark CLI and the sidecar are left out because jemalloc's build script wants a cross `gcc`): +An x86-only `cfg` arm never compiles on an Apple machine, so type-check the other target before pushing anything `cfg`-gated (needs only `rustup target add x86_64-unknown-linux-gnu`; `allocator`, `bench_harness`, the benchmark CLI and the sidecar are left out because jemalloc's build script wants a cross `gcc`): ```bash RUSTFLAGS="-C target-cpu=znver4" \ - cargo check --release --workspace --exclude leanvm-bench --exclude leanvm-sidecar --exclude bench_harness --target x86_64-unknown-linux-gnu + cargo check --release --workspace --exclude leanvm-bench --exclude leanvm-sidecar --exclude bench_harness --exclude allocator --target x86_64-unknown-linux-gnu ``` `RUSTFLAGS` replaces `.cargo/config.toml`'s `target-cpu=native`; a target-specific `CARGO_TARGET_*_RUSTFLAGS` would be merged with it and hand the Apple CPU to the x86 target. @@ -99,7 +100,7 @@ CI compares every PR with its base, never with a history: `bench.yml` runs both ## The allocator -The prover's buffers are plain `Vec`s and the libraries impose no allocator. A proof allocates and frees gigabytes, so the benchmark CLI, the sidecar and the benches install jemalloc with `dirty_decay_ms:-1` (`bench_harness::Jemalloc`), which keeps freed pages mapped for the next proof at the cost of resident memory, and on Linux with `thp:always`, which backs its extents with transparent huge pages. Experimental skips a buffer's zero-fill with `primitives::uninit_vec` or `filled_vec` (`T: Plain`, every element written before it is read); stable never reads uninitialized memory, and writes a large buffer's first pass with the pool (`parallel::filled`, or `parallel::map_collect` when every element is computed anyway), since a zero-fill on one thread costs a whole-proof factor on many cores. Both drop large buffers where their last use ends. +The prover's buffers are plain `Vec`s and the libraries impose no allocator. A proof allocates and frees gigabytes, so the benchmark CLI, the sidecar and the benches install jemalloc with `dirty_decay_ms:-1` (`allocator::Jemalloc`), which keeps freed pages mapped for the next proof at the cost of resident memory, and on Linux with `thp:always`, which backs its extents with transparent huge pages. Experimental skips a buffer's zero-fill with `primitives::uninit_vec` or `filled_vec` (`T: Plain`, every element written before it is read); stable never reads uninitialized memory, and writes a large buffer's first pass with the pool (`parallel::filled`, or `parallel::map_collect` when every element is computed anyway), since a zero-fill on one thread costs a whole-proof factor on many cores. Both drop large buffers where their last use ends. ## The thread pool (`parallel`) diff --git a/Cargo.lock b/Cargo.lock index ae37fea9ad..b322f796fd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,6 +31,13 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator" +version = "0.1.0" +dependencies = [ + "tikv-jemallocator", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -94,9 +101,9 @@ dependencies = [ name = "bench_harness" version = "0.1.0" dependencies = [ + "allocator", "isa", "libc", - "tikv-jemallocator", "tracing-forest", "tracing-subscriber", ] @@ -422,7 +429,7 @@ name = "leanvm-sidecar" version = "0.1.0" dependencies = [ "aggregation", - "bench_harness", + "allocator", "clap", "experimental_lean_vm_core", "lean_vm", diff --git a/Cargo.toml b/Cargo.toml index 15c3fb53ff..3afa937e0d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "crates/verifier", + "crates/allocator", "crates/bench_harness", "crates/keccak256", "crates/fiat_shamir_prover", @@ -30,6 +31,7 @@ needless_range_loop = "allow" too_many_arguments = "allow" [workspace.dependencies] +allocator = { path = "crates/allocator" } bench_harness = { path = "crates/bench_harness" } verifier = { path = "crates/verifier" } keccak256 = { path = "crates/keccak256" } diff --git a/bins/leanvm-sidecar/Cargo.toml b/bins/leanvm-sidecar/Cargo.toml index 047a660451..94948c2246 100644 --- a/bins/leanvm-sidecar/Cargo.toml +++ b/bins/leanvm-sidecar/Cargo.toml @@ -17,7 +17,7 @@ workspace = true sidecar = { workspace = true, features = ["server"] } stable_lean_vm_core = { path = "../../crates/stable_prover/lean_vm_core" } experimental_lean_vm_core = { path = "../../crates/experimental_prover/lean_vm_core" } -bench_harness.workspace = true +allocator.workspace = true clap = { version = "4", features = ["derive"] } [dev-dependencies] diff --git a/bins/leanvm-sidecar/src/main.rs b/bins/leanvm-sidecar/src/main.rs index 81ff209655..5413607cc8 100644 --- a/bins/leanvm-sidecar/src/main.rs +++ b/bins/leanvm-sidecar/src/main.rs @@ -2,7 +2,7 @@ //! specifies the protocol. #[global_allocator] -static ALLOCATOR: bench_harness::Jemalloc = bench_harness::Jemalloc; +static ALLOCATOR: allocator::Jemalloc = allocator::Jemalloc; use clap::Parser; use experimental_lean_vm_core::ExperimentalProver; diff --git a/crates/allocator/Cargo.toml b/crates/allocator/Cargo.toml new file mode 100644 index 0000000000..d626e48872 --- /dev/null +++ b/crates/allocator/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "allocator" +version.workspace = true +edition.workspace = true +publish = false + +[lints] +workspace = true + +[dependencies] +tikv-jemallocator.workspace = true diff --git a/crates/bench_harness/src/allocator.rs b/crates/allocator/src/lib.rs similarity index 94% rename from crates/bench_harness/src/allocator.rs rename to crates/allocator/src/lib.rs index cf1d158f59..116f00d702 100644 --- a/crates/bench_harness/src/allocator.rs +++ b/crates/allocator/src/lib.rs @@ -1,4 +1,4 @@ -//! The global allocator of the CLI and the proving benchmarks. +//! The global allocator of the benchmark CLI, the proving benchmarks and the sidecar. //! //! A proof allocates gigabytes of short-lived buffers and frees them before it returns. //! diff --git a/crates/bench_harness/Cargo.toml b/crates/bench_harness/Cargo.toml index f94ff6b85e..a6eb66f0f0 100644 --- a/crates/bench_harness/Cargo.toml +++ b/crates/bench_harness/Cargo.toml @@ -8,8 +8,8 @@ publish = false workspace = true [dependencies] +allocator.workspace = true isa.workspace = true libc.workspace = true -tikv-jemallocator.workspace = true tracing-forest.workspace = true tracing-subscriber.workspace = true diff --git a/crates/bench_harness/src/lib.rs b/crates/bench_harness/src/lib.rs index aa5d11de01..fb4686a875 100644 --- a/crates/bench_harness/src/lib.rs +++ b/crates/bench_harness/src/lib.rs @@ -5,7 +5,6 @@ use std::io::{IsTerminal, Write}; use std::time::{Duration, Instant}; -mod allocator; mod trace; pub use allocator::Jemalloc; pub use trace::{TraceSuppressed, init_tracing, init_tracing_from_env, suppress_tracing}; From f24a7d05d8586796990e7d2217a23d509567fbc7 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 05:22:08 -0500 Subject: [PATCH 16/34] sidecar README --- bins/leanvm-sidecar/README.md | 78 +++++++++++++++-------------------- 1 file changed, 34 insertions(+), 44 deletions(-) diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index e2713d2dd7..a9b80dfba2 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -1,6 +1,6 @@ # leanvm-sidecar -A process that proves for clients on the same machine, over a Unix stream socket. It runs the library's one proving call, `aggregate`, so a client in any language gets the prover's full speed, with its own allocator, thread pool and retained memory, in a process of its own. It also verifies proofs, so a client builds no prover at all. +Generates and verifies leanVM proofs, in a dedicated process, communicating with your client (in any langage) over a Unix socket. ## Running @@ -11,21 +11,19 @@ target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar | flag | default | meaning | | --- | --- | --- | -| `--prover` | required | `stable` or `experimental`: both give the same proofs, experimental faster | +| `--prover` | required | `stable` or `experimental` (same proofs, experimental faster) | | `--socket` | required | the socket to listen on | -| `--threads` | 0 | the prover's thread count, 0 for one per core | +| `--threads` | 0 | thread count, 0 for one per core | -- The sidecar starts the prover's thread pool, compiles the aggregation program and warms up before it listens, so the socket appears once it is ready. It says so on standard error, then logs one line per `aggregate` request there. -- The warm-up, two discarded proofs, makes the first proofs as fast as the next ones: without it they pay for touching the memory the prover then keeps. A proof larger than any before it still pays for its extra memory. -- A sidecar holds a lock on the file beside its socket, the socket's path with `.lock` appended, while it runs. A second sidecar on the same socket fails at once. A socket left by a sidecar that stopped is replaced; a path that is some other file, or a socket another program listens on, is not. -- Put the socket in a directory only the sidecar's user can write, such as `/run/leanvm`, not `/tmp`: there another user could take its path, or its lock's, first. -- The socket takes the process's umask, and connecting needs write permission on it. Restrict it to the clients the sidecar serves: it trusts them with its resources. Any of them can keep it proving, and every connection may hold about twice the max request in memory. -- Requests are proven one at a time, across all connections, in the order they are received in full. A request whose client disconnects before its proof starts is dropped; one already proving finishes, and its proof is dropped. A client that gives up on a request should therefore close its connection, and send the request again, if at all, on a new one. -- Stop the sidecar with a signal: it holds nothing to save. A request in flight is lost, and its client sees the connection close. The same goes if the prover panics, which exits the process: run the sidecar under a supervisor that restarts it. +- It warms up (two discarded proofs) before listening, so the socket appears once it is ready. It logs to standard error. +- One sidecar per socket: it locks `.lock`, and replaces a stale socket left by one that stopped. +- Put the socket in a directory only the sidecar's user can write (not `/tmp`), and let only trusted clients connect: any of them can keep it proving, and a connection may hold about twice the max request in memory. +- Requests are proven one at a time, in the order they arrive. Closing the connection drops a request whose proof has not started. +- Stop it with a signal; requests in flight are lost. A prover panic exits the process, so run it under a supervisor. ## Clients -From Rust, `lean_vm::sidecar::Client` speaks the protocol, with the library's arguments but for the blobs, which go as they are, and proofs, which are their bytes: +From Rust, `lean_vm::sidecar::Client` takes the library's arguments, with blobs as they are and proofs as bytes: ```rust let mut sidecar = lean_vm::sidecar::Client::connect("/run/leanvm/sidecar.sock", None)?; @@ -33,17 +31,15 @@ let (claims, proof) = sidecar.aggregate(&children, &raw_lean_xmss, &[], &[], Out assert!(sidecar.verify(&claims, &proof)?); ``` -With `lean_vm = { default-features = false }`, a client builds no prover: it proves and verifies through the sidecar alone, so it cannot verify while the sidecar is down or restarting. With the default `prover` feature, the library also proves and verifies in the client's process (`AggregateProof::from_bytes`, then `verify`), which takes a sidecar of the same leanVM build. +With `lean_vm = { default-features = false }` the client builds no prover and verifies only through the sidecar. With the default `prover` feature it can also verify itself (`AggregateProof::from_bytes`, then `verify`), against a sidecar of the same leanVM build. `connect`'s timeout (`set_timeout` changes it) bounds each read and write; after one, drop the client. -`connect`'s timeout, which `Client::set_timeout` changes, bounds how long each read or write waits; after a timeout, drop the client. - -From another language, a client is the protocol below: connect, read the hello, then write a request and read its response, as many times as needed. +From another language, implement the protocol below. ## Protocol, version 1 -Every integer is little-endian. A **frame** is a `u64` length, then that many bytes of body. A **bytes** field is a `u64` length, then that many bytes. A **count** is a `u64`. +Integers are little-endian. A **frame** and a **bytes** field are a `u64` length, then that many bytes; a **count** is a `u64`. -On each connection the sidecar first sends a hello frame. The client then sends request frames, and after each one reads a response frame before sending the next. Shutting down only the writing half of the connection counts as closing it: the sidecar drops a request it has not started proving. +On connecting, the client reads a hello frame, then sends request frames, reading each one's response before sending the next. A connection whose writing half is shut down counts as closed. ### Hello @@ -51,32 +47,30 @@ On each connection the sidecar first sends a hello frame. The client then sends | --- | --- | --- | | magic | 8 bytes | `leanvmsc` | | version | `u32` | 1 | -| max request | `u64` | the longest request body the sidecar takes, 64 MiB | -| verifier | bytes | the verifier's version tag, `leanvm-v1` | -| program | 32 bytes | the aggregation program's Fiat-Shamir IV, `Bytecode::fiat_shamir_iv` | +| max request | `u64` | the longest request body taken, 64 MiB | +| verifier | bytes | the verifier's tag, `leanvm-v1` | +| program | 32 bytes | the aggregation program's Fiat-Shamir IV, which identifies the leanVM build | | prover | bytes | `stable` or `experimental` | -Every version keeps the magic and the version first. A client checks them and the verifier tag, and disconnects if any differs from what it expects: a proof for another verifier does not verify. The program identifies the leanVM build, whose proofs only that build's library verifies. A client must not send a request body longer than the max request: the sidecar would close the connection without reading it, and the client could fail mid-write. +The magic and the version come first in every version. A client checks them and the verifier tag, and sends no request body longer than the max request. ### Requests -A request's first byte is its operation: 1 for `aggregate`, 2 for `verify`. - -**Aggregate**, the library's `aggregate` arguments: +**Aggregate**, the library's `aggregate`: | field | type | meaning | | --- | --- | --- | | operation | `u8` | 1 | -| rate | `u8` | `log_inv_rate`, 1 to 4: higher gives a smaller proof, slower | +| rate | `u8` | `log_inv_rate`, 1 to 4 | | children | count, then per child: claims (bytes), proof (bytes) | at most 16 | | leanXMSS signatures | count, then per signature: pubkey (32 bytes), message (32 bytes), signature (2404 bytes) | | | leanSPHINCS signatures | count, then per signature: pubkey (32 bytes), message (32 bytes), signature (7740 bytes) | | -| blobs | count, then per blob: 8192 symbols of 16 bytes each | at most 32 | -| output | `u8`, then for 1 the claims (bytes) | 0: every covered claim; 1: exactly these claims, each covered by the inputs | +| blobs | count, then per blob: 8192 symbols, each a `u128` | at most 32 | +| output | `u8`, then for 1 the claims (bytes) | 0: every covered claim; 1: exactly these | -Keys, signatures and proofs are the libraries' byte forms (`to_bytes`), and a symbol is its 128 bits as a little-endian `u128`, as LeanDA hashes it. Signatures may come in any order, and a repeated claim, the same pubkey and message twice, counts once. As with the library, the sidecar does not verify children: an invalid one costs a failed proof, status 3 or 4, so a child from an untrusted source is verified first. +Keys, signatures and proofs are the libraries' `to_bytes`. Signatures may come in any order, and a repeated claim counts once. Children are not verified: an invalid one fails the proof (status 3 or 4), so verify untrusted children first. -**Verify**, whether a proof proves claims, as `AggregateProof::verify` says. It does not wait for the proofs being made. +**Verify**, `AggregateProof::verify`, answered without waiting for proofs in progress: | field | type | | --- | --- | @@ -84,36 +78,32 @@ Keys, signatures and proofs are the libraries' byte forms (`to_bytes`), and a sy | claims | bytes | | proof | bytes | -A proof whose bytes do not decode is invalid; claims that do not decode make the request malformed. - -**Claims**, as `AggregateClaims::to_bytes` writes them: +**Claims**, `AggregateClaims::to_bytes`: | field | type | | --- | --- | | leanXMSS claims | count, then per claim: pubkey (32 bytes), message (32 bytes) | | leanSPHINCS claims | count, then per claim: pubkey (32 bytes), message (32 bytes) | -| DA commitment | `u8`, 0 for none or 1, then for 1 the root (32 bytes) | +| DA commitment | `u8`: 0, or 1 then the root (32 bytes) | -A proof verifies only against claims whose lists are strictly sorted, comparing each claim's 64 bytes, pubkey then message, as a byte string. The claims a response carries are; the output claims of a request may come in any order. +A proof verifies only against claim lists strictly sorted by their 64 bytes; a response's claims are. Output claims may come in any order. ### Response | field | type | meaning | | --- | --- | --- | | status | `u8` | below | -| for 0, to `aggregate`: claims, proof | bytes, bytes | what the proof proves, and the proof (`AggregateProof::from_bytes` reads it) | -| for 0, to `verify`: valid | `u8` | 1 if the proof proves the claims, else 0 | -| otherwise: message | bytes | UTF-8, why there is no answer | +| for 0, to `aggregate` | claims (bytes), proof (bytes) | what the proof proves, and the proof | +| for 0, to `verify` | `u8` | 1 if the proof proves the claims, else 0 (also for a proof that does not decode) | +| otherwise | bytes | a UTF-8 message | | status | name | meaning | | --- | --- | --- | | 0 | ok | | -| 1 | invalid input | the request is inconsistent: a rate out of range, too many children or blobs, nothing to publish, an output claim the inputs do not cover | -| 2 | invalid signature | a signature is invalid | +| 1 | invalid input | a rate out of range, too many children or blobs, nothing to publish, an output claim not covered | +| 2 | invalid signature | | | 3 | invalid child | a child is invalid, or its proof does not decode | -| 4 | proving failed | an input the checks above missed is invalid, or the request is too large for one proof | -| 5 | malformed | the request does not decode, or is longer than the hello's max request | - -A request with several problems gets the status of the first one found. A `verify` request gets only 0 or 5. +| 4 | proving failed | an invalid input the checks missed, or too large for one proof | +| 5 | malformed | the request does not decode, or exceeds the max request | -After any response the connection stays usable, except after a request longer than the max request: the sidecar answers it with status 5 without reading it, then closes the connection. +A request with several problems gets the first one found; `verify` gets only 0 or 5. The connection stays usable after any response but to a request over the max request, which is refused unread and closes it. From e70f14dfc52c568e22066e9d3457d9d6b754798b Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 05:42:19 -0500 Subject: [PATCH 17/34] sidecar: at most 16 connections at once, so the memory requests hold is bounded Each connection holds at most one request, queued or proving, of about twice the max request, but nothing bounded the connections, so a client opening them in a loop could exhaust memory. The accept loop now takes one of 16 slots before accepting, and a connection's thread frees it when it ends, even by panicking: about 2 GiB at most. A connection beyond the cap waits for its hello rather than being refused, so the protocol is unchanged; checked live: the 17th waits, and gets its hello once one closes. The README says again how access is controlled: the socket takes the umask, and connecting needs write permission on it. Co-Authored-By: Claude Opus 5.5 --- bins/leanvm-sidecar/README.md | 3 +- crates/sidecar/src/server.rs | 56 ++++++++++++++++++++++++++++++----- 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index a9b80dfba2..a5b33286a2 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -17,7 +17,8 @@ target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar - It warms up (two discarded proofs) before listening, so the socket appears once it is ready. It logs to standard error. - One sidecar per socket: it locks `.lock`, and replaces a stale socket left by one that stopped. -- Put the socket in a directory only the sidecar's user can write (not `/tmp`), and let only trusted clients connect: any of them can keep it proving, and a connection may hold about twice the max request in memory. +- The socket takes the process's umask, and connecting needs write permission on it: let only trusted clients connect, since any of them can keep it proving. Put it in a directory only the sidecar's user can write (not `/tmp`). +- It serves 16 connections at once, each holding at most about twice the max request in memory; more wait to be accepted. - Requests are proven one at a time, in the order they arrive. Closing the connection drops a request whose proof has not started. - Stop it with a signal; requests in flight are lost. A prover panic exits the process, so run it under a supervisor. diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs index d22a0b4f4e..42b562f1b9 100644 --- a/crates/sidecar/src/server.rs +++ b/crates/sidecar/src/server.rs @@ -18,8 +18,8 @@ use std::os::fd::AsRawFd; use std::os::unix::fs::FileTypeExt; use std::os::unix::net::{UnixListener, UnixStream}; use std::path::{Path, PathBuf}; -use std::sync::Arc; use std::sync::mpsc::{self, Receiver, Sender}; +use std::sync::{Arc, Condvar, Mutex}; use std::thread; use std::time::Duration; @@ -36,6 +36,10 @@ pub struct Config { /// signatures, while bounding the memory a connection holds. const MAX_REQUEST: u64 = 64 << 20; +/// The most connections served at once; more wait to be accepted. With [`MAX_REQUEST`], it bounds the memory +/// requests hold, queued ones included: each connection has at most one. +const MAX_CONNECTIONS: usize = 16; + /// A request, its client's connection, and where its outcome goes. type Job = (Request, UnixStream, Sender); @@ -204,13 +208,16 @@ fn aggregate(request: Request) -> Outcome { } fn accept(listener: &UnixListener, jobs: &Sender, hello: Arc<[u8]>) { - for stream in listener.incoming() { - match stream { - Ok(stream) => { + let open = Arc::new((Mutex::new(0), Condvar::new())); + loop { + let slot = Slot::take(&open); + match listener.accept() { + Ok((stream, _)) => { let (jobs, hello) = (jobs.clone(), hello.clone()); - let spawned = thread::Builder::new() - .name("connection".into()) - .spawn(move || connection(stream, &jobs, &hello)); + let spawned = thread::Builder::new().name("connection".into()).spawn(move || { + let _slot = slot; + connection(stream, &jobs, &hello); + }); if let Err(e) = spawned { eprintln!("a connection dropped: {e}"); } @@ -224,6 +231,29 @@ fn accept(listener: &UnixListener, jobs: &Sender, hello: Arc<[u8]>) { } } +/// One of the [`MAX_CONNECTIONS`], freed when dropped, even by a connection's thread that panics. +struct Slot(Arc<(Mutex, Condvar)>); + +impl Slot { + /// Waits for a free slot and takes it. + fn take(open: &Arc<(Mutex, Condvar)>) -> Self { + let (count, freed) = &**open; + let mut count = freed + .wait_while(count.lock().unwrap(), |n| *n == MAX_CONNECTIONS) + .unwrap(); + *count += 1; + Self(open.clone()) + } +} + +impl Drop for Slot { + fn drop(&mut self) { + let (count, freed) = &*self.0; + *count.lock().unwrap() -= 1; + freed.notify_one(); + } +} + /// Serves one connection: its hello, then each request in turn, until the client closes it. fn connection(mut stream: UnixStream, jobs: &Sender, hello: &[u8]) { if stream.write_all(hello).is_err() { @@ -500,6 +530,18 @@ mod tests { )); } + #[test] + fn connections_beyond_the_cap_wait() { + let open = Arc::new((Mutex::new(0), Condvar::new())); + let mut slots: Vec<_> = (0..MAX_CONNECTIONS).map(|_| Slot::take(&open)).collect(); + let (taken, waiter) = mpsc::channel(); + let open2 = open.clone(); + thread::spawn(move || taken.send(Slot::take(&open2)).unwrap()); + assert!(waiter.recv_timeout(Duration::from_millis(100)).is_err()); + slots.pop(); + waiter.recv().unwrap(); + } + #[test] fn responses_round_trip() { let proven = (AggregateClaims::default(), vec![1, 2, 3]); From ee2dff3099e4097a573e9e4850bb93cbb3d7e248 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 06:03:48 -0500 Subject: [PATCH 18/34] sidecar README: the connection cap without its memory, and no supervisor advice Co-Authored-By: Claude Opus 5.5 --- bins/leanvm-sidecar/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index a5b33286a2..01709ae201 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -18,9 +18,9 @@ target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar - It warms up (two discarded proofs) before listening, so the socket appears once it is ready. It logs to standard error. - One sidecar per socket: it locks `.lock`, and replaces a stale socket left by one that stopped. - The socket takes the process's umask, and connecting needs write permission on it: let only trusted clients connect, since any of them can keep it proving. Put it in a directory only the sidecar's user can write (not `/tmp`). -- It serves 16 connections at once, each holding at most about twice the max request in memory; more wait to be accepted. +- It serves 16 connections at once; more wait to be accepted. - Requests are proven one at a time, in the order they arrive. Closing the connection drops a request whose proof has not started. -- Stop it with a signal; requests in flight are lost. A prover panic exits the process, so run it under a supervisor. +- Stop it with a signal; requests in flight are lost. ## Clients From 9ed3b2c51d7e8f07db9902cccc4f9d2167e8886e Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 06:10:32 -0500 Subject: [PATCH 19/34] lean_vm: two exclusive modes, `prover` (the default) or `sidecar` With `prover`, the library proves and verifies in the process and has no sidecar client; with `sidecar` (and `default-features = false`), it builds no prover and proves and verifies only through the sidecar. Both at once is a compile error, so forgetting `default-features = false` does not silently build the prover. The sidecar is a plain optional dependency, re-exported as `lean_vm::sidecar`; the end-to-end test uses the `sidecar` crate's client directly, and CI lints the sidecar mode. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/rust.yml | 4 ++-- AGENTS.md | 2 +- bins/leanvm-sidecar/README.md | 4 ++-- bins/leanvm-sidecar/tests/sidecar.rs | 4 ++-- crates/lean_vm/Cargo.toml | 9 +++------ crates/lean_vm/src/lib.rs | 21 ++++++++++----------- 6 files changed, 20 insertions(+), 24 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 008330d8eb..4d55e912d3 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -66,8 +66,8 @@ jobs: run: cargo clippyall - name: Clippy (the API on the experimental prover) run: cargo clippy --release -p lean_vm --all-targets --features experimental_prover -- -D warnings - - name: Clippy (the API without the prover, the sidecar's client alone) - run: cargo clippy --release -p lean_vm --all-targets --no-default-features -- -D warnings + - name: Clippy (the API in its sidecar mode) + run: cargo clippy --release -p lean_vm --all-targets --no-default-features --features sidecar -- -D warnings # Clippy type-checks every arm the ISA selects, so it also does `cargo check`'s work. # Nothing runs: GitHub's x86-64 runners do not reliably have AVX-512. diff --git a/AGENTS.md b/AGENTS.md index ffd06bc1c4..19f4f14c14 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,7 +40,7 @@ Both prove the same protocol and give the same proof bytes, pinned by `bench/tes | `claims` | `AggregateClaims` and `OutputClaims`, and the claims' byte form: what both the library and the sidecar's client name | | `aggregation` | recursive signature and DA aggregation, generic over the prover: the guest's constants and witness, and the entry points | | `sidecar` | proving and verifying in another process: the protocol of `bins/leanvm-sidecar` and its Rust client, which builds no prover; with the `server` feature, the server (generic over the prover) | -| `lean_vm` | the public API, not generic: it proves with the stable tree, or with the `experimental_prover` feature the experimental one; without its default `prover` feature, it builds no prover and offers the sidecar's client alone | +| `lean_vm` | the public API, not generic, in one of two modes: with its default `prover` feature, it proves and verifies in the process, with the stable tree, or with the `experimental_prover` feature the experimental one; with the `sidecar` feature instead, it builds no prover, and proves and verifies only through the sidecar's client | `aggregation` proves on whichever prover it is given, but derives the guest's constants and a child's witness from the stable tree, the reference: its bus blocks and their coordinates (`lean_vm_core::leaf`, whose term order the guest's arrays follow), the stacking, the grinds, the WHIR shapes and the NTT twiddles; both trees lay a proof out alike. diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index 01709ae201..24926d3173 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -24,7 +24,7 @@ target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar ## Clients -From Rust, `lean_vm::sidecar::Client` takes the library's arguments, with blobs as they are and proofs as bytes: +From Rust, `lean_vm = { default-features = false, features = ["sidecar"] }` builds no prover, and `lean_vm::sidecar::Client` proves and verifies, with the library's arguments, blobs as they are and proofs as bytes: ```rust let mut sidecar = lean_vm::sidecar::Client::connect("/run/leanvm/sidecar.sock", None)?; @@ -32,7 +32,7 @@ let (claims, proof) = sidecar.aggregate(&children, &raw_lean_xmss, &[], &[], Out assert!(sidecar.verify(&claims, &proof)?); ``` -With `lean_vm = { default-features = false }` the client builds no prover and verifies only through the sidecar. With the default `prover` feature it can also verify itself (`AggregateProof::from_bytes`, then `verify`), against a sidecar of the same leanVM build. `connect`'s timeout (`set_timeout` changes it) bounds each read and write; after one, drop the client. +`connect`'s timeout (`set_timeout` changes it) bounds each read and write; after one, drop the client. From another language, implement the protocol below. diff --git a/bins/leanvm-sidecar/tests/sidecar.rs b/bins/leanvm-sidecar/tests/sidecar.rs index 4784d4d963..9e3c293ba6 100644 --- a/bins/leanvm-sidecar/tests/sidecar.rs +++ b/bins/leanvm-sidecar/tests/sidecar.rs @@ -1,10 +1,10 @@ -//! The sidecar binary end to end, through the library's client: a leaf with every kind of input, a narrowed node over +//! The sidecar binary end to end, through the Rust client: a leaf with every kind of input, a narrowed node over //! it from a second connection, verifying, refusals that leave the connection usable, an unknown operation sent as //! another language's client would, and the socket's life across a second sidecar and a restart. use aggregation::signers_cache; -use lean_vm::sidecar::{Client, Rejection, SidecarError}; use lean_vm::{AggregateClaims, AggregateProof, OutputClaims, lean_da}; +use sidecar::{Client, Rejection, SidecarError}; use stable_lean_vm_core::StableProver; use std::io::{Read, Write}; use std::os::unix::net::UnixStream; diff --git a/crates/lean_vm/Cargo.toml b/crates/lean_vm/Cargo.toml index b459580175..2130ded747 100644 --- a/crates/lean_vm/Cargo.toml +++ b/crates/lean_vm/Cargo.toml @@ -16,17 +16,14 @@ aggregation = { workspace = true, optional = true } lean_da = { workspace = true, optional = true } stable_lean_vm_core = { path = "../stable_prover/lean_vm_core", optional = true } experimental_lean_vm_core = { path = "../experimental_prover/lean_vm_core", optional = true } - -# The sidecar speaks over a Unix socket. -[target.'cfg(unix)'.dependencies] -sidecar.workspace = true +sidecar = { workspace = true, optional = true } [features] default = ["prover"] -# Prove and verify here. Without it, through the sidecar only, and no prover is built. +# The two modes, one or the other (the crate docs). prover = ["dep:isa", "dep:aggregation", "dep:lean_da", "dep:stable_lean_vm_core"] -# Prove with the experimental tree rather than the stable one. experimental_prover = ["prover", "dep:experimental_lean_vm_core"] +sidecar = ["dep:sidecar"] [[test]] name = "api" diff --git a/crates/lean_vm/src/lib.rs b/crates/lean_vm/src/lib.rs index ca0963ab86..d146edeac2 100644 --- a/crates/lean_vm/src/lib.rs +++ b/crates/lean_vm/src/lib.rs @@ -3,13 +3,17 @@ //! - leanSPHINCS (stateless, for the execution layer) //! - leanDA (PQ blobs) //! -//! With the default `prover` feature, it proves and verifies here: proving runs the stable prover, or with the -//! `experimental_prover` feature the experimental one, and both give the same proofs, byte for byte. Without it, it -//! builds no prover, and proves and verifies through the sidecar, [`sidecar::Client`]: the same claims and -//! signatures, proofs as their bytes. +//! Two modes, one or the other: +//! - `prover`, the default: proves and verifies in this process, on the stable prover, or with `experimental_prover` +//! the experimental one (the same proofs, byte for byte). +//! - `sidecar`, with `default-features = false`: builds no prover, and proves and verifies through the sidecar +//! (`bins/leanvm-sidecar`) with `sidecar::Client`, proofs as their bytes. //! //! Example: [`crates/lean_vm/tests/api.rs`](https://github.com/leanEthereum/leanVM/blob/main/crates/lean_vm/tests/api.rs). +#[cfg(all(feature = "prover", feature = "sidecar"))] +compile_error!("`prover` and `sidecar` are two modes: pick one (`sidecar` with `default-features = false`)"); + pub use claims::{AggregateClaims, LeanSphincsClaim, LeanXmssClaim, OutputClaims}; pub use verifier::{MAX_LOG_INV_RATE, MIN_LOG_INV_RATE}; @@ -33,10 +37,5 @@ pub mod lean_sphincs { }; } -/// Proving and verifying in another process, the sidecar (`bins/leanvm-sidecar`), which keeps the prover's memory, -/// threads and warm state out of this one. A client built without the `prover` feature verifies only there, so not -/// while its sidecar is down. -#[cfg(unix)] -pub mod sidecar { - pub use ::sidecar::{Blob, Client, Rejection, SidecarError}; -} +#[cfg(feature = "sidecar")] +pub use ::sidecar; From eb678f5c58f248b0078fa5ed2aa41979407d5ff2 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 06:11:46 -0500 Subject: [PATCH 20/34] add todo --- crates/lean_vm/src/prover.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/lean_vm/src/prover.rs b/crates/lean_vm/src/prover.rs index 0b6f0392e2..bbf9d3d8f5 100644 --- a/crates/lean_vm/src/prover.rs +++ b/crates/lean_vm/src/prover.rs @@ -11,6 +11,7 @@ type P = experimental_lean_vm_core::ExperimentalProver; #[cfg(not(feature = "experimental_prover"))] type P = stable_lean_vm_core::StableProver; +// TODO: make leanDA a self contained dependency, so that it canmove to lib.rs pub mod lean_da { pub use ::lean_da::{ BLOB_SYMBOLS, Blob, BlobCountError, BlobMatrix, CELL_SYMBOLS, CELLS_PER_ROW, Codeword, DA_MAX_ROWS, From 787f09d39c3321c6a5d03ff97940c2ac0305506c Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 07:14:50 -0500 Subject: [PATCH 21/34] lean_vm: stable by default, `experimental_prover` or `sidecar`; the sidecar picks its prover at build time A user names the mode in one word: nothing for the stable prover in this process, `experimental_prover` for the experimental one, `sidecar` to prove and verify through the sidecar. `sidecar` with `experimental_prover` is a compile error that says where the choice belongs: the sidecar binary, which now builds the stable prover, or the experimental one with its own `experimental_prover` feature, instead of taking `--prover`, so it links one prover. In sidecar mode the library still compiles the stable prover (about 5 s on a clean build, 38 crates against 10) but hides and never runs it, so the workspace allows Cargo's unused-dependencies lint; CI's cargo machete still finds unused dependencies. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/rust.yml | 6 +++--- AGENTS.md | 4 ++-- Cargo.toml | 4 ++++ bins/leanvm-sidecar/Cargo.toml | 6 +++++- bins/leanvm-sidecar/README.md | 7 ++++--- bins/leanvm-sidecar/src/main.rs | 23 +++++------------------ bins/leanvm-sidecar/tests/sidecar.rs | 11 +++++++++-- crates/lean_vm/Cargo.toml | 19 +++++++------------ crates/lean_vm/src/lib.rs | 20 +++++++++++--------- crates/lean_vm/src/prover.rs | 2 +- crates/lean_vm/tests/api.rs | 2 ++ 11 files changed, 53 insertions(+), 51 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 4d55e912d3..66c8a0dd2e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -64,10 +64,10 @@ jobs: - uses: Swatinem/rust-cache@v2 - name: Clippy run: cargo clippyall - - name: Clippy (the API on the experimental prover) - run: cargo clippy --release -p lean_vm --all-targets --features experimental_prover -- -D warnings + - name: Clippy (the API and the sidecar on the experimental prover) + run: cargo clippy --release -p lean_vm -p leanvm-sidecar --all-targets --features experimental_prover -- -D warnings - name: Clippy (the API in its sidecar mode) - run: cargo clippy --release -p lean_vm --all-targets --no-default-features --features sidecar -- -D warnings + run: cargo clippy --release -p lean_vm --all-targets --features sidecar -- -D warnings # Clippy type-checks every arm the ISA selects, so it also does `cargo check`'s work. # Nothing runs: GitHub's x86-64 runners do not reliably have AVX-512. diff --git a/AGENTS.md b/AGENTS.md index 19f4f14c14..028b25ac72 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,7 +40,7 @@ Both prove the same protocol and give the same proof bytes, pinned by `bench/tes | `claims` | `AggregateClaims` and `OutputClaims`, and the claims' byte form: what both the library and the sidecar's client name | | `aggregation` | recursive signature and DA aggregation, generic over the prover: the guest's constants and witness, and the entry points | | `sidecar` | proving and verifying in another process: the protocol of `bins/leanvm-sidecar` and its Rust client, which builds no prover; with the `server` feature, the server (generic over the prover) | -| `lean_vm` | the public API, not generic, in one of two modes: with its default `prover` feature, it proves and verifies in the process, with the stable tree, or with the `experimental_prover` feature the experimental one; with the `sidecar` feature instead, it builds no prover, and proves and verifies only through the sidecar's client | +| `lean_vm` | the public API, not generic: it proves and verifies in the process, with the stable tree, or with the `experimental_prover` feature the experimental one; with the `sidecar` feature, it proves and verifies only through the sidecar's client, and runs no prover | `aggregation` proves on whichever prover it is given, but derives the guest's constants and a child's witness from the stable tree, the reference: its bus blocks and their coordinates (`lean_vm_core::leaf`, whose term order the guest's arrays follow), the stacking, the grinds, the WHIR shapes and the NTT twiddles; both trees lay a proof out alike. @@ -54,7 +54,7 @@ Each tree holds, in dependency order, leaves first: | `keccak_gkr` | the prover of the VM's `KECCAK` argument | | `lean_vm_core` | arithmetization: tables, bus, constraints, `cpu::execute` and `cpu::prove` of an `isa::Program` and its witness streams; `cpu::verify` wraps `verifier`; the tree's `isa::Prover` | -Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. The sidecar is `bins/leanvm-sidecar`, which serves `aggregate` over a Unix socket so that clients in any language get the prover in a process of its own; its `README.md` is the protocol's specification, so a change to the protocol in `crates/sidecar` (`wire.rs` and the server's half in `server.rs`), or to the claims' byte form (`AggregateClaims::to_bytes`), lands there. It also verifies, for clients without the library's prover. +Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. The sidecar is `bins/leanvm-sidecar`, which serves `aggregate` over a Unix socket so that clients in any language get the prover in a process of its own; its `README.md` is the protocol's specification, so a change to the protocol in `crates/sidecar` (`wire.rs` and the server's half in `server.rs`), or to the claims' byte form (`AggregateClaims::to_bytes`), lands there. It also verifies, for clients that run no prover. ## Building and testing diff --git a/Cargo.toml b/Cargo.toml index 3afa937e0d..37c7f5f870 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,10 @@ edition = "2024" needless_range_loop = "allow" too_many_arguments = "allow" +# With its `sidecar` feature, `lean_vm` still builds its prover, unused; CI's `cargo machete` finds unused dependencies. +[workspace.lints.cargo] +unused_dependencies = "allow" + [workspace.dependencies] allocator = { path = "crates/allocator" } bench_harness = { path = "crates/bench_harness" } diff --git a/bins/leanvm-sidecar/Cargo.toml b/bins/leanvm-sidecar/Cargo.toml index 94948c2246..b86d719a6a 100644 --- a/bins/leanvm-sidecar/Cargo.toml +++ b/bins/leanvm-sidecar/Cargo.toml @@ -16,10 +16,14 @@ workspace = true [dependencies] sidecar = { workspace = true, features = ["server"] } stable_lean_vm_core = { path = "../../crates/stable_prover/lean_vm_core" } -experimental_lean_vm_core = { path = "../../crates/experimental_prover/lean_vm_core" } +experimental_lean_vm_core = { path = "../../crates/experimental_prover/lean_vm_core", optional = true } allocator.workspace = true clap = { version = "4", features = ["derive"] } +[features] +# Run the experimental prover rather than the stable one. +experimental_prover = ["dep:experimental_lean_vm_core"] + [dev-dependencies] lean_vm = { path = "../../crates/lean_vm" } aggregation = { workspace = true, features = ["signers-cache"] } diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index 24926d3173..ce24489fa4 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -6,12 +6,13 @@ Generates and verifies leanVM proofs, in a dedicated process, communicating with ```bash cargo build --release -p leanvm-sidecar -target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar.sock +target/release/leanvm-sidecar --socket /run/leanvm/sidecar.sock ``` +It runs the stable prover, or, built with `--features experimental_prover`, the experimental one (the same proofs, faster). + | flag | default | meaning | | --- | --- | --- | -| `--prover` | required | `stable` or `experimental` (same proofs, experimental faster) | | `--socket` | required | the socket to listen on | | `--threads` | 0 | thread count, 0 for one per core | @@ -24,7 +25,7 @@ target/release/leanvm-sidecar --prover experimental --socket /run/leanvm/sidecar ## Clients -From Rust, `lean_vm = { default-features = false, features = ["sidecar"] }` builds no prover, and `lean_vm::sidecar::Client` proves and verifies, with the library's arguments, blobs as they are and proofs as bytes: +From Rust, `lean_vm = { features = ["sidecar"] }` runs no prover, and `lean_vm::sidecar::Client` proves and verifies, with the library's arguments, blobs as they are and proofs as bytes: ```rust let mut sidecar = lean_vm::sidecar::Client::connect("/run/leanvm/sidecar.sock", None)?; diff --git a/bins/leanvm-sidecar/src/main.rs b/bins/leanvm-sidecar/src/main.rs index 5413607cc8..20ab2d703c 100644 --- a/bins/leanvm-sidecar/src/main.rs +++ b/bins/leanvm-sidecar/src/main.rs @@ -5,27 +5,17 @@ static ALLOCATOR: allocator::Jemalloc = allocator::Jemalloc; use clap::Parser; -use experimental_lean_vm_core::ExperimentalProver; -use stable_lean_vm_core::StableProver; use std::path::PathBuf; use std::process::ExitCode; -/// Which of the two provers runs. -#[derive(Clone, Copy, clap::ValueEnum)] -enum Prover { - /// The stable prover. - Stable, - /// The experimental prover. - Experimental, -} +#[cfg(feature = "experimental_prover")] +type P = experimental_lean_vm_core::ExperimentalProver; +#[cfg(not(feature = "experimental_prover"))] +type P = stable_lean_vm_core::StableProver; #[derive(Parser)] #[command(name = "leanvm-sidecar")] struct Cli { - /// The prover to run. - #[arg(long, value_enum)] - prover: Prover, - /// The Unix socket to listen on. It appears once the sidecar is ready. #[arg(long)] socket: PathBuf, @@ -41,10 +31,7 @@ fn main() -> ExitCode { socket: cli.socket, threads: cli.threads, }; - let Err(e) = match cli.prover { - Prover::Stable => sidecar::serve::(&config), - Prover::Experimental => sidecar::serve::(&config), - }; + let Err(e) = sidecar::serve::

(&config); eprintln!("leanvm-sidecar: {}: {e}", config.socket.display()); ExitCode::FAILURE } diff --git a/bins/leanvm-sidecar/tests/sidecar.rs b/bins/leanvm-sidecar/tests/sidecar.rs index 9e3c293ba6..e5ba46b4e7 100644 --- a/bins/leanvm-sidecar/tests/sidecar.rs +++ b/bins/leanvm-sidecar/tests/sidecar.rs @@ -33,7 +33,7 @@ impl Drop for Socket { fn command(socket: &Path) -> Command { let mut command = Command::new(env!("CARGO_BIN_EXE_leanvm-sidecar")); - command.args(["--prover", "stable", "--socket"]).arg(socket); + command.arg("--socket").arg(socket); command } @@ -67,7 +67,14 @@ fn sidecar_end_to_end() { ))); let socket = &guard.0; let (first, mut client) = start(socket); - assert_eq!(client.prover(), "stable"); + assert_eq!( + client.prover(), + if cfg!(feature = "experimental_prover") { + "experimental" + } else { + "stable" + } + ); let xmss = signers_cache::get_signers::(3); let sphincs = signers_cache::get_lean_sphincs_signers::(1); diff --git a/crates/lean_vm/Cargo.toml b/crates/lean_vm/Cargo.toml index 2130ded747..f46cbe32a8 100644 --- a/crates/lean_vm/Cargo.toml +++ b/crates/lean_vm/Cargo.toml @@ -11,20 +11,15 @@ claims.workspace = true verifier.workspace = true lean_xmss.workspace = true lean_sphincs.workspace = true -isa = { workspace = true, optional = true } -aggregation = { workspace = true, optional = true } -lean_da = { workspace = true, optional = true } -stable_lean_vm_core = { path = "../stable_prover/lean_vm_core", optional = true } +isa.workspace = true +aggregation.workspace = true +lean_da.workspace = true +stable_lean_vm_core = { path = "../stable_prover/lean_vm_core" } experimental_lean_vm_core = { path = "../experimental_prover/lean_vm_core", optional = true } sidecar = { workspace = true, optional = true } [features] -default = ["prover"] -# The two modes, one or the other (the crate docs). -prover = ["dep:isa", "dep:aggregation", "dep:lean_da", "dep:stable_lean_vm_core"] -experimental_prover = ["prover", "dep:experimental_lean_vm_core"] +# Prove with the experimental prover rather than the stable one. +experimental_prover = ["dep:experimental_lean_vm_core"] +# Prove and verify through the sidecar rather than in this process. sidecar = ["dep:sidecar"] - -[[test]] -name = "api" -required-features = ["prover"] diff --git a/crates/lean_vm/src/lib.rs b/crates/lean_vm/src/lib.rs index d146edeac2..01489202ea 100644 --- a/crates/lean_vm/src/lib.rs +++ b/crates/lean_vm/src/lib.rs @@ -3,23 +3,25 @@ //! - leanSPHINCS (stateless, for the execution layer) //! - leanDA (PQ blobs) //! -//! Two modes, one or the other: -//! - `prover`, the default: proves and verifies in this process, on the stable prover, or with `experimental_prover` -//! the experimental one (the same proofs, byte for byte). -//! - `sidecar`, with `default-features = false`: builds no prover, and proves and verifies through the sidecar -//! (`bins/leanvm-sidecar`) with `sidecar::Client`, proofs as their bytes. +//! By default it proves and verifies in this process, on the stable prover, or with the `experimental_prover` feature +//! on the experimental one (the same proofs, byte for byte). With the `sidecar` feature, it proves and verifies only +//! through the sidecar (`bins/leanvm-sidecar`), with `sidecar::Client` and proofs as their bytes, and runs no prover +//! itself. //! //! Example: [`crates/lean_vm/tests/api.rs`](https://github.com/leanEthereum/leanVM/blob/main/crates/lean_vm/tests/api.rs). -#[cfg(all(feature = "prover", feature = "sidecar"))] -compile_error!("`prover` and `sidecar` are two modes: pick one (`sidecar` with `default-features = false`)"); +#[cfg(all(feature = "sidecar", feature = "experimental_prover"))] +compile_error!( + "`experimental_prover` picks the prover of this process, and with `sidecar` this process does not prove: build the \ + sidecar with `--features experimental_prover` instead" +); pub use claims::{AggregateClaims, LeanSphincsClaim, LeanXmssClaim, OutputClaims}; pub use verifier::{MAX_LOG_INV_RATE, MIN_LOG_INV_RATE}; -#[cfg(feature = "prover")] +#[cfg(not(feature = "sidecar"))] mod prover; -#[cfg(feature = "prover")] +#[cfg(not(feature = "sidecar"))] pub use prover::{AggregateError, AggregateProof, MAX_RECURSIONS, aggregate, lean_da, setup}; pub mod lean_xmss { diff --git a/crates/lean_vm/src/prover.rs b/crates/lean_vm/src/prover.rs index bbf9d3d8f5..478203091a 100644 --- a/crates/lean_vm/src/prover.rs +++ b/crates/lean_vm/src/prover.rs @@ -1,4 +1,4 @@ -//! What the `prover` feature brings: proving and verifying here, and LeanDA's encoding. +//! What the library offers without the `sidecar` feature: proving and verifying here, and LeanDA's encoding. use crate::{AggregateClaims, OutputClaims}; use ::lean_sphincs::{LeanSphincsPublicKey, LeanSphincsSignature}; diff --git a/crates/lean_vm/tests/api.rs b/crates/lean_vm/tests/api.rs index e4f06a6688..5a34f44e6c 100644 --- a/crates/lean_vm/tests/api.rs +++ b/crates/lean_vm/tests/api.rs @@ -1,3 +1,5 @@ +#![cfg(not(feature = "sidecar"))] + use lean_vm::*; const MSG_0: lean_xmss::Message = [0; lean_xmss::MESSAGE_LEN]; From b3d86945074d6ea2b7835f2fc83bd28c328b6b2e Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 07:40:04 -0500 Subject: [PATCH 22/34] sidecar: `serve` takes the socket and the thread count, no Config duplicating the CLI's Co-Authored-By: Claude Opus 5.5 --- bins/leanvm-sidecar/src/main.rs | 8 ++------ crates/sidecar/src/lib.rs | 2 +- crates/sidecar/src/server.rs | 34 +++++++++++++-------------------- 3 files changed, 16 insertions(+), 28 deletions(-) diff --git a/bins/leanvm-sidecar/src/main.rs b/bins/leanvm-sidecar/src/main.rs index 20ab2d703c..1385c3e7b7 100644 --- a/bins/leanvm-sidecar/src/main.rs +++ b/bins/leanvm-sidecar/src/main.rs @@ -27,11 +27,7 @@ struct Cli { fn main() -> ExitCode { let cli = Cli::parse(); - let config = sidecar::Config { - socket: cli.socket, - threads: cli.threads, - }; - let Err(e) = sidecar::serve::

(&config); - eprintln!("leanvm-sidecar: {}: {e}", config.socket.display()); + let Err(e) = sidecar::serve::

(&cli.socket, cli.threads); + eprintln!("leanvm-sidecar: {}: {e}", cli.socket.display()); ExitCode::FAILURE } diff --git a/crates/sidecar/src/lib.rs b/crates/sidecar/src/lib.rs index 67df334f52..4b5bcf5226 100644 --- a/crates/sidecar/src/lib.rs +++ b/crates/sidecar/src/lib.rs @@ -12,7 +12,7 @@ mod wire; pub use client::Client; #[cfg(feature = "server")] -pub use server::{Config, serve}; +pub use server::serve; /// One blob, as `lean_da` takes it: its symbols, each 128 bits. pub type Blob = [u128; 1 << verifier::da::LOG_BLOB_SYMBOLS]; diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs index 42b562f1b9..8630aa2676 100644 --- a/crates/sidecar/src/server.rs +++ b/crates/sidecar/src/server.rs @@ -17,21 +17,12 @@ use std::io::{self, Write}; use std::os::fd::AsRawFd; use std::os::unix::fs::FileTypeExt; use std::os::unix::net::{UnixListener, UnixStream}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::sync::mpsc::{self, Receiver, Sender}; use std::sync::{Arc, Condvar, Mutex}; use std::thread; use std::time::Duration; -/// How a sidecar serves. -#[derive(Clone, Debug)] -pub struct Config { - /// The socket to listen on. A stale one, left by a sidecar that stopped, is replaced. - pub socket: PathBuf, - /// The prover's thread count (0: one per core). - pub threads: usize, -} - /// The longest request body taken, in bytes: room for every blob and child a request may have and thousands of /// signatures, while bounding the memory a connection holds. const MAX_REQUEST: u64 = 64 << 20; @@ -43,22 +34,23 @@ const MAX_CONNECTIONS: usize = 16; /// A request, its client's connection, and where its outcome goes. type Job = (Request, UnixStream, Sender); -/// Starts prover `P`, builds the aggregation program and warms up, then listens on `config.socket` and serves until the -/// process ends, proving on the calling thread. The socket appears once the sidecar is ready to prove. Returns only -/// if it cannot listen, which a second sidecar on the same socket learns before it starts anything. +/// Starts prover `P` on `threads` threads (0: one per core), builds the aggregation program and warms up, then listens +/// on `socket`, replacing a stale one a sidecar that stopped left, and serves until the process ends, proving on the +/// calling thread. The socket appears once the sidecar is ready to prove. Returns only if it cannot listen, which a +/// second sidecar on the same socket learns before it starts anything. /// /// Call it from the main thread: a panic while proving, a bug, then ends the process. -pub fn serve(config: &Config) -> io::Result { - std::os::unix::net::SocketAddr::from_pathname(&config.socket)?; - let _lock = lock(&config.socket)?; - remove_stale(&config.socket)?; - if config.threads > 0 { - P::set_num_threads(config.threads); +pub fn serve(socket: &Path, threads: usize) -> io::Result { + std::os::unix::net::SocketAddr::from_pathname(socket)?; + let _lock = lock(socket)?; + remove_stale(socket)?; + if threads > 0 { + P::set_num_threads(threads); } P::init(); let program = aggregation::unified_guest().bytecode().fiat_shamir_iv(); warm_up::

(); - let listener = UnixListener::bind(&config.socket)?; + let listener = UnixListener::bind(socket)?; let hello = encode_hello(&Hello { max_request: MAX_REQUEST, tag: verifier::TAG.to_vec(), @@ -69,7 +61,7 @@ pub fn serve(config: &Config) -> io::Result { thread::Builder::new() .name("accept".into()) .spawn(move || accept(&listener, &jobs, hello.into()))?; - eprintln!("serving {} with the {} prover", config.socket.display(), P::NAME); + eprintln!("serving {} with the {} prover", socket.display(), P::NAME); prove_forever::

(&queue) } From 1ddcb3e5bb450039462e4990a99ef87f00164f00 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 08:58:28 -0500 Subject: [PATCH 23/34] sidecar client: `connect` takes only the socket, waiting as long as the sidecar takes No fixed timeout fits every request, and a sidecar that dies closes the connection, so a call fails at once rather than hanging; `set_timeout` stays for a client that wants a bound. Co-Authored-By: Claude Opus 5.5 --- bins/leanvm-sidecar/README.md | 4 ++-- bins/leanvm-sidecar/tests/sidecar.rs | 4 ++-- crates/sidecar/src/client.rs | 11 +++++------ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index ce24489fa4..2400e6b3fe 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -28,12 +28,12 @@ It runs the stable prover, or, built with `--features experimental_prover`, the From Rust, `lean_vm = { features = ["sidecar"] }` runs no prover, and `lean_vm::sidecar::Client` proves and verifies, with the library's arguments, blobs as they are and proofs as bytes: ```rust -let mut sidecar = lean_vm::sidecar::Client::connect("/run/leanvm/sidecar.sock", None)?; +let mut sidecar = lean_vm::sidecar::Client::connect("/run/leanvm/sidecar.sock")?; let (claims, proof) = sidecar.aggregate(&children, &raw_lean_xmss, &[], &[], OutputClaims::All, 2)?; assert!(sidecar.verify(&claims, &proof)?); ``` -`connect`'s timeout (`set_timeout` changes it) bounds each read and write; after one, drop the client. +A call waits as long as the sidecar takes, unless `set_timeout` bounds it; after a timeout, drop the client. From another language, implement the protocol below. diff --git a/bins/leanvm-sidecar/tests/sidecar.rs b/bins/leanvm-sidecar/tests/sidecar.rs index e5ba46b4e7..421cb9ba1d 100644 --- a/bins/leanvm-sidecar/tests/sidecar.rs +++ b/bins/leanvm-sidecar/tests/sidecar.rs @@ -42,7 +42,7 @@ fn start(socket: &Path) -> (Sidecar, Client) { let mut sidecar = Sidecar(command(socket).spawn().unwrap()); let start = Instant::now(); loop { - if let Ok(client) = Client::connect(socket, None) { + if let Ok(client) = Client::connect(socket) { return (sidecar, client); } assert!(sidecar.0.try_wait().unwrap().is_none(), "the sidecar exited"); @@ -91,7 +91,7 @@ fn sidecar_end_to_end() { // A second connection while the first stays open: the leaf and one more signature, keeping only one leanXMSS // claim of each. - let mut other = Client::connect(socket, None).unwrap(); + let mut other = Client::connect(socket).unwrap(); let mut kept: Vec<_> = [&xmss[0], &xmss[2]].iter().map(|(pk, m, _)| (*pk, *m)).collect(); kept.sort(); let only = AggregateClaims { diff --git a/crates/sidecar/src/client.rs b/crates/sidecar/src/client.rs index 966e21da4c..83e6ca8b35 100644 --- a/crates/sidecar/src/client.rs +++ b/crates/sidecar/src/client.rs @@ -29,11 +29,9 @@ pub struct Client { impl Client { /// Connects to the sidecar listening on `socket`, and checks it speaks this protocol version and proves for this - /// verifier. `timeout` is [`Self::set_timeout`]'s, already for reading the sidecar's hello. - pub fn connect(socket: impl AsRef, timeout: Option) -> Result { + /// verifier. + pub fn connect(socket: impl AsRef) -> Result { let mut stream = UnixStream::connect(socket)?; - stream.set_read_timeout(timeout)?; - stream.set_write_timeout(timeout)?; let body = match wire::read_frame(&mut stream, SHORT_ROOM)? { Incoming::Body(body) => body, Incoming::End => return Err(SidecarError::Protocol("closed before its hello".into())), @@ -65,8 +63,9 @@ impl Client { } /// How long a call may wait on the sidecar, reading or writing, before it fails with [`SidecarError::Io`]; `None`, - /// the default, waits as long as it takes. Dropping the client then closes the connection, so the sidecar drops - /// the request unless it has started proving it. + /// the default, waits as long as it takes (a sidecar that dies closes the connection, failing the call at once). + /// Dropping the client then closes the connection, so the sidecar drops the request unless it has started proving + /// it. pub fn set_timeout(&self, timeout: Option) -> std::io::Result<()> { self.stream.set_read_timeout(timeout)?; self.stream.set_write_timeout(timeout) From 55e5a0ad997cba50fe84d2e76c86cf63cef74ebd Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 09:23:06 -0500 Subject: [PATCH 24/34] sidecar test: verify only through the sidecar, as a client does The leaf and the node are checked with the client's `verify`, and one proof against other claims is refused, so a `verify` that accepts everything still fails the test. It imports `lean_vm` in its default mode: in its `sidecar` mode, as a dev-dependency, `cargo test` would build `lean_vm` so for the whole workspace and its `api` test would run nothing. Co-Authored-By: Claude Opus 5.5 --- bins/leanvm-sidecar/tests/sidecar.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/bins/leanvm-sidecar/tests/sidecar.rs b/bins/leanvm-sidecar/tests/sidecar.rs index 421cb9ba1d..1db647ec8c 100644 --- a/bins/leanvm-sidecar/tests/sidecar.rs +++ b/bins/leanvm-sidecar/tests/sidecar.rs @@ -3,7 +3,7 @@ //! another language's client would, and the socket's life across a second sidecar and a restart. use aggregation::signers_cache; -use lean_vm::{AggregateClaims, AggregateProof, OutputClaims, lean_da}; +use lean_vm::{AggregateClaims, OutputClaims, lean_da}; use sidecar::{Client, Rejection, SidecarError}; use stable_lean_vm_core::StableProver; use std::io::{Read, Write}; @@ -82,7 +82,7 @@ fn sidecar_end_to_end() { let leaf = client .aggregate(&[], &xmss[..2], &sphincs, &blobs, OutputClaims::All, 2) .unwrap(); - assert!(AggregateProof::from_bytes(&leaf.1).unwrap().verify(&leaf.0)); + assert!(client.verify(&leaf.0, &leaf.1).unwrap()); assert_eq!(leaf.0.num_signature_claims(), 3); assert_eq!( leaf.0.da_commitment, @@ -102,10 +102,8 @@ fn sidecar_end_to_end() { .aggregate(&[leaf], &xmss[2..], &[], &[], OutputClaims::Only(&only), 1) .unwrap(); assert_eq!(node.0, only); - assert!(AggregateProof::from_bytes(&node.1).unwrap().verify(&only)); - - // Verifying, from either connection, while nothing proves. - assert!(client.verify(&node.0, &node.1).unwrap()); + assert!(client.verify(&only, &node.1).unwrap()); + // Not against other claims. assert!(!other.verify(&AggregateClaims::default(), &node.1).unwrap()); // Refusals leave the connection in step: two in a row. From b72bbcb28ceeba1d21630ed4ea676e0baecde06a Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 10:06:00 -0500 Subject: [PATCH 25/34] doc --- AGENTS.md | 4 ++-- bins/leanvm-sidecar/README.md | 2 +- crates/lean_vm/src/lib.rs | 10 ++++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 028b25ac72..179272971a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,7 +40,7 @@ Both prove the same protocol and give the same proof bytes, pinned by `bench/tes | `claims` | `AggregateClaims` and `OutputClaims`, and the claims' byte form: what both the library and the sidecar's client name | | `aggregation` | recursive signature and DA aggregation, generic over the prover: the guest's constants and witness, and the entry points | | `sidecar` | proving and verifying in another process: the protocol of `bins/leanvm-sidecar` and its Rust client, which builds no prover; with the `server` feature, the server (generic over the prover) | -| `lean_vm` | the public API, not generic: it proves and verifies in the process, with the stable tree, or with the `experimental_prover` feature the experimental one; with the `sidecar` feature, it proves and verifies only through the sidecar's client, and runs no prover | +| `lean_vm` | the public API, not generic: it proves and verifies in the process, with the stable tree, or with the `experimental_prover` feature the experimental one; with the `sidecar` feature, it neither proves nor verifies itself, and does both through the sidecar's client | `aggregation` proves on whichever prover it is given, but derives the guest's constants and a child's witness from the stable tree, the reference: its bus blocks and their coordinates (`lean_vm_core::leaf`, whose term order the guest's arrays follow), the stacking, the grinds, the WHIR shapes and the NTT twiddles; both trees lay a proof out alike. @@ -54,7 +54,7 @@ Each tree holds, in dependency order, leaves first: | `keccak_gkr` | the prover of the VM's `KECCAK` argument | | `lean_vm_core` | arithmetization: tables, bus, constraints, `cpu::execute` and `cpu::prove` of an `isa::Program` and its witness streams; `cpu::verify` wraps `verifier`; the tree's `isa::Prover` | -Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. The sidecar is `bins/leanvm-sidecar`, which serves `aggregate` over a Unix socket so that clients in any language get the prover in a process of its own; its `README.md` is the protocol's specification, so a change to the protocol in `crates/sidecar` (`wire.rs` and the server's half in `server.rs`), or to the claims' byte form (`AggregateClaims::to_bytes`), lands there. It also verifies, for clients that run no prover. +Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. The sidecar is `bins/leanvm-sidecar`, which serves `aggregate` over a Unix socket so that clients in any language get the prover in a process of its own; its `README.md` is the protocol's specification, so a change to the protocol in `crates/sidecar` (`wire.rs` and the server's half in `server.rs`), or to the claims' byte form (`AggregateClaims::to_bytes`), lands there. It also verifies, for clients that do not verify themselves. ## Building and testing diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index 2400e6b3fe..971394d363 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -25,7 +25,7 @@ It runs the stable prover, or, built with `--features experimental_prover`, the ## Clients -From Rust, `lean_vm = { features = ["sidecar"] }` runs no prover, and `lean_vm::sidecar::Client` proves and verifies, with the library's arguments, blobs as they are and proofs as bytes: +From Rust, with `lean_vm = { features = ["sidecar"] }`, the client neither proves nor verifies itself: `lean_vm::sidecar::Client` does both, with the library's arguments, blobs as they are and proofs as bytes: ```rust let mut sidecar = lean_vm::sidecar::Client::connect("/run/leanvm/sidecar.sock")?; diff --git a/crates/lean_vm/src/lib.rs b/crates/lean_vm/src/lib.rs index 01489202ea..5f2a537d18 100644 --- a/crates/lean_vm/src/lib.rs +++ b/crates/lean_vm/src/lib.rs @@ -3,10 +3,12 @@ //! - leanSPHINCS (stateless, for the execution layer) //! - leanDA (PQ blobs) //! -//! By default it proves and verifies in this process, on the stable prover, or with the `experimental_prover` feature -//! on the experimental one (the same proofs, byte for byte). With the `sidecar` feature, it proves and verifies only -//! through the sidecar (`bins/leanvm-sidecar`), with `sidecar::Client` and proofs as their bytes, and runs no prover -//! itself. +//! Where proofs are made and checked: +//! - by default, in this process, on the stable prover; +//! - with the `experimental_prover` feature, in this process, on the experimental prover (the same proofs); +//! - with the `sidecar` feature, in a second process, the sidecar, which you launch with +//! `cargo run --release -p leanvm-sidecar -- --socket ` (`bins/leanvm-sidecar`): this process sends it proving +//! and verification requests through `sidecar::Client`. //! //! Example: [`crates/lean_vm/tests/api.rs`](https://github.com/leanEthereum/leanVM/blob/main/crates/lean_vm/tests/api.rs). From 43c8628569c31f3cd42356a2331c7235feee843b Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 10:20:55 -0500 Subject: [PATCH 26/34] lean_vm: prover.rs is in_process.rs, the counterpart of the sidecar mode Co-Authored-By: Claude Opus 5.5 --- crates/lean_vm/src/{prover.rs => in_process.rs} | 0 crates/lean_vm/src/lib.rs | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename crates/lean_vm/src/{prover.rs => in_process.rs} (100%) diff --git a/crates/lean_vm/src/prover.rs b/crates/lean_vm/src/in_process.rs similarity index 100% rename from crates/lean_vm/src/prover.rs rename to crates/lean_vm/src/in_process.rs diff --git a/crates/lean_vm/src/lib.rs b/crates/lean_vm/src/lib.rs index 5f2a537d18..3ecba3fb99 100644 --- a/crates/lean_vm/src/lib.rs +++ b/crates/lean_vm/src/lib.rs @@ -22,9 +22,9 @@ pub use claims::{AggregateClaims, LeanSphincsClaim, LeanXmssClaim, OutputClaims} pub use verifier::{MAX_LOG_INV_RATE, MIN_LOG_INV_RATE}; #[cfg(not(feature = "sidecar"))] -mod prover; +mod in_process; #[cfg(not(feature = "sidecar"))] -pub use prover::{AggregateError, AggregateProof, MAX_RECURSIONS, aggregate, lean_da, setup}; +pub use in_process::{AggregateError, AggregateProof, MAX_RECURSIONS, aggregate, lean_da, setup}; pub mod lean_xmss { pub use lean_xmss::{ From 00be740a26bcd3bde163144d7c0d663d07557f20 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 10:43:47 -0500 Subject: [PATCH 27/34] final review fixes: the experimental pool test tests again, no program in the hello, one claims spec - The experimental pool test fixed the pool's performance count, which is no longer its size wherever efficiency cores exist, so it panicked on its first line and never reached the one under test; it now uses `num_threads()`, as stable's does. - The hello's program IV was sent and decoded, but nothing checked or exposed it: dropped from protocol version 1, unreleased. - The claims' byte form is specified once, in the sidecar's README; `crates/claims` points there. - The warm-up's doc had the rate's direction backwards. Co-Authored-By: Claude Opus 5.5 --- bins/leanvm-sidecar/README.md | 1 - crates/claims/src/lib.rs | 5 ++--- crates/experimental_prover/parallel/src/topology.rs | 4 ++-- crates/sidecar/src/server.rs | 9 +++------ crates/sidecar/src/wire.rs | 3 --- 5 files changed, 7 insertions(+), 15 deletions(-) diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index 971394d363..ae287616e9 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -51,7 +51,6 @@ On connecting, the client reads a hello frame, then sends request frames, readin | version | `u32` | 1 | | max request | `u64` | the longest request body taken, 64 MiB | | verifier | bytes | the verifier's tag, `leanvm-v1` | -| program | 32 bytes | the aggregation program's Fiat-Shamir IV, which identifies the leanVM build | | prover | bytes | `stable` or `experimental` | The magic and the version come first in every version. A client checks them and the verifier tag, and sends no request body longer than the max request. diff --git a/crates/claims/src/lib.rs b/crates/claims/src/lib.rs index 033dbaa440..c1290c3e17 100644 --- a/crates/claims/src/lib.rs +++ b/crates/claims/src/lib.rs @@ -34,9 +34,8 @@ pub enum OutputClaims<'a> { Only(&'a AggregateClaims), } -// Wire format of claims: the leanXMSS claims (a u64 count, then each claim's pubkey and message), the leanSPHINCS -// claims alike, then the DA commitment (a byte, 0 for none or 1, then the 32-byte root). Little-endian, and -// canonical: decoding rejects any other flag and trailing bytes. Order is kept, sorted or not. +// The claims' byte form is specified in `bins/leanvm-sidecar/README.md` (Claims). Decoding is canonical: it rejects +// any other flag and trailing bytes. Order is kept, sorted or not. const _: () = assert!(lean_xmss::PUB_KEY_SIZE == 32 && lean_sphincs::PUB_KEY_SIZE == 32); const _: () = assert!(lean_xmss::MESSAGE_LEN == 32 && lean_sphincs::MESSAGE_LEN == 32); diff --git a/crates/experimental_prover/parallel/src/topology.rs b/crates/experimental_prover/parallel/src/topology.rs index 4d5b41d568..da3b4e9fa7 100644 --- a/crates/experimental_prover/parallel/src/topology.rs +++ b/crates/experimental_prover/parallel/src/topology.rs @@ -185,8 +185,8 @@ mod tests { #[test] #[should_panic(expected = "already runs")] fn a_fixed_shape_refuses_another() { - set_num_threads(topology().perf); - set_num_threads(topology().perf + 1); + set_num_threads(num_threads()); + set_num_threads(num_threads() + 1); } #[test] diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs index 8630aa2676..284a19f481 100644 --- a/crates/sidecar/src/server.rs +++ b/crates/sidecar/src/server.rs @@ -48,13 +48,11 @@ pub fn serve(socket: &Path, threads: usize) -> io::Result P::set_num_threads(threads); } P::init(); - let program = aggregation::unified_guest().bytecode().fiat_shamir_iv(); warm_up::

(); let listener = UnixListener::bind(socket)?; let hello = encode_hello(&Hello { max_request: MAX_REQUEST, tag: verifier::TAG.to_vec(), - program, prover: P::NAME.into(), }); let (jobs, queue) = mpsc::channel(); @@ -66,8 +64,9 @@ pub fn serve(socket: &Path, threads: usize) -> io::Result } /// A fresh process's first proofs are slower, touching memory the allocator then keeps: this pays for it before the -/// first request, with a discarded leaf and a full recursion over it, at rate 1/4, the one used in practice (a higher -/// rate still pays for its extra memory on its first proof). A blob is the one input that needs no key. +/// first request, with a discarded leaf and a full recursion over it, at rate 1/4, the one used in practice (a lower +/// rate, a larger `log_inv_rate`, still pays for its extra memory on its first proof). A blob is the one input that +/// needs no key. fn warm_up() { let blobs = lean_da::BlobMatrix::build(&[[0; lean_da::BLOB_SYMBOLS]]).expect("one blob"); let rate = 2; @@ -299,7 +298,6 @@ fn encode_hello(hello: &Hello) -> Vec { f.raw(&wire::VERSION.to_le_bytes()); f.u64(hello.max_request); f.bytes(&hello.tag); - f.raw(&hello.program); f.bytes(hello.prover.as_bytes()); f.finish() } @@ -452,7 +450,6 @@ mod tests { let hello = Hello { max_request: 1 << 30, tag: verifier::TAG.to_vec(), - program: [7; 32], prover: "stable".into(), }; let bytes = body(encode_hello(&hello)); diff --git a/crates/sidecar/src/wire.rs b/crates/sidecar/src/wire.rs index 8849fe5024..c7386feed7 100644 --- a/crates/sidecar/src/wire.rs +++ b/crates/sidecar/src/wire.rs @@ -140,8 +140,6 @@ impl<'a> Cursor<'a> { pub(crate) struct Hello { pub max_request: u64, pub tag: Vec, - /// The aggregation program's Fiat-Shamir IV, which pins the program and the verifier tag. - pub program: [u8; 32], pub prover: String, } @@ -158,7 +156,6 @@ pub(crate) fn decode_hello(body: &[u8]) -> Option { let hello = Hello { max_request: c.u64()?, tag: c.bytes()?.to_vec(), - program: *c.array()?, prover: c.string()?, }; c.is_empty().then_some(hello) From 5be8c39746a9e78f1341e00f1f659b75b88c029d Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 10:46:27 -0500 Subject: [PATCH 28/34] simplify --- AGENTS.md | 2 +- Cargo.lock | 6 ++++- bench/Cargo.toml | 1 + bench/src/main.rs | 6 ++--- crates/bench_harness/Cargo.toml | 1 - crates/bench_harness/src/lib.rs | 1 - .../experimental_prover/keccak_gkr/Cargo.toml | 3 ++- .../keccak_gkr/benches/hash_batch.rs | 2 +- .../parallel/src/topology.rs | 5 ++++- crates/experimental_prover/pcs/Cargo.toml | 1 + .../pcs/benches/throughput.rs | 2 +- crates/isa/src/prover.rs | 2 +- crates/lean_vm/src/in_process.rs | 4 +--- crates/sidecar/src/client.rs | 18 +++++++-------- crates/sidecar/src/server.rs | 6 ++--- crates/sidecar/src/wire.rs | 22 +++++++++---------- crates/stable_prover/keccak_gkr/Cargo.toml | 3 ++- .../keccak_gkr/benches/hash_batch.rs | 2 +- crates/stable_prover/parallel/src/lib.rs | 5 ++++- crates/stable_prover/pcs/Cargo.toml | 1 + .../stable_prover/pcs/benches/throughput.rs | 2 +- 21 files changed, 50 insertions(+), 45 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 179272971a..a4ea0bdb41 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -35,7 +35,7 @@ Both prove the same protocol and give the same proof bytes, pinned by `bench/tes | `lean_xmss` | leanXMSS over Keccak-256 | | `lean_sphincs` | leanSPHINCS over Keccak-256 | | `allocator` | the jemalloc global allocator and its configuration (below), for the benchmark CLI, the benches and the sidecar; never linked by a library | -| `bench_harness` | benchmark harness (`Plan`, `Timing`, `--tracing`, Bencher output; it re-exports `allocator::Jemalloc`); never linked by a prover | +| `bench_harness` | benchmark harness (`Plan`, `Timing`, `--tracing`, Bencher output); never linked by a prover | | `lean_da` | additive Reed-Solomon blob encoding (the stable tree's NTT), commitments, membership vectors | | `claims` | `AggregateClaims` and `OutputClaims`, and the claims' byte form: what both the library and the sidecar's client name | | `aggregation` | recursive signature and DA aggregation, generic over the prover: the guest's constants and witness, and the entry points | diff --git a/Cargo.lock b/Cargo.lock index b322f796fd..b549c86c91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -101,7 +101,6 @@ dependencies = [ name = "bench_harness" version = "0.1.0" dependencies = [ - "allocator", "isa", "libc", "tracing-forest", @@ -226,6 +225,7 @@ dependencies = [ name = "experimental_keccak_gkr" version = "0.1.0" dependencies = [ + "allocator", "bench_harness", "experimental_keccak_gkr", "experimental_parallel", @@ -262,6 +262,7 @@ dependencies = [ name = "experimental_pcs" version = "0.1.0" dependencies = [ + "allocator", "bench_harness", "experimental_parallel", "experimental_primitives", @@ -410,6 +411,7 @@ name = "leanvm-bench" version = "0.1.0" dependencies = [ "aggregation", + "allocator", "bench_harness", "clap", "experimental_lean_vm_core", @@ -619,6 +621,7 @@ checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" name = "stable_keccak_gkr" version = "0.1.0" dependencies = [ + "allocator", "bench_harness", "fiat_shamir_prover", "isa", @@ -653,6 +656,7 @@ version = "0.1.0" name = "stable_pcs" version = "0.1.0" dependencies = [ + "allocator", "bench_harness", "fiat_shamir_prover", "isa", diff --git a/bench/Cargo.toml b/bench/Cargo.toml index c323fbb0c4..fb2cf98efc 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -23,6 +23,7 @@ lean_xmss.workspace = true lean_sphincs.workspace = true stable_lean_vm_core = { path = "../crates/stable_prover/lean_vm_core" } experimental_lean_vm_core = { path = "../crates/experimental_prover/lean_vm_core" } +allocator.workspace = true bench_harness.workspace = true rand.workspace = true tracing.workspace = true diff --git a/bench/src/main.rs b/bench/src/main.rs index 6fd3a46e43..55b4c53f96 100644 --- a/bench/src/main.rs +++ b/bench/src/main.rs @@ -1,7 +1,7 @@ //! Benchmark CLI for signature and blob proofs, recursion, and the Fibonacci demo. #[global_allocator] -static ALLOCATOR: bench_harness::Jemalloc = bench_harness::Jemalloc; +static ALLOCATOR: allocator::Jemalloc = allocator::Jemalloc; use clap::{Parser, Subcommand}; use experimental_lean_vm_core::ExperimentalProver; @@ -13,9 +13,7 @@ mod report; /// Run the CLI's command on prover `P`. fn run(cli: Cli) { - if cli.threads > 0 { - P::set_num_threads(cli.threads); - } + P::set_num_threads(cli.threads); P::init(); let plan = bench_harness::Plan::new(cli.repeat, cli.cooldown); if cli.tracing && !matches!(&cli.command, Command::Recursion { .. }) { diff --git a/crates/bench_harness/Cargo.toml b/crates/bench_harness/Cargo.toml index a6eb66f0f0..dbf4d7fff0 100644 --- a/crates/bench_harness/Cargo.toml +++ b/crates/bench_harness/Cargo.toml @@ -8,7 +8,6 @@ publish = false workspace = true [dependencies] -allocator.workspace = true isa.workspace = true libc.workspace = true tracing-forest.workspace = true diff --git a/crates/bench_harness/src/lib.rs b/crates/bench_harness/src/lib.rs index fb4686a875..8eaef797a1 100644 --- a/crates/bench_harness/src/lib.rs +++ b/crates/bench_harness/src/lib.rs @@ -6,7 +6,6 @@ use std::io::{IsTerminal, Write}; use std::time::{Duration, Instant}; mod trace; -pub use allocator::Jemalloc; pub use trace::{TraceSuppressed, init_tracing, init_tracing_from_env, suppress_tracing}; /// One line of live progress on stderr. diff --git a/crates/experimental_prover/keccak_gkr/Cargo.toml b/crates/experimental_prover/keccak_gkr/Cargo.toml index cf4dd71f1a..2988871610 100644 --- a/crates/experimental_prover/keccak_gkr/Cargo.toml +++ b/crates/experimental_prover/keccak_gkr/Cargo.toml @@ -13,13 +13,14 @@ primitives = { package = "experimental_primitives", path = "../primitives" } fiat_shamir = { package = "fiat_shamir_prover", path = "../../fiat_shamir_prover" } pcs = { package = "experimental_pcs", path = "../pcs" } tracing.workspace = true +allocator = { workspace = true, optional = true } bench_harness = { workspace = true, optional = true } isa = { workspace = true, optional = true } verifier.workspace = true [features] # The standalone batch benchmark (`batch_bench`), for the `hash_batch` target. -bench = ["dep:bench_harness", "dep:isa", "primitives/test-util"] +bench = ["dep:allocator", "dep:bench_harness", "dep:isa", "primitives/test-util"] [dev-dependencies] keccak_gkr = { package = "experimental_keccak_gkr", path = ".", features = ["bench"] } diff --git a/crates/experimental_prover/keccak_gkr/benches/hash_batch.rs b/crates/experimental_prover/keccak_gkr/benches/hash_batch.rs index 3413850fd1..74f9ccb9cd 100644 --- a/crates/experimental_prover/keccak_gkr/benches/hash_batch.rs +++ b/crates/experimental_prover/keccak_gkr/benches/hash_batch.rs @@ -15,7 +15,7 @@ use bench_harness::{Plan, env_usize}; use pcs::whir::LOG_INV_RATE_0; #[global_allocator] -static ALLOCATOR: bench_harness::Jemalloc = bench_harness::Jemalloc; +static ALLOCATOR: allocator::Jemalloc = allocator::Jemalloc; fn main() { bench_harness::init_tracing_from_env(); diff --git a/crates/experimental_prover/parallel/src/topology.rs b/crates/experimental_prover/parallel/src/topology.rs index da3b4e9fa7..117b9a6fa6 100644 --- a/crates/experimental_prover/parallel/src/topology.rs +++ b/crates/experimental_prover/parallel/src/topology.rs @@ -29,8 +29,11 @@ pub(crate) fn topology() -> Topology { } /// Fixes the worker count before the pool starts: performance workers first, efficiency workers only beyond the -/// performance cores; `1` is sequential. Panics if it is already another. +/// performance cores; `1` is sequential and `0` keeps the default. Panics if it is already another. pub fn set_num_threads(n: usize) { + if n == 0 { + return; + } let set = TOPOLOGY .get_or_init(|| { let efficiency = n.saturating_sub(perf_cores()).min(efficiency_cores()); diff --git a/crates/experimental_prover/pcs/Cargo.toml b/crates/experimental_prover/pcs/Cargo.toml index 0b775a1f73..f92be5e734 100644 --- a/crates/experimental_prover/pcs/Cargo.toml +++ b/crates/experimental_prover/pcs/Cargo.toml @@ -16,6 +16,7 @@ verifier.workspace = true [dev-dependencies] primitives = { package = "experimental_primitives", path = "../primitives", features = ["test-util"] } +allocator.workspace = true bench_harness.workspace = true isa.workspace = true diff --git a/crates/experimental_prover/pcs/benches/throughput.rs b/crates/experimental_prover/pcs/benches/throughput.rs index d0f3c7d10f..5bba1a35b4 100644 --- a/crates/experimental_prover/pcs/benches/throughput.rs +++ b/crates/experimental_prover/pcs/benches/throughput.rs @@ -34,7 +34,7 @@ use primitives::{ }; #[global_allocator] -static ALLOCATOR: bench_harness::Jemalloc = bench_harness::Jemalloc; +static ALLOCATOR: allocator::Jemalloc = allocator::Jemalloc; fn main() { bench_harness::init_tracing_from_env(); diff --git a/crates/isa/src/prover.rs b/crates/isa/src/prover.rs index 175b9bbc0d..050c2c0884 100644 --- a/crates/isa/src/prover.rs +++ b/crates/isa/src/prover.rs @@ -12,7 +12,7 @@ pub trait Prover: 'static { /// Warm up the prover's thread pool, so that no timed work pays for starting it. Idempotent. fn init(); - /// Fixes the thread pool's size (at least 1) before it starts. Panics if it is already another. + /// Fixes the thread pool's size before it starts; `0` keeps the default. Panics if it is already another. fn set_num_threads(n: usize); /// Prove an execution of `program` on `public_input` at the code rate `2^-log_inv_rate`, or say why there is none. diff --git a/crates/lean_vm/src/in_process.rs b/crates/lean_vm/src/in_process.rs index 478203091a..10f3ef8c4e 100644 --- a/crates/lean_vm/src/in_process.rs +++ b/crates/lean_vm/src/in_process.rs @@ -25,9 +25,7 @@ pub mod lean_da { /// # Panics /// If the pool already runs with another thread count. pub fn setup(threads: usize) { - if threads > 0 { - P::set_num_threads(threads); - } + P::set_num_threads(threads); P::init(); aggregation::unified_guest(); } diff --git a/crates/sidecar/src/client.rs b/crates/sidecar/src/client.rs index 83e6ca8b35..50e09b807e 100644 --- a/crates/sidecar/src/client.rs +++ b/crates/sidecar/src/client.rs @@ -92,22 +92,14 @@ impl Client { let request = wire::encode_request(children, raw_lean_xmss, raw_lean_sphincs, blobs, output, log_inv_rate); // The claims a response carries are no longer than the request. let body = self.call(&request, request.len() as u64 + PROOF_ROOM)?; - match wire::decode_response(&body) { - Some(Ok(proven)) => Ok(proven), - Some(Err((rejection, message))) => Err(SidecarError::Rejected(rejection, message)), - None => Err(SidecarError::Protocol("a response that does not decode".into())), - } + answer(wire::decode_response(&body)) } /// The library's `AggregateProof::verify`, run by the sidecar: whether `proof` proves `claims`. Bytes that do not /// decode prove nothing. pub fn verify(&mut self, claims: &AggregateClaims, proof: &[u8]) -> Result { let body = self.call(&wire::encode_verify(claims, proof), SHORT_ROOM)?; - match wire::decode_verdict(&body) { - Some(Ok(valid)) => Ok(valid), - Some(Err((rejection, message))) => Err(SidecarError::Rejected(rejection, message)), - None => Err(SidecarError::Protocol("a response that does not decode".into())), - } + answer(wire::decode_verdict(&body)) } /// Sends a request frame and reads the response's body, at most `max_response` bytes. @@ -132,3 +124,9 @@ impl Client { } } } + +/// A decoded response as the client's result. +fn answer(decoded: Option>) -> Result { + let decoded = decoded.ok_or_else(|| SidecarError::Protocol("a response that does not decode".into()))?; + decoded.map_err(|(rejection, message)| SidecarError::Rejected(rejection, message)) +} diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs index 284a19f481..32c3312566 100644 --- a/crates/sidecar/src/server.rs +++ b/crates/sidecar/src/server.rs @@ -44,9 +44,7 @@ pub fn serve(socket: &Path, threads: usize) -> io::Result std::os::unix::net::SocketAddr::from_pathname(socket)?; let _lock = lock(socket)?; remove_stale(socket)?; - if threads > 0 { - P::set_num_threads(threads); - } + P::set_num_threads(threads); P::init(); warm_up::

(); let listener = UnixListener::bind(socket)?; @@ -290,7 +288,7 @@ fn connection(mut stream: UnixStream, jobs: &Sender, hello: &[u8]) { const XMSS_RECORD: usize = lean_xmss::PUB_KEY_SIZE + lean_xmss::MESSAGE_LEN + lean_xmss::SIG_SIZE; const SPHINCS_RECORD: usize = lean_sphincs::PUB_KEY_SIZE + lean_sphincs::MESSAGE_LEN + lean_sphincs::SIG_SIZE; -const BLOB_BYTES: usize = 16 * lean_da::BLOB_SYMBOLS; +const BLOB_BYTES: usize = size_of::(); fn encode_hello(hello: &Hello) -> Vec { let mut f = Frame::new(); diff --git a/crates/sidecar/src/wire.rs b/crates/sidecar/src/wire.rs index c7386feed7..d5c19ced42 100644 --- a/crates/sidecar/src/wire.rs +++ b/crates/sidecar/src/wire.rs @@ -216,23 +216,23 @@ pub(crate) fn encode_verify(claims: &AggregateClaims, proof: &[u8]) -> Vec { /// `None` if the body is not a verify request's response. pub(crate) fn decode_verdict(body: &[u8]) -> Option> { - let mut c = Cursor(body); - let result = match c.u8()? { - OK => Ok(match c.u8()? { - 0 => false, - 1 => true, - _ => return None, - }), - status => Err((Rejection::from_code(status)?, c.string()?)), - }; - c.is_empty().then_some(result) + decode(body, |c| match c.u8()? { + 0 => Some(false), + 1 => Some(true), + _ => None, + }) } /// `None` if the body is not an aggregate request's response. pub(crate) fn decode_response(body: &[u8]) -> Option { + decode(body, |c| Some((c.claims()?, c.bytes()?.to_vec()))) +} + +/// A response: `ok` reads what follows status 0, any other status is a rejection and its message. +fn decode(body: &[u8], ok: impl FnOnce(&mut Cursor) -> Option) -> Option> { let mut c = Cursor(body); let result = match c.u8()? { - OK => Ok((c.claims()?, c.bytes()?.to_vec())), + OK => Ok(ok(&mut c)?), status => Err((Rejection::from_code(status)?, c.string()?)), }; c.is_empty().then_some(result) diff --git a/crates/stable_prover/keccak_gkr/Cargo.toml b/crates/stable_prover/keccak_gkr/Cargo.toml index d000631966..5f4b232196 100644 --- a/crates/stable_prover/keccak_gkr/Cargo.toml +++ b/crates/stable_prover/keccak_gkr/Cargo.toml @@ -13,13 +13,14 @@ primitives = { package = "stable_primitives", path = "../primitives" } fiat_shamir = { package = "fiat_shamir_prover", path = "../../fiat_shamir_prover" } pcs = { package = "stable_pcs", path = "../pcs" } tracing.workspace = true +allocator = { workspace = true, optional = true } bench_harness = { workspace = true, optional = true } isa = { workspace = true, optional = true } verifier.workspace = true [features] # The standalone batch benchmark (`batch_bench`), for the `hash_batch` target. -bench = ["dep:bench_harness", "dep:isa", "primitives/test-util"] +bench = ["dep:allocator", "dep:bench_harness", "dep:isa", "primitives/test-util"] [dev-dependencies] keccak_gkr = { package = "stable_keccak_gkr", path = ".", features = ["bench"] } diff --git a/crates/stable_prover/keccak_gkr/benches/hash_batch.rs b/crates/stable_prover/keccak_gkr/benches/hash_batch.rs index 664379a72e..92d306ce2a 100644 --- a/crates/stable_prover/keccak_gkr/benches/hash_batch.rs +++ b/crates/stable_prover/keccak_gkr/benches/hash_batch.rs @@ -15,7 +15,7 @@ use bench_harness::{Plan, env_usize}; use verifier::MIN_LOG_INV_RATE; #[global_allocator] -static ALLOCATOR: bench_harness::Jemalloc = bench_harness::Jemalloc; +static ALLOCATOR: allocator::Jemalloc = allocator::Jemalloc; fn main() { bench_harness::init_tracing_from_env(); diff --git a/crates/stable_prover/parallel/src/lib.rs b/crates/stable_prover/parallel/src/lib.rs index 49d8da5323..a75593bfcd 100644 --- a/crates/stable_prover/parallel/src/lib.rs +++ b/crates/stable_prover/parallel/src/lib.rs @@ -24,8 +24,11 @@ pub fn num_threads() -> usize { }) } -/// Fixes [`num_threads`] (at least 1) before the pool starts. Panics if it is already another. +/// Fixes [`num_threads`] before the pool starts; `0` keeps the default. Panics if it is already another. pub fn set_num_threads(n: usize) { + if n == 0 { + return; + } let set = *NUM_THREADS.get_or_init(|| n); assert_eq!(set, n, "the thread pool already runs with {set} threads"); } diff --git a/crates/stable_prover/pcs/Cargo.toml b/crates/stable_prover/pcs/Cargo.toml index b55f61fff1..7249807175 100644 --- a/crates/stable_prover/pcs/Cargo.toml +++ b/crates/stable_prover/pcs/Cargo.toml @@ -16,6 +16,7 @@ verifier.workspace = true [dev-dependencies] primitives = { package = "stable_primitives", path = "../primitives", features = ["test-util"] } +allocator.workspace = true bench_harness.workspace = true isa.workspace = true diff --git a/crates/stable_prover/pcs/benches/throughput.rs b/crates/stable_prover/pcs/benches/throughput.rs index c4f463ffd3..8036f17a23 100644 --- a/crates/stable_prover/pcs/benches/throughput.rs +++ b/crates/stable_prover/pcs/benches/throughput.rs @@ -25,7 +25,7 @@ use primitives::{field::F64, test_rng::Rng}; use verifier::MIN_LOG_INV_RATE; #[global_allocator] -static ALLOCATOR: bench_harness::Jemalloc = bench_harness::Jemalloc; +static ALLOCATOR: allocator::Jemalloc = allocator::Jemalloc; fn main() { bench_harness::init_tracing_from_env(); From cdd01ee28d11ad73c49cde2ec6d8a3a1e2f174e3 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 10:59:42 -0500 Subject: [PATCH 29/34] guest-export: machete ignores `claims`, used by the aggregation sources it copies CI's `cargo machete --with-metadata` (0.9.2) failed on it; it passes now. Co-Authored-By: Claude Opus 5.5 --- formal/lean_vm/scripts/guest-export/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/formal/lean_vm/scripts/guest-export/Cargo.toml b/formal/lean_vm/scripts/guest-export/Cargo.toml index 33c7ff21a6..2e3c68ac3b 100644 --- a/formal/lean_vm/scripts/guest-export/Cargo.toml +++ b/formal/lean_vm/scripts/guest-export/Cargo.toml @@ -19,7 +19,7 @@ build = "build.rs" # The overlay's sources (`build.rs` copies `aggregation` into `OUT_DIR`) use these, which # cargo-machete, reading this crate's own sources, cannot see. [package.metadata.cargo-machete] -ignored = ["fiat_shamir_prover", "isa", "keccak256", "parallel", "pcs", "primitives", "tracing"] +ignored = ["claims", "fiat_shamir_prover", "isa", "keccak256", "parallel", "pcs", "primitives", "tracing"] [workspace] From 00892e1b069d83f8c9dbb37d1239a0e5c592ab81 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 11:16:53 -0500 Subject: [PATCH 30/34] sidecar: the lock file is opened without following a symlink or waiting on a FIFO A FIFO at `.lock` hung startup forever, silently, and a dangling symlink there made the sidecar create an empty file at its target; both now fail at once (ENXIO, ELOOP). Either needs a directory others can write, which the README rules out. Co-Authored-By: Claude Opus 5.5 --- crates/sidecar/src/server.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs index 32c3312566..a633b3108e 100644 --- a/crates/sidecar/src/server.rs +++ b/crates/sidecar/src/server.rs @@ -15,7 +15,7 @@ use std::convert::Infallible; use std::fs::{File, TryLockError}; use std::io::{self, Write}; use std::os::fd::AsRawFd; -use std::os::unix::fs::FileTypeExt; +use std::os::unix::fs::{FileTypeExt, OpenOptionsExt}; use std::os::unix::net::{UnixListener, UnixStream}; use std::path::Path; use std::sync::mpsc::{self, Receiver, Sender}; @@ -75,11 +75,17 @@ fn warm_up() { .expect("the warm-up's recursion proves"); } -/// Locks `.lock`, which the process holds until it ends, so that one sidecar at a time owns the socket. +/// Locks `.lock`, which the process holds until it ends, so that one sidecar at a time owns the socket. A +/// symlink or a FIFO there is refused rather than followed or waited on. fn lock(socket: &Path) -> io::Result { let mut path = socket.as_os_str().to_owned(); path.push(".lock"); - let file = File::options().create(true).truncate(false).write(true).open(&path)?; + let file = File::options() + .create(true) + .truncate(false) + .write(true) + .custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK) + .open(&path)?; match file.try_lock() { Ok(()) => Ok(file), Err(TryLockError::WouldBlock) => Err(io::Error::new( From afe13b0070d906b3baf5dad77fa9c364482f6a0e Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 11:17:29 -0500 Subject: [PATCH 31/34] sidecar client: one 16 MiB room for every response, beyond the request's length It replaces 64 KiB for a hello or a verdict and 256 MiB for a proof: proofs take a few hundred KiB, and the claims a response carries are no longer than the request. Co-Authored-By: Claude Opus 5.5 --- crates/sidecar/src/client.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/crates/sidecar/src/client.rs b/crates/sidecar/src/client.rs index 50e09b807e..9a76233667 100644 --- a/crates/sidecar/src/client.rs +++ b/crates/sidecar/src/client.rs @@ -10,10 +10,9 @@ use std::os::unix::net::UnixStream; use std::path::Path; use std::time::Duration; -/// Room for a hello or a verdict, far more than either takes. -const SHORT_ROOM: u64 = 1 << 16; -/// Room for a proof in a response, far more than one takes. -const PROOF_ROOM: u64 = 1 << 28; +/// The longest response read beyond the request's length, which bounds the claims it carries: room for a hello, a +/// verdict or a proof, far more than any takes. +const ROOM: u64 = 16 << 20; /// A connection to a sidecar, for one request at a time. Proofs are their bytes, as the library's /// `AggregateProof::to_bytes` writes them and `from_bytes` reads them. @@ -32,7 +31,7 @@ impl Client { /// verifier. pub fn connect(socket: impl AsRef) -> Result { let mut stream = UnixStream::connect(socket)?; - let body = match wire::read_frame(&mut stream, SHORT_ROOM)? { + let body = match wire::read_frame(&mut stream, ROOM)? { Incoming::Body(body) => body, Incoming::End => return Err(SidecarError::Protocol("closed before its hello".into())), // Not a hello, as `hello_version` then says. @@ -90,20 +89,19 @@ impl Client { log_inv_rate: usize, ) -> Result<(AggregateClaims, Vec), SidecarError> { let request = wire::encode_request(children, raw_lean_xmss, raw_lean_sphincs, blobs, output, log_inv_rate); - // The claims a response carries are no longer than the request. - let body = self.call(&request, request.len() as u64 + PROOF_ROOM)?; + let body = self.call(&request)?; answer(wire::decode_response(&body)) } /// The library's `AggregateProof::verify`, run by the sidecar: whether `proof` proves `claims`. Bytes that do not /// decode prove nothing. pub fn verify(&mut self, claims: &AggregateClaims, proof: &[u8]) -> Result { - let body = self.call(&wire::encode_verify(claims, proof), SHORT_ROOM)?; + let body = self.call(&wire::encode_verify(claims, proof))?; answer(wire::decode_verdict(&body)) } - /// Sends a request frame and reads the response's body, at most `max_response` bytes. - fn call(&mut self, request: &[u8], max_response: u64) -> Result, SidecarError> { + /// Sends a request frame and reads the response's body. + fn call(&mut self, request: &[u8]) -> Result, SidecarError> { let len = request.len() as u64 - 8; if len > self.max_request { return Err(SidecarError::Rejected( @@ -115,7 +113,7 @@ impl Client { )); } self.stream.write_all(request)?; - match wire::read_frame(&mut self.stream, max_response)? { + match wire::read_frame(&mut self.stream, request.len() as u64 + ROOM)? { Incoming::Body(body) => Ok(body), Incoming::End => Err(std::io::Error::from(std::io::ErrorKind::UnexpectedEof).into()), Incoming::TooLarge(len) => Err(SidecarError::Protocol(format!( From 0cb3ac8103f7dd8876c84ea60cf23b5a67c0b8ff Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 11:18:27 -0500 Subject: [PATCH 32/34] docs: AGENTS.md's x86 check and allocator row hold again, the sidecar's errors and `serve` say what they do - `bench_harness` no longer pulls in jemalloc, so the x86 check no longer leaves it out. - `keccak_gkr` (both trees) takes `allocator` as a dev-dependency, as `pcs` does: only its bench uses it, so no library links it, as AGENTS.md says. - `SidecarError::Incompatible` also covers a peer that is no sidecar, `Rejected` a request the client refuses itself, and `serve` returns on any error before serving. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 ++-- crates/experimental_prover/keccak_gkr/Cargo.toml | 4 ++-- crates/lean_vm/src/in_process.rs | 2 +- crates/sidecar/src/lib.rs | 5 +++-- crates/sidecar/src/server.rs | 4 ++-- crates/stable_prover/keccak_gkr/Cargo.toml | 4 ++-- 6 files changed, 12 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a4ea0bdb41..c207ec5aa0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -70,11 +70,11 @@ cargo fmt --all # max_width = 120 cargo machete --with-metadata # unused dependencies, dev ones included ``` -An x86-only `cfg` arm never compiles on an Apple machine, so type-check the other target before pushing anything `cfg`-gated (needs only `rustup target add x86_64-unknown-linux-gnu`; `allocator`, `bench_harness`, the benchmark CLI and the sidecar are left out because jemalloc's build script wants a cross `gcc`): +An x86-only `cfg` arm never compiles on an Apple machine, so type-check the other target before pushing anything `cfg`-gated (needs only `rustup target add x86_64-unknown-linux-gnu`; `allocator`, the benchmark CLI and the sidecar are left out because jemalloc's build script wants a cross `gcc`): ```bash RUSTFLAGS="-C target-cpu=znver4" \ - cargo check --release --workspace --exclude leanvm-bench --exclude leanvm-sidecar --exclude bench_harness --exclude allocator --target x86_64-unknown-linux-gnu + cargo check --release --workspace --exclude leanvm-bench --exclude leanvm-sidecar --exclude allocator --target x86_64-unknown-linux-gnu ``` `RUSTFLAGS` replaces `.cargo/config.toml`'s `target-cpu=native`; a target-specific `CARGO_TARGET_*_RUSTFLAGS` would be merged with it and hand the Apple CPU to the x86 target. diff --git a/crates/experimental_prover/keccak_gkr/Cargo.toml b/crates/experimental_prover/keccak_gkr/Cargo.toml index 2988871610..435f22f7c5 100644 --- a/crates/experimental_prover/keccak_gkr/Cargo.toml +++ b/crates/experimental_prover/keccak_gkr/Cargo.toml @@ -13,16 +13,16 @@ primitives = { package = "experimental_primitives", path = "../primitives" } fiat_shamir = { package = "fiat_shamir_prover", path = "../../fiat_shamir_prover" } pcs = { package = "experimental_pcs", path = "../pcs" } tracing.workspace = true -allocator = { workspace = true, optional = true } bench_harness = { workspace = true, optional = true } isa = { workspace = true, optional = true } verifier.workspace = true [features] # The standalone batch benchmark (`batch_bench`), for the `hash_batch` target. -bench = ["dep:allocator", "dep:bench_harness", "dep:isa", "primitives/test-util"] +bench = ["dep:bench_harness", "dep:isa", "primitives/test-util"] [dev-dependencies] +allocator.workspace = true keccak_gkr = { package = "experimental_keccak_gkr", path = ".", features = ["bench"] } [[bench]] diff --git a/crates/lean_vm/src/in_process.rs b/crates/lean_vm/src/in_process.rs index 10f3ef8c4e..6ee3dcf781 100644 --- a/crates/lean_vm/src/in_process.rs +++ b/crates/lean_vm/src/in_process.rs @@ -11,7 +11,7 @@ type P = experimental_lean_vm_core::ExperimentalProver; #[cfg(not(feature = "experimental_prover"))] type P = stable_lean_vm_core::StableProver; -// TODO: make leanDA a self contained dependency, so that it canmove to lib.rs +// TODO: make leanDA a self-contained dependency, so that it can move to lib.rs pub mod lean_da { pub use ::lean_da::{ BLOB_SYMBOLS, Blob, BlobCountError, BlobMatrix, CELL_SYMBOLS, CELLS_PER_ROW, Codeword, DA_MAX_ROWS, diff --git a/crates/sidecar/src/lib.rs b/crates/sidecar/src/lib.rs index 4b5bcf5226..6dba61d96b 100644 --- a/crates/sidecar/src/lib.rs +++ b/crates/sidecar/src/lib.rs @@ -52,11 +52,12 @@ impl Rejection { pub enum SidecarError { /// The socket failed, or the sidecar closed it: it stopped, or crashed. Connect again. Io(std::io::Error), - /// The sidecar speaks another protocol version, or proves for another verifier. + /// The peer is not a leanVM sidecar, speaks another protocol version, or proves for another verifier. Incompatible(String), /// The sidecar's bytes are not the protocol. Connect again. Protocol(String), - /// The sidecar made no proof, and says why. The connection stays usable. + /// The request was refused, by the sidecar or, for one longer than the sidecar takes, by the client, which says + /// why. The connection stays usable. Rejected(Rejection, String), } diff --git a/crates/sidecar/src/server.rs b/crates/sidecar/src/server.rs index a633b3108e..94780a7a94 100644 --- a/crates/sidecar/src/server.rs +++ b/crates/sidecar/src/server.rs @@ -36,8 +36,8 @@ type Job = (Request, UnixStream, Sender); /// Starts prover `P` on `threads` threads (0: one per core), builds the aggregation program and warms up, then listens /// on `socket`, replacing a stale one a sidecar that stopped left, and serves until the process ends, proving on the -/// calling thread. The socket appears once the sidecar is ready to prove. Returns only if it cannot listen, which a -/// second sidecar on the same socket learns before it starts anything. +/// calling thread. The socket appears once the sidecar is ready to prove. Returns only on an error before serving, +/// which a second sidecar on the same socket gets before it starts anything. /// /// Call it from the main thread: a panic while proving, a bug, then ends the process. pub fn serve(socket: &Path, threads: usize) -> io::Result { diff --git a/crates/stable_prover/keccak_gkr/Cargo.toml b/crates/stable_prover/keccak_gkr/Cargo.toml index 5f4b232196..d7c53062a0 100644 --- a/crates/stable_prover/keccak_gkr/Cargo.toml +++ b/crates/stable_prover/keccak_gkr/Cargo.toml @@ -13,16 +13,16 @@ primitives = { package = "stable_primitives", path = "../primitives" } fiat_shamir = { package = "fiat_shamir_prover", path = "../../fiat_shamir_prover" } pcs = { package = "stable_pcs", path = "../pcs" } tracing.workspace = true -allocator = { workspace = true, optional = true } bench_harness = { workspace = true, optional = true } isa = { workspace = true, optional = true } verifier.workspace = true [features] # The standalone batch benchmark (`batch_bench`), for the `hash_batch` target. -bench = ["dep:allocator", "dep:bench_harness", "dep:isa", "primitives/test-util"] +bench = ["dep:bench_harness", "dep:isa", "primitives/test-util"] [dev-dependencies] +allocator.workspace = true keccak_gkr = { package = "stable_keccak_gkr", path = ".", features = ["bench"] } [[bench]] From 0143ec2815a8fe1284cb9066afab9ae1dbf174af Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 11:20:38 -0500 Subject: [PATCH 33/34] lean_vm: MAX_RECURSIONS in every mode, so a sidecar client building a tree knows the fan-out Co-Authored-By: Claude Opus 5.5 --- crates/lean_vm/src/in_process.rs | 4 ++-- crates/lean_vm/src/lib.rs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/lean_vm/src/in_process.rs b/crates/lean_vm/src/in_process.rs index 6ee3dcf781..8cf14a2530 100644 --- a/crates/lean_vm/src/in_process.rs +++ b/crates/lean_vm/src/in_process.rs @@ -3,7 +3,7 @@ use crate::{AggregateClaims, OutputClaims}; use ::lean_sphincs::{LeanSphincsPublicKey, LeanSphincsSignature}; use ::lean_xmss::{LeanXmssPublicKey, LeanXmssSignature}; -pub use aggregation::{AggregateError, AggregateProof, MAX_RECURSIONS}; +pub use aggregation::{AggregateError, AggregateProof}; use isa::Prover; #[cfg(feature = "experimental_prover")] @@ -32,7 +32,7 @@ pub fn setup(threads: usize) { /// Aggregate leanXMSS and leanSPHINCS signatures, blobs and the claims of child proofs. /// -/// - `children`: at most [`MAX_RECURSIONS`], each with its claims. +/// - `children`: at most [`MAX_RECURSIONS`](crate::MAX_RECURSIONS), each with its claims. /// - `raw_lean_xmss`, `raw_lean_sphincs`: `(public_key, message, signature)`, in any order. /// - `blobs`: the encoded blobs, if any ([`lean_da::BlobMatrix::build`]). /// - `output`: which covered claims the proof attests. diff --git a/crates/lean_vm/src/lib.rs b/crates/lean_vm/src/lib.rs index 3ecba3fb99..51b3fa350e 100644 --- a/crates/lean_vm/src/lib.rs +++ b/crates/lean_vm/src/lib.rs @@ -18,13 +18,14 @@ compile_error!( sidecar with `--features experimental_prover` instead" ); +pub use aggregation::MAX_RECURSIONS; pub use claims::{AggregateClaims, LeanSphincsClaim, LeanXmssClaim, OutputClaims}; pub use verifier::{MAX_LOG_INV_RATE, MIN_LOG_INV_RATE}; #[cfg(not(feature = "sidecar"))] mod in_process; #[cfg(not(feature = "sidecar"))] -pub use in_process::{AggregateError, AggregateProof, MAX_RECURSIONS, aggregate, lean_da, setup}; +pub use in_process::{AggregateError, AggregateProof, aggregate, lean_da, setup}; pub mod lean_xmss { pub use lean_xmss::{ From 1822b560b1e6624eb5db8359bf132af0492d2cf2 Mon Sep 17 00:00:00 2001 From: Tom Wambsgans Date: Wed, 18 Nov 2026 11:58:01 -0500 Subject: [PATCH 34/34] sidecar: a Go client, tested in CI against the sidecar `bins/leanvm-sidecar/clients/go` (package `leanvmsidecar`, 152 lines): `Dial` checks the hello, then `Aggregate` and `Verify` take and return claims, proofs, signatures and blobs as the README's byte forms, so the client only frames requests and splits responses. Refusals are a `*Rejection` with the status and message; responses are capped. Its test proves a blob-only leaf and a node over it, verifies both ways and checks that a refusal leaves the connection usable; CI's Test job runs it, with gofmt and go vet, against the sidecar `cargo testall` builds. #768's Go client speaks another protocol (shared-memory rings), so this one is new. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/rust.yml | 13 ++ AGENTS.md | 2 +- bins/leanvm-sidecar/README.md | 2 +- bins/leanvm-sidecar/clients/go/client.go | 152 ++++++++++++++++++ bins/leanvm-sidecar/clients/go/client_test.go | 55 +++++++ bins/leanvm-sidecar/clients/go/go.mod | 3 + 6 files changed, 225 insertions(+), 2 deletions(-) create mode 100644 bins/leanvm-sidecar/clients/go/client.go create mode 100644 bins/leanvm-sidecar/clients/go/client_test.go create mode 100644 bins/leanvm-sidecar/clients/go/go.mod diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 66c8a0dd2e..f63d8113be 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -39,6 +39,19 @@ jobs: run: cargo test --release -p keccak256 env: RUSTFLAGS: -C target-cpu=x86-64 + - uses: actions/setup-go@v5 + with: + go-version-file: bins/leanvm-sidecar/clients/go/go.mod + # Against the sidecar `cargo testall` built. + - name: Test (the sidecar's Go client) + working-directory: bins/leanvm-sidecar/clients/go + run: | + test -z "$(gofmt -l .)" + go vet ./... + socket=$RUNNER_TEMP/sidecar.sock + ../../../../target/release/leanvm-sidecar --socket "$socket" & + until [ -S "$socket" ]; do kill -0 $! && sleep 1; done + LEANVM_SIDECAR_SOCKET=$socket go test -count=1 ./... # The NEON kernels, as bench.yml's arm64 testbed builds them: the x86 job's haswell reaches none of them. test-arm64: diff --git a/AGENTS.md b/AGENTS.md index c207ec5aa0..8f209136c0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,7 +54,7 @@ Each tree holds, in dependency order, leaves first: | `keccak_gkr` | the prover of the VM's `KECCAK` argument | | `lean_vm_core` | arithmetization: tables, bus, constraints, `cpu::execute` and `cpu::prove` of an `isa::Program` and its witness streams; `cpu::verify` wraps `verifier`; the tree's `isa::Prover` | -Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. The sidecar is `bins/leanvm-sidecar`, which serves `aggregate` over a Unix socket so that clients in any language get the prover in a process of its own; its `README.md` is the protocol's specification, so a change to the protocol in `crates/sidecar` (`wire.rs` and the server's half in `server.rs`), or to the claims' byte form (`AggregateClaims::to_bytes`), lands there. It also verifies, for clients that do not verify themselves. +Paths below such as `crates/lean_vm_core` mean the crate in either tree. `crates/lean_vm/src/lib.rs` is the only thing a user imports: every other crate is `publish = false`, so a new user-facing item is a re-export there, and `crates/lean_vm/tests/api.rs` is its end-to-end use. A re-exported type brings its `pub` methods along, so one the workspace needs but users should not see is `#[doc(hidden)] pub fn __name`, the double underscore marking it internal at every call site. The guest is zkDSL, `guests/lean_ethereum.py`. The benchmark CLI is `bench/` (`leanvm-bench`), run with `cargo leanvm-bench --prover stable|experimental ` (`aggregate`, `recursion`, `fibonacci`; the prover is required, with no default), the benchmarks generic over the prover. The sidecar is `bins/leanvm-sidecar`, which serves `aggregate` over a Unix socket so that clients in any language get the prover in a process of its own; its `README.md` is the protocol's specification, so a change to the protocol in `crates/sidecar` (`wire.rs` and the server's half in `server.rs`), or to the claims' byte form (`AggregateClaims::to_bytes`), lands there and in its Go client (`clients/go`), which CI tests against the sidecar. It also verifies, for clients that do not verify themselves. ## Building and testing diff --git a/bins/leanvm-sidecar/README.md b/bins/leanvm-sidecar/README.md index ae287616e9..cafd916d43 100644 --- a/bins/leanvm-sidecar/README.md +++ b/bins/leanvm-sidecar/README.md @@ -35,7 +35,7 @@ assert!(sidecar.verify(&claims, &proof)?); A call waits as long as the sidecar takes, unless `set_timeout` bounds it; after a timeout, drop the client. -From another language, implement the protocol below. +From Go, `clients/go` (package `leanvmsidecar`) offers the same, `Dial`, then `Aggregate` and `Verify`, with claims, proofs, signatures and blobs in the byte forms below. From another language, implement the protocol below. ## Protocol, version 1 diff --git a/bins/leanvm-sidecar/clients/go/client.go b/bins/leanvm-sidecar/clients/go/client.go new file mode 100644 index 0000000000..637799e24e --- /dev/null +++ b/bins/leanvm-sidecar/clients/go/client.go @@ -0,0 +1,152 @@ +// Package leanvmsidecar proves and verifies leanVM aggregates through leanvm-sidecar, over its Unix socket. Claims, +// proofs, signatures and blobs are byte strings in the forms bins/leanvm-sidecar/README.md specifies. +// +// A Client serves one call at a time. After an error other than a *Rejection, close it and dial again. A request +// longer than the sidecar takes (64 MiB) fails so: the sidecar refuses it unread and closes the connection. +package leanvmsidecar + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "net" +) + +// The longest response read: a request's claims, at most its 64 MiB, and room for a proof. +const maxResponse = 80 << 20 + +var errMalformed = errors.New("leanvm-sidecar: a response that does not decode") + +// Rejection is the sidecar's refusal of a request (status 1 to 5), which leaves the connection usable. +type Rejection struct { + Status byte + Message string +} + +func (r *Rejection) Error() string { + return fmt.Sprintf("leanvm-sidecar refused the request (status %d): %s", r.Status, r.Message) +} + +// Proven is a proof and the claims it proves. +type Proven struct{ Claims, Proof []byte } + +// Client is a connection to a sidecar. +type Client struct{ conn net.Conn } + +// Dial connects to the sidecar listening on socket, and checks it speaks protocol version 1 for verifier leanvm-v1. +func Dial(socket string) (*Client, error) { + conn, err := net.Dial("unix", socket) + if err != nil { + return nil, err + } + hello, err := readFrame(conn) + if err == nil && (len(hello) < 20 || string(hello[:12]) != "leanvmsc\x01\x00\x00\x00" || + !bytes.HasPrefix(hello[20:], appendField(nil, []byte("leanvm-v1")))) { + err = errors.New("leanvm-sidecar: not a sidecar of protocol version 1 for verifier leanvm-v1") + } + if err != nil { + conn.Close() + return nil, err + } + return &Client{conn}, nil +} + +// Close closes the connection; the sidecar drops a request it has not started proving. +func (c *Client) Close() error { return c.conn.Close() } + +// Aggregate proves the children's claims and the signatures, and commits to the blobs, at the code rate +// 2^-logInvRate (1 to 4). A leanXMSS or leanSPHINCS signature is its pubkey, message and signature, end to end. With +// only nil the proof publishes every claim it covers, else exactly those claims. Children are not verified: verify an +// untrusted one first. +func (c *Client) Aggregate(children []Proven, leanXmss, leanSphincs, blobs [][]byte, only []byte, + logInvRate byte) (Proven, error) { + b := binary.LittleEndian.AppendUint64([]byte{1, logInvRate}, uint64(len(children))) + for _, child := range children { + b = appendField(appendField(b, child.Claims), child.Proof) + } + for _, list := range [][][]byte{leanXmss, leanSphincs, blobs} { + b = binary.LittleEndian.AppendUint64(b, uint64(len(list))) + for _, item := range list { + b = append(b, item...) + } + } + if only == nil { + b = append(b, 0) + } else { + b = appendField(append(b, 1), only) + } + r, err := c.call(b) + if err != nil { + return Proven{}, err + } + claims, r, ok1 := field(r) + proof, r, ok2 := field(r) + if !ok1 || !ok2 || len(r) != 0 { + return Proven{}, errMalformed + } + return Proven{claims, proof}, nil +} + +// Verify says whether proof proves claims, as the library's AggregateProof::verify does. +func (c *Client) Verify(claims, proof []byte) (bool, error) { + r, err := c.call(appendField(appendField([]byte{2}, claims), proof)) + if err != nil { + return false, err + } + if len(r) != 1 || r[0] > 1 { + return false, errMalformed + } + return r[0] == 1, nil +} + +// call sends a request and returns its successful response, past the status. +func (c *Client) call(request []byte) ([]byte, error) { + if _, err := c.conn.Write(appendField(nil, request)); err != nil { + return nil, err + } + r, err := readFrame(c.conn) + if err != nil { + return nil, err + } + if len(r) == 0 { + return nil, errMalformed + } + if r[0] != 0 { + message, rest, ok := field(r[1:]) + if !ok || len(rest) != 0 { + return nil, errMalformed + } + return nil, &Rejection{r[0], string(message)} + } + return r[1:], nil +} + +func readFrame(conn io.Reader) ([]byte, error) { + var prefix [8]byte + if _, err := io.ReadFull(conn, prefix[:]); err != nil { + return nil, err + } + n := binary.LittleEndian.Uint64(prefix[:]) + if n > maxResponse { + return nil, errMalformed + } + body := make([]byte, n) + _, err := io.ReadFull(conn, body) + return body, err +} + +// appendField appends v after its u64 length. +func appendField(b, v []byte) []byte { + return append(binary.LittleEndian.AppendUint64(b, uint64(len(v))), v...) +} + +// field splits a length-prefixed field off the front of b. +func field(b []byte) (v, rest []byte, ok bool) { + if len(b) < 8 || binary.LittleEndian.Uint64(b) > uint64(len(b)-8) { + return nil, nil, false + } + n := 8 + binary.LittleEndian.Uint64(b) + return b[8:n], b[n:], true +} diff --git a/bins/leanvm-sidecar/clients/go/client_test.go b/bins/leanvm-sidecar/clients/go/client_test.go new file mode 100644 index 0000000000..9bc3f6cf59 --- /dev/null +++ b/bins/leanvm-sidecar/clients/go/client_test.go @@ -0,0 +1,55 @@ +package leanvmsidecar + +import ( + "bytes" + "errors" + "os" + "testing" +) + +// Against a running sidecar, whose socket LEANVM_SIDECAR_SOCKET names: a blob-only leaf, which needs no keys, a node +// over it, verifying both ways, and a refusal that leaves the connection usable. +func TestSidecar(t *testing.T) { + socket := os.Getenv("LEANVM_SIDECAR_SOCKET") + if socket == "" { + t.Skip("LEANVM_SIDECAR_SOCKET names no sidecar") + } + c, err := Dial(socket) + if err != nil { + t.Fatal(err) + } + defer c.Close() + + blob := make([]byte, 8192*16) + for i := range blob { + blob[i] = byte(i) + } + leaf, err := c.Aggregate(nil, nil, nil, [][]byte{blob}, nil, 2) + if err != nil { + t.Fatal(err) + } + node, err := c.Aggregate([]Proven{leaf}, nil, nil, nil, nil, 1) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(node.Claims, leaf.Claims) { + t.Fatal("the node publishes other claims than its leaf") + } + noClaims := make([]byte, 17) + for _, check := range []struct { + claims, proof []byte + valid bool + }{{leaf.Claims, leaf.Proof, true}, {node.Claims, node.Proof, true}, {noClaims, node.Proof, false}, {node.Claims, nil, false}} { + if valid, err := c.Verify(check.claims, check.proof); err != nil || valid != check.valid { + t.Fatalf("verify: %v, %v, want %v", valid, err, check.valid) + } + } + + var rejection *Rejection + if _, err := c.Aggregate(nil, nil, nil, [][]byte{blob}, nil, 0); !errors.As(err, &rejection) || rejection.Status != 1 { + t.Fatalf("rate 0: %v", err) + } + if valid, err := c.Verify(leaf.Claims, leaf.Proof); err != nil || !valid { + t.Fatalf("after a refusal: %v, %v", valid, err) + } +} diff --git a/bins/leanvm-sidecar/clients/go/go.mod b/bins/leanvm-sidecar/clients/go/go.mod new file mode 100644 index 0000000000..2afff0845a --- /dev/null +++ b/bins/leanvm-sidecar/clients/go/go.mod @@ -0,0 +1,3 @@ +module github.com/leanEthereum/leanVM/bins/leanvm-sidecar/clients/go + +go 1.22