From 210a7a93b556b598d3504d6ed332e0e140e3584d Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Sun, 8 Nov 2026 21:11:46 +0000 Subject: [PATCH 01/12] constraints: expand each table's state once, fold with the next message A table's first active round now builds its unfolded state (lin, left_g, right_g over all its rows) and its message in one pass over the K columns, so the join no longer reads them a second time. Every fold then computes the next round's message from the blocks it just wrote (each task folds block k of all four quarters into block k of both next halves), so a round is one pass over the state instead of a message sweep plus a fold sweep. Below sixteen rows per half the old fold and message stay. message_k, join and their row loader are gone. Proof bytes unchanged (PROOFDIGEST f090f5e4ee18ffad). Profile (perf, xmss 900 rate 1, 3 passes per run): the sumcheck kernels were the K-column passes (message_k ~40%, join ~43%; loads and stores hottest in join, so bandwidth bound), the folded rounds only ~18%. Kernel samples per run, interleaved: base 1763/2337/2501/2291/2268/2171/2606 (median 2291, ~380 ms CPU per pass), new 1566/1700/1525/1990/1706 (median 1700, ~283 ms), -26%. Prove constraints minus Constraint evals, tracing wall on the loaded host, 10-11 runs each: base min 148 ms median 234 ms, new min 119 ms median 189 ms. Whole pass (ab.sh 4x4): base cpu min 11.72 med 11.99 s, new min 11.54 med 12.17 s, inside the noise (expected gain ~0.8% CPU). Peak memory 3.111 -> 3.131 GiB. Rebased onto e5d75f858 (after #724): the digests and measurements above were taken on the older base 8577d1e21. On the new base the series leaves the proofs byte-identical: PROOFDIGEST 7d91b8513bdd4b03 (aggregate --xmss 900 --log-inv-rate 1), 3c41e153f8b2ee37 (aggregate --sphincs 410 --log-inv-rate 1), aa4b2ebd7a6989b2 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). --- crates/leanvm_core/src/constraints.rs | 286 ++++++++++++++------------ 1 file changed, 152 insertions(+), 134 deletions(-) diff --git a/crates/leanvm_core/src/constraints.rs b/crates/leanvm_core/src/constraints.rs index a95ae5fe43..4adbb89bf1 100644 --- a/crates/leanvm_core/src/constraints.rs +++ b/crates/leanvm_core/src/constraints.rs @@ -79,7 +79,7 @@ const FOLD_CHUNK: usize = 1 << 10; /// A table's summand `P(v) = constant + lin(v) + Σ_g v[left_g] · right_g(v)`, its products grouped /// by a shared column. Its homogeneous quadratic part is `Q(v) = Σ_g v[left_g] · right_g(v)`. /// -/// The `right_g` and `lin` are linear, so a joined table folds them as single `E` columns +/// The `right_g` and `lin` are linear, so an active table folds them as single `E` columns /// instead of folding every column: its state is `[lin, left_0, right_0, left_1, right_1, ..]`. struct Form { constant: F192, @@ -144,7 +144,7 @@ impl Form { } } - /// Columns of the folded state: `lin`, then `left_g` and `right_g` per group. + /// Columns of the state: `lin`, then `left_g` and `right_g` per group. fn width(&self) -> usize { 1 + 2 * self.groups.len() } @@ -176,31 +176,9 @@ fn splat(terms: &Lin) -> Vec<(usize, F192x8)> { terms.iter().map(|&(c, w)| (c, F192x8::splat(w))).collect() } -/// A worker's scratch for [`load_rows`] over `n` columns: `end` and `sum`. -fn rows_scratch(n: usize) -> (Vec, Vec) { - let zero = F64x8::from_array(&[F64::ZERO; 8]); - (vec![zero; n], vec![zero; n]) -} - -/// Loads the vectors of rows `i` and `i + half` of every column in `read`: `end` takes the upper vector -/// with `at_one` and the lower without, `sum` their sum. -#[inline(always)] -fn load_rows( - cols: &[&[F64]], - read: &[usize], - i: usize, - half: usize, - at_one: bool, - (end, sum): &mut (Vec, Vec), -) { - for &c in read { - let (lo, hi) = (load_k(cols[c], i), load_k(cols[c], i + half)); - (end[c], sum[c]) = (if at_one { hi } else { lo }, lo + hi); - } -} - -/// A joined table's folded state: per vector of eight rows, its `width` columns side by side, -/// `state[b * width + c]`. Below eight rows, one vector per column holds them all. +/// A table's state: per vector of eight rows, its `width` columns side by side, `state[b * width + c]`, +/// the blocks of the rows below the round's `half` first, then those above. Below eight rows, one vector +/// per column holds them all. type State = Vec; /// The blocks task `t` covers out of `n`. @@ -216,12 +194,14 @@ fn finish([endpoint, slope]: [Prod8; 2], constant: F192) -> [F192; 2] { [endpoint.reduce() + constant, slope.reduce()] } -/// An active round of a table that has not joined: its message from the `K` columns, in place. -/// Per vector of rows, the linear forms are `E x K` mixed dots, and every product of the -/// summand is one mixed product against a reduced `right_g`. -fn message_k(form: &Form, cols: &[&[F64]], eqr: &[F192], half: usize, at_one: bool) -> [F192; 2] { +/// The first round a table is active: its message, and its state over all its rows, unfolded, in one +/// pass over the `K` columns, which no later round reads. The linear forms are `E x K` mixed dots, and +/// every product of the summand is one mixed product against a reduced `right_g`. +fn expand(form: &Form, cols: &[&[F64]], eqr: &[F192], half: usize, at_one: bool) -> (State, [F192; 2]) { if half < 8 { - return message_e(form, &form.small_state(cols, 2 * half), eqr, half, at_one); + let state = form.small_state(cols, 2 * half); + let message = message_e(form, &state, eqr, half, at_one); + return (state, message); } let lin = splat(&form.lin); let rights: Vec<_> = form.groups.iter().map(|(_, right)| splat(right)).collect(); @@ -229,33 +209,55 @@ fn message_k(form: &Form, cols: &[&[F64]], eqr: &[F192], half: usize, at_one: bo terms .iter() .fold(Prod8::ZERO, |acc, &(c, w)| acc ^ w.mul_base_unreduced(vals[c])) + .reduce_lanes() }; - let blocks = half / 8; - let acc = parallel::map_reduce_with_state( - blocks.div_ceil(TASK_BLOCKS), - || rows_scratch(cols.len()), - || [Prod8::ZERO; 2], - |rows, acc, t| { - for b in task_blocks(t, blocks) { - let i = 8 * b; - load_rows(cols, &form.used, i, half, at_one, rows); - let (end, sum) = &*rows; - let (mut endpoint, mut slope) = (dot(&lin, end), Prod8::ZERO); - for ((left, _), right) in form.groups.iter().zip(&rights) { - endpoint ^= dot(right, end).reduce_lanes().mul_base_unreduced(end[*left]); - slope ^= dot(right, sum).reduce_lanes().mul_base_unreduced(sum[*left]); - } - let eq = load_e(eqr, i); - acc[0] ^= endpoint.reduce_lanes().mul_unreduced(eq); - acc[1] ^= slope.reduce_lanes().mul_unreduced(eq); - } - }, - xor2, - ); - finish(acc, form.constant) + let entries = |vals: &[F64x8], out: &mut [F192x8]| { + out[0] = dot(&lin, vals); + for (g, ((left, _), right)) in form.groups.iter().zip(&rights).enumerate() { + out[1 + 2 * g] = F192x8::from(vals[*left]); + out[2 + 2 * g] = dot(right, vals); + } + }; + let (blocks, width) = (half / 8, form.width()); + let zero = F64x8::from_array(&[F64::ZERO; 8]); + let mut acc = [Prod8::ZERO; 2]; + // SAFETY: every task writes all `width` entries of each of its blocks in both halves. + let state = unsafe { + primitives::filled_vec(2 * blocks * width, |state| { + let (lows, highs) = state.split_at_mut(blocks * width); + acc = parallel::map_reduce_chunks_mut( + [lows, highs], + [TASK_BLOCKS * width; 2], + || (vec![zero; cols.len()], vec![zero; cols.len()]), + || [Prod8::ZERO; 2], + |(lo, hi), acc, t, [lows, highs]| { + let outs = lows.chunks_exact_mut(width).zip(highs.chunks_exact_mut(width)); + for (b, (lo_out, hi_out)) in task_blocks(t, blocks).zip(outs) { + let i = 8 * b; + for &c in &form.used { + (lo[c], hi[c]) = (load_k(cols[c], i), load_k(cols[c], i + half)); + } + entries(lo, lo_out); + entries(hi, hi_out); + let (end, end_k) = if at_one { (&*hi_out, &*hi) } else { (&*lo_out, &*lo) }; + let (mut endpoint, mut slope) = (Prod8::ZERO, Prod8::ZERO); + for (g, (left, _)) in form.groups.iter().enumerate() { + endpoint ^= end[2 + 2 * g].mul_base_unreduced(end_k[*left]); + slope ^= (lo_out[2 + 2 * g] + hi_out[2 + 2 * g]).mul_base_unreduced(lo[*left] + hi[*left]); + } + let eq = load_e(eqr, i); + acc[0] ^= (endpoint.reduce_lanes() + end[0]).mul_unreduced(eq); + acc[1] ^= slope.reduce_lanes().mul_unreduced(eq); + } + }, + xor2, + ); + }) + }; + (state, finish(acc, form.constant)) } -/// An active round of a joined table, from its folded state. +/// An active round of a table, from its state. fn message_e(form: &Form, state: &State, eqr: &[F192], half: usize, at_one: bool) -> [F192; 2] { if half < 8 { // Fewer row pairs than a vector: `[Σ_i eqr[i]·end_i, Σ_i eqr[i]·Q(lo_i + hi_i)]` by lanes. @@ -280,15 +282,7 @@ fn message_e(form: &Form, state: &State, eqr: &[F192], half: usize, at_one: bool |acc, t| { for b in task_blocks(t, blocks) { let (lo, hi) = (&lows[b * width..][..width], &highs[b * width..][..width]); - let end = if at_one { hi } else { lo }; - let (mut endpoint, mut slope) = (Prod8::ZERO, Prod8::ZERO); - for g in (1..width).step_by(2) { - endpoint ^= end[g].mul_unreduced(end[g + 1]); - slope ^= (lo[g] + hi[g]).mul_unreduced(lo[g + 1] + hi[g + 1]); - } - let eq = load_e(eqr, 8 * b); - acc[0] ^= (endpoint.reduce_lanes() + end[0]).mul_unreduced(eq); - acc[1] ^= slope.reduce_lanes().mul_unreduced(eq); + block_message(lo, hi, load_e(eqr, 8 * b), at_one, acc); } }, xor2, @@ -296,50 +290,59 @@ fn message_e(form: &Form, state: &State, eqr: &[F192], half: usize, at_one: bool finish(acc, form.constant) } -/// The round a table joins, after its challenge `r`: the state `lo + r·(lo + hi)` of every -/// entry, built straight from the `K` columns. By linearity `lin` and `right_g` take the -/// terms `w·lo + (w·r)·(lo + hi)`, so no row pays an `E x E` product. -fn join(form: &Form, cols: &[&[F64]], r: F192, half: usize) -> State { - if half < 8 { - let mut state = form.small_state(cols, 2 * half); - fold_state(&mut state, r, half); - return state; +/// Adds one block's terms to a table's message: `lo` and `hi` are its state entries in the +/// two halves, `eq` its rows' eq weights. +#[inline(always)] +fn block_message(lo: &[F192x8], hi: &[F192x8], eq: F192x8, at_one: bool, acc: &mut [Prod8; 2]) { + let end = if at_one { hi } else { lo }; + let (mut endpoint, mut slope) = (Prod8::ZERO, Prod8::ZERO); + for g in (1..lo.len()).step_by(2) { + endpoint ^= end[g].mul_unreduced(end[g + 1]); + slope ^= (lo[g] + hi[g]).mul_unreduced(lo[g + 1] + hi[g + 1]); } - let (blocks, width) = (half / 8, form.width()); - let pair = |terms: &Lin| -> Vec<_> { - terms - .iter() - .map(|&(c, w)| (c, F192x8::splat(w), F192x8::splat(w * r))) - .collect() - }; - let lin = pair(&form.lin); - let rights: Vec<_> = form.groups.iter().map(|(_, right)| pair(right)).collect(); + acc[0] ^= (endpoint.reduce_lanes() + end[0]).mul_unreduced(eq); + acc[1] ^= slope.reduce_lanes().mul_unreduced(eq); +} + +/// [`fold_state`], and the next round's message from the folded state, in one pass: a task folds +/// block `k` of every quarter into block `k` of both next halves, the pair that message reads. +/// `eqr` and `at_one` are the next round's; `None` below sixteen rows per half. +fn fold_and_message( + form: &Form, + state: &mut State, + r: F192, + half: usize, + eqr: &[F192], + at_one: bool, +) -> Option<[F192; 2]> { + if half < 16 { + fold_state(state, r, half); + return None; + } + let (pairs, width) = (half / 16, form.width()); let r8 = F192x8::splat(r); - // SAFETY: every task writes all `width` entries of each of its blocks. - unsafe { - primitives::filled_vec(blocks * width, |state| { - parallel::chunks_mut(state, TASK_BLOCKS * width, |t, out| { - let mut rows = rows_scratch(cols.len()); - for (b, out) in task_blocks(t, blocks).zip(out.chunks_exact_mut(width)) { - load_rows(cols, &form.used, 8 * b, half, false, &mut rows); - let (lo, sum) = &rows; - let dot = |terms: &[(usize, F192x8, F192x8)]| { - terms - .iter() - .fold(Prod8::ZERO, |acc, &(c, w, wr)| { - acc ^ w.mul_base_unreduced(lo[c]) ^ wr.mul_base_unreduced(sum[c]) - }) - .reduce_lanes() - }; - out[0] = dot(&lin); - for (g, ((left, _), right)) in form.groups.iter().zip(&rights).enumerate() { - out[1 + 2 * g] = F192x8::from(lo[*left]) + r8.mul_base_unreduced(sum[*left]).reduce_lanes(); - out[2 + 2 * g] = dot(right); - } + let (left, right) = state.split_at_mut(2 * pairs * width); + let (q0, q1) = left.split_at_mut(pairs * width); + let (q2, q3) = right.split_at_mut(pairs * width); + let acc = parallel::map_reduce_chunks_mut( + [q0, q1, q2, q3], + [TASK_BLOCKS * width; 4], + || (), + || [Prod8::ZERO; 2], + |(), acc, t, [lo, hi, upper_lo, upper_hi]| { + let quarters = lo.chunks_exact_mut(width).zip(hi.chunks_exact_mut(width)); + let uppers = upper_lo.chunks_exact(width).zip(upper_hi.chunks_exact(width)); + for (k, ((lo, hi), (upper_lo, upper_hi))) in task_blocks(t, pairs).zip(quarters.zip(uppers)) { + for (v, &u) in lo.iter_mut().zip(upper_lo).chain(hi.iter_mut().zip(upper_hi)) { + *v = *v + (*v + u) * r8; } - }); - }) - } + block_message(lo, hi, load_e(eqr, 8 * k), at_one, acc); + } + }, + xor2, + ); + state.truncate(2 * pairs * width); + Some(finish(acc, form.constant)) } /// The state folded by `r`, `v[i] = v[i] + r·(v[i] + v[i + half])` in every column, the upper half dropped. @@ -372,26 +375,35 @@ struct Table<'a> { weight: F192, form: Form, cols: &'a [&'a [F64]], - /// The folded state, once the table has joined: a few `E` columns, the batch's largest transients. + /// The state, from the table's first active round: a few `E` columns, the batch's largest transients. state: Option, + /// The next round's message, when the last fold built it. + next: Option<[F192; 2]>, } impl Table<'_> { - /// The message of the round that binds variable `m`, while the table is active. - fn message(&self, eqr: &[F192], m: usize, at_one: bool) -> [F192; 2] { + /// The message of the round that binds variable `m`, while the table is active. The first such + /// round expands the table's state. + fn message(&mut self, eqr: &[F192], m: usize, at_one: bool) -> [F192; 2] { + if let Some(p) = self.next.take() { + return p; + } let (form, half) = (&self.form, 1 << m); match &self.state { Some(state) => message_e(form, state, eqr, half, at_one), - None => message_k(form, self.cols, eqr, half, at_one), + None => { + let (state, p) = expand(form, self.cols, eqr, half, at_one); + self.state = Some(state); + p + } } } - /// Binds variable `m` to `r`: the join of a table that has not joined, the fold of one that has. - fn bind(&mut self, r: F192, m: usize) { - match &mut self.state { - Some(state) => fold_state(state, r, 1 << m), - None => self.state = Some(join(&self.form, self.cols, r, 1 << m)), - } + /// Binds variable `m` to `r`, folding the state, and builds the next round's message on the way + /// against its `eqr` and `at_one`. + fn bind(&mut self, r: F192, m: usize, eqr: &[F192], at_one: bool) { + let state = self.state.as_mut().expect("an active table has its state"); + self.next = fold_and_message(&self.form, state, r, 1 << m, eqr, at_one); } } @@ -422,6 +434,7 @@ pub fn prove(airs: &[Air<'_>], zeta: &[F192], ps: &mut ProverState) -> Vec], zeta: &[F192], ps: &mut ProverState) -> Vec], zeta: &[F192], ps: &mut ProverState) -> Vec 0 && zeta[m - 1].is_zero(); for t in &mut tables { if t.tau > m { t.weight *= eq_k; - t.bind(rk, m); + t.bind(rk, m, &eqr, at_one_next); } else { t.weight *= rk; } @@ -554,8 +569,9 @@ mod tests { }) } - /// Every round of a table's life, at row counts on both sides of one vector: the join's - /// message and state, then each fold and joined message, against the naive formulas. + /// Every round of a table's life, at row counts on both sides of one vector: the expanded + /// state's message, then each fold, its state's message and the next round's message the fold + /// builds, against the naive formulas. #[test] fn round_messages_match_naive_evaluation() { let mut s = 7; @@ -569,26 +585,28 @@ mod tests { .map(|c| c.iter().map(|&x| F192::from(x)).collect()) .collect(); let mut eqr = eq_table(&(0..m).map(|_| random_e(&mut s)).collect::>()); - let mut state: Option = None; + let mut state = State::new(); + let mut next: Option<([F192; 2], bool)> = None; let mut half = 1usize << m; while half >= 1 { for at_one in [false, true] { - let got = match &state { - None => message_k(&form, &views, &eqr, half, at_one), - Some(st) => message_e(&form, st, &eqr, half, at_one), - }; - assert_eq!( - got, - naive_message(&pows, &naive_cols, &eqr, half, at_one), - "m {m} half {half}" - ); + let want = naive_message(&pows, &naive_cols, &eqr, half, at_one); + if half == 1 << m { + let (st, p) = expand(&form, &views, &eqr, half, at_one); + assert_eq!(p, want, "expand, m {m}"); + state = st; + } + assert_eq!(message_e(&form, &state, &eqr, half, at_one), want, "m {m} half {half}"); + if let Some((fused, _)) = next.filter(|n| n.1 == at_one) { + assert_eq!(fused, want, "fused, m {m} half {half}"); + } } let r = random_e(&mut s); shrink_eq_high(&mut eqr); - match &mut state { - None => state = Some(join(&form, &views, r, half)), - Some(st) => fold_state(st, r, half), - } + let at_one = half.is_multiple_of(32); + let fused = fold_and_message(&form, &mut state, r, half, &eqr, at_one); + assert_eq!(fused.is_some(), half >= 16, "m {m} half {half}"); + next = fused.map(|p| (p, at_one)); for c in &mut naive_cols { *c = (0..half).map(|i| interp(c[i], c[i + half], r)).collect(); } From ee76820d6d15cb8b0d8875fc5d9e4f860100be88 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Sun, 8 Nov 2026 21:41:59 +0000 Subject: [PATCH 02/12] leanvm_core: prove the bus GKR two rounds per pass with enough threads Port of upstream #557 (d6f597e3) onto main's batched triple GKR. The vertical kernels read a layer once per sumcheck round. The new `paired` kernels take the rounds in pairs: four rows sharing all but their two lowest index bits form a block, each child bilinear in those bits, and one pass sums eq(q) times the product of the four children over the blocks, a degree-four bivariate grid that gives round j's message and, once its challenge is drawn, round j+1's. The next pass folds every block into one row at both challenges and sums the next pair's grid from the folded rows while they are in registers. The round messages are the same polynomials, so the proof is unchanged. Main's properties are kept: the three trees run in one dispatch, the implicit identity tail and the identity runs are skipped (an identity quad adds only to the grid's constant term, which neither message reads, and folds to an unwritten identity quad), and the rows stay transposed between passes, eight blocks per `Quad` of F192x8 lanes, regrouped with lane deinterleaves after each fold. `F192x8::load_blocks` transposes eight runs of sixteen elements for the first pass of a layer. The pairs multiply more (the first pass of a layer costs about twice the old message pass) to read and write a layer less often, so they run only with at least PAIRED_THREADS = 8 pool threads (upstream measured even at 4 and a gain at 8); fewer threads keep the one-round vertical kernels. xmss 900, log-inv-rate 1, PROOFDIGEST f090f5e4ee18ffad unchanged (default threads and LEANVM_NUM_THREADS=1). 7950X3D, loaded host, ab.sh 3 rounds x 4 repeats, CPU and wall per proving pass, lv-base -> this: load ~75: cpu med 11.08 -> 10.99 s, min 10.76 -> 10.58; wall med 2.80 -> 2.82 s, min 2.06 -> 2.19 load ~95: cpu med 11.35 -> 11.58 s, min 10.67 -> 11.05; wall med 4.11 -> 3.91 s, min 3.20 -> 2.93 load 230-580: cpu med 12.45 -> 12.46 s, min 11.88 -> 11.84; wall med 6.54 -> 6.98 s, min 5.60 -> 5.52 Whole-pass CPU is unchanged within noise. Bus GKR stage wall (tracing, 16 threads, load ~75, 4 interleaved runs): 305/360/364/489 ms -> 267/287/288/474 ms, median about -20%; at load 160-600 it was 341-495 -> 305-492 ms. perf samples in leanvm_core::gkr over two passes at 16 threads: kernels 2525/2338 -> 2578/2884, all GKR symbols including the dispatcher's wait 3045/3188 -> 2966/3298, so the stage's CPU is about even. At one thread (pairs forced on, before the gate): GKR samples 1827/1867 -> 2114/2243 (+15-20%), whole-pass CPU med 8.19 -> 8.40 s (ab.sh 2 x 2), the regression upstream reported. With the gate a one-thread proof runs the previous kernels. Rebased onto e5d75f858 (after #724): the digests and measurements above were taken on the older base 8577d1e21. On the new base the series leaves the proofs byte-identical: PROOFDIGEST 7d91b8513bdd4b03 (aggregate --xmss 900 --log-inv-rate 1), 3c41e153f8b2ee37 (aggregate --sphincs 410 --log-inv-rate 1), aa4b2ebd7a6989b2 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). Co-Authored-By: Claude Opus 5.5 --- crates/leanvm_core/src/gkr.rs | 495 +++++++++++++++++++++++++++++- crates/primitives/src/field/x8.rs | 11 + 2 files changed, 490 insertions(+), 16 deletions(-) diff --git a/crates/leanvm_core/src/gkr.rs b/crates/leanvm_core/src/gkr.rs index c04afeff36..30ff50e61e 100644 --- a/crates/leanvm_core/src/gkr.rs +++ b/crates/leanvm_core/src/gkr.rs @@ -7,7 +7,8 @@ //! into `E` upstream, [`crate::leaf`]). //! //! On AVX-512 the large rounds run eight pairs of rows at a time in transposed form -//! (`vertical`), and skip the runs of rows known to be the identity. +//! (`vertical`), or with enough threads two rounds per pass, eight blocks of four rows at +//! a time (`paired`), and skip the runs of rows known to be the identity. use std::ops::Range; @@ -266,6 +267,12 @@ pub(crate) const SKIP_IDENTITY: bool = cfg!(target_arch = "x86_64"); #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] const VERTICAL_ROWS: usize = 1 << 10; +/// Threads from which a large layer runs its rounds two per pass ([`paired`]). Those +/// kernels multiply more to read the layer half as often, which pays only once the +/// threads together wait on memory; fewer run the rounds one at a time. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] +const PAIRED_THREADS: usize = 8; + /// Two binary product levels contracted into one degree-four layer. struct QuaternaryLayerState { /// Four child tables interleaved in their original order. This lets the @@ -279,8 +286,13 @@ struct QuaternaryLayerState { groups: Vec, #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] spare: Vec, + /// Once folded by the paired kernels, the rows live here instead. + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + quads: Vec, + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + spare_quads: Vec, /// Runs of rows known to be the identity, sorted: the kernels skip them, and - /// the vertical ones leave their groups unwritten. + /// the vertical ones leave their groups and quads unwritten. ones: Vec>, } @@ -298,6 +310,10 @@ impl QuaternaryLayerState { groups: Vec::new(), #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] spare: Vec::new(), + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + quads: Vec::new(), + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + spare_quads: Vec::new(), ones: Vec::new(), } } @@ -338,6 +354,27 @@ impl QuaternaryLayerState { } } + /// The paired kernels' view of the rows. + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + fn paired_input(&self) -> paired::Input<'_> { + let data = if self.quads.is_empty() { + paired::Data::Rows(&self.values) + } else { + paired::Data::Quads(&self.quads) + }; + paired::Input { data, ones: &self.ones } + } + + /// Move the rows the paired kernels left back into `values`. + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + fn leave_quads(&mut self) { + if !self.quads.is_empty() { + self.values = paired::to_rows(&self.quads, &self.ones, self.logical_rows); + self.quads = Vec::new(); + self.spare_quads = Vec::new(); + } + } + fn round_message_scalar(&self, eq: &SplitEq) -> [F192; 4] { let stored_rows = self.values.len() / 4; let full_pairs = stored_rows / 2; @@ -530,6 +567,26 @@ fn fold_and_message(trees: &mut [QuaternaryLayerState; 3], challenge: F192, eq: trees.each_mut().map(|tree| tree.fold_and_message_scalar(challenge, eq)) } +/// Bind every tree's next two variables to `r` in the paired kernels, returning the +/// polynomial of the two rounds after when given their weights. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] +fn fold_pair(trees: &mut [QuaternaryLayerState; 3], r: [F192; 2], eq: Option<&SplitEq>) -> Option<[paired::Grid; 3]> { + let mut spares = trees.each_mut().map(|tree| std::mem::take(&mut tree.spare_quads)); + let grids = paired::fold( + trees.each_ref().map(QuaternaryLayerState::paired_input), + r, + eq, + spares.each_mut(), + ); + for (tree, spare) in trees.iter_mut().zip(spares) { + tree.spare_quads = std::mem::replace(&mut tree.quads, spare); + tree.values = Vec::new(); + tree.logical_rows /= 4; + tree.ones = coarsen(&tree.ones, 2); + } + grids +} + /// The AVX-512 kernels: eight pairs of rows at a time, transposed so that every /// product is lane-wise ([`primitives::field::x8`]). After its first fold a layer /// keeps its rows transposed, in [`Group`]s, until it is small enough for the scalar @@ -543,7 +600,7 @@ mod vertical { use std::ops::Range; /// Whether `ones` covers `rows`. - fn covered(ones: &[Range], rows: Range) -> bool { + pub(super) fn covered(ones: &[Range], rows: Range) -> bool { let i = ones.partition_point(|r| r.start <= rows.start); i > 0 && ones[i - 1].end >= rows.end } @@ -779,6 +836,331 @@ mod vertical { } } +/// The AVX-512 kernels that run a layer's rounds two per pass. Four rows sharing all but +/// their two lowest index bits form a block, and each child of a block is bilinear in those +/// two bits: +/// +/// ```text +/// V_c(Y1, Y2) = v_00 + Y1 (v_00 + v_10) + Y2 (v_00 + v_01) + Y1 Y2 (v_00 + v_10 + v_01 + v_11) +/// ``` +/// +/// One pass sums `eq(q)·∏_c V_c(Y1, Y2)` over the blocks `q`, a [`Grid`] that gives the +/// first round's message, and the second's once `Y1` is bound. The next pass folds each +/// block into one row at both challenges and sums the next pair's grid from the folded +/// rows while they are in registers, so a layer is read once every two rounds. +/// +/// Eight blocks run at a time, transposed so that every product is lane-wise +/// ([`primitives::field::x8`]). After its first fold a layer keeps its rows so, in +/// [`Quad`]s, until it is small enough for the scalar kernels. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] +mod paired { + use super::vertical::covered; + use super::{SplitEq, coarsen}; + use primitives::field::F192; + use primitives::field::x8::{F192x8, Sum8}; + use primitives::stream::Stream; + use std::ops::Range; + + /// Eight blocks of four rows: `q[4b + c]` lane `p` is child `c` of row `4p + b`. + pub(super) type Quad = [F192x8; 16]; + + /// Quads per parallel task. + const TASK: usize = 32; + + /// Quads below which a dispatch costs more than it saves. + const PARALLEL: usize = 64; + + /// Quad `i` of a row table, the identity past its end. + #[inline(always)] + fn load_quad(values: &[F192], i: usize) -> Quad { + match values.get(128 * i..128 * i + 128) { + Some(run) => F192x8::load_blocks(run.try_into().unwrap()), + None => { + let mut padded = [F192::ONE; 128]; + let tail = &values[(128 * i).min(values.len())..]; + padded[..tail.len()].copy_from_slice(tail); + F192x8::load_blocks(&padded) + } + } + } + + /// Each block of `q` at `Y1 = r[0]`, then `Y2 = r[1]`: lane `p` of child `c` is block `p`'s row. + #[inline(always)] + fn fold_quad(q: &Quad, r: [F192x8; 2]) -> [F192x8; 4] { + std::array::from_fn(|c| { + let low = q[c] + r[0] * (q[c] + q[4 + c]); + let high = q[8 + c] + r[0] * (q[8 + c] + q[12 + c]); + low + r[1] * (low + high) + }) + } + + /// Four folded quads, rows `8k..8k + 8` in `rows[k]`, as the quad of the eight blocks they make. + #[inline(always)] + fn regroup(rows: [[F192x8; 4]; 4]) -> Quad { + let mut q = [F192x8::splat(F192::ZERO); 16]; + for c in 0..4 { + let (even01, odd01) = F192x8::deinterleave(rows[0][c], rows[1][c]); + let (even23, odd23) = F192x8::deinterleave(rows[2][c], rows[3][c]); + (q[c], q[8 + c]) = F192x8::deinterleave(even01, even23); + (q[4 + c], q[12 + c]) = F192x8::deinterleave(odd01, odd23); + } + q + } + + /// A value added by XOR: an element, a vector of them, or an array of either. + trait Xor: Copy { + fn xor(self, rhs: Self) -> Self; + } + + impl Xor for F192 { + #[inline(always)] + fn xor(self, rhs: Self) -> Self { + self + rhs + } + } + + impl Xor for F192x8 { + #[inline(always)] + fn xor(self, rhs: Self) -> Self { + self + rhs + } + } + + impl Xor for [T; N] { + #[inline(always)] + fn xor(self, rhs: Self) -> Self { + std::array::from_fn(|i| self[i].xor(rhs[i])) + } + } + + /// The product of two linear polynomials by Karatsuba, over the coefficient product `mul`. + #[inline(always)] + fn product2([u0, u1]: [T; 2], [w0, w1]: [T; 2], mul: impl Fn(T, T) -> P) -> [P; 3] { + let (low, high, sum) = (mul(u0, w0), mul(u1, w1), mul(u0.xor(u1), w0.xor(w1))); + [low, sum.xor(low).xor(high), high] + } + + /// Karatsuba's six products of two quadratics: the three of equal degrees, then the + /// product of each pair's sums. Being combined by sums alone ([`combine3`]), products + /// summed over many operands are combined once, after the sum. + #[inline(always)] + fn terms3(u: [T; 3], w: [T; 3], mul: impl Fn(T, T) -> P) -> [P; 6] { + let sum = |i: usize, j: usize| mul(u[i].xor(u[j]), w[i].xor(w[j])); + [ + mul(u[0], w[0]), + mul(u[1], w[1]), + mul(u[2], w[2]), + sum(0, 1), + sum(0, 2), + sum(1, 2), + ] + } + + /// The product's coefficients from [`terms3`]: each pair's sum holds its cross term + /// and the two products of equal degrees. + fn combine3([z0, z1, z2, s01, s02, s12]: [P; 6]) -> [P; 5] { + [ + z0, + s01.xor(z0).xor(z1), + s02.xor(z0).xor(z2).xor(z1), + s12.xor(z1).xor(z2), + z2, + ] + } + + /// The running sums of block products' Karatsuba terms, in `Y2` then `Y1`. + type Sums = [[Sum8; 6]; 6]; + + /// Add the terms of eight blocks' `weight·∏_c V_c`. The weight scales child 0, and with it the product. + #[inline(always)] + fn add_quad(sums: &mut Sums, q: &Quad, weight: F192x8) { + // Child `c` as `[[1, Y1], [Y2, Y1 Y2]]` coefficients. + let child = |c: usize| { + let y1 = q[c] + q[4 + c]; + [[q[c], y1], [q[c] + q[8 + c], y1 + q[8 + c] + q[12 + c]]] + }; + let [[a, b], [c, d]] = child(0); + let first = [[a * weight, b * weight], [c * weight, d * weight]]; + // Two pairs of bilinear factors make two biquadratics, nine products each. + let biquadratic = |a, b| product2(a, b, |u, w| product2(u, w, |x: F192x8, y| x * y)); + let (left, right) = (biquadratic(first, child(1)), biquadratic(child(2), child(3))); + for (sums, (u, w)) in sums.iter_mut().zip(terms3(left, right, |u, w| (u, w))) { + for (sum, (x, y)) in sums.iter_mut().zip(terms3(u, w, |x, y| (x, y))) { + *sum ^= x.mul_sum(y); + } + } + } + + /// `eq` at the eight blocks of quad `i`. + #[inline(always)] + fn weight(eq: &SplitEq, i: usize) -> F192x8 { + let shift = eq.lo_bits - 3; + eq.lo8[i & ((1 << shift) - 1)] * F192x8::splat(eq.hi[i >> shift]) + } + + /// `P(Y1, Y2) = Σ_q eq(q)·∏_c V_c(Y1, Y2)` of two consecutive rounds, its coefficients + /// indexed `[Y2 power][Y1 power]`. `Y1` is bound first. + pub(super) struct Grid([[F192; 5]; 5]); + + impl Grid { + fn new(sums: [[F192; 6]; 6]) -> Self { + Self(combine3(sums.map(combine3))) + } + + /// The first round's `[X¹..X⁴]`, its halves weighted by `eq(p, Y2)`. In characteristic + /// two that is `1 + p` at `Y2 = 0` and `p` at `Y2 = 1`, so `[X^k] = P_{0,k} + p·Σ_{m≥1} P_{m,k}`. + pub(super) fn first_round(&self, p: F192) -> [F192; 4] { + let [low, upper @ ..] = &self.0; + std::array::from_fn(|i| low[i + 1] + p * upper.iter().fold(F192::ZERO, |sum, row| sum + row[i + 1])) + } + + /// The second round's `[X¹..X⁴]` once `Y1` is bound to `r`: `[X^m] = Σ_k r^k P_{m,k}`. + pub(super) fn second_round(&self, r: F192) -> [F192; 4] { + std::array::from_fn(|i| self.0[i + 1].iter().rev().fold(F192::ZERO, |sum, &c| sum * r + c)) + } + } + + pub(super) enum Data<'a> { + Rows(&'a [F192]), + Quads(&'a [Quad]), + } + + /// A layer's rows as the kernels read them, with the runs known to be the identity. + pub(super) struct Input<'a> { + pub(super) data: Data<'a>, + pub(super) ones: &'a [Range], + } + + impl Input<'_> { + fn len(&self) -> usize { + match self.data { + Data::Rows(values) => values.len().div_ceil(128), + Data::Quads(quads) => quads.len(), + } + } + + /// Quad `i`, or `None` where it is the identity: past the end or where `ones` says so. + #[inline(always)] + fn get(&self, i: usize) -> Option { + if i >= self.len() || covered(self.ones, 32 * i..32 * i + 32) { + return None; + } + Some(match self.data { + Data::Rows(values) => load_quad(values, i), + Data::Quads(quads) => quads[i], + }) + } + } + + /// Each tree's grid from `task(t, quads)`'s reduced sums, over its `lens[t]` quads, + /// the three trees in one dispatch. + fn grids(lens: [usize; 3], task: impl Fn(usize, Range) -> [[F192; 6]; 6] + Sync) -> [Grid; 3] { + let tasks: Vec<(usize, usize)> = (0..3) + .flat_map(|t| (0..lens[t]).step_by(TASK).map(move |first| (t, first))) + .collect(); + let zero = [[[F192::ZERO; 6]; 6]; 3]; + let run = |i: usize| { + let (t, first) = tasks[i]; + let mut sums = zero; + sums[t] = task(t, first..(first + TASK).min(lens[t])); + sums + }; + let add = |a: [[[F192; 6]; 6]; 3], b| a.xor(b); + let sums = if lens.iter().sum::() > PARALLEL { + parallel::map_reduce(tasks.len(), || zero, run, add) + } else { + (0..tasks.len()).map(run).fold(zero, add) + }; + sums.map(Grid::new) + } + + /// Every tree's grid. An identity quad adds only to `P_{0,0}`, which neither message reads. + pub(super) fn grid(inputs: [Input; 3], eq: &SplitEq) -> [Grid; 3] { + grids(inputs.each_ref().map(Input::len), |t, quads| { + let mut sums = [[Sum8::ZERO; 6]; 6]; + for i in quads { + if let Some(q) = inputs[t].get(i) { + add_quad(&mut sums, &q, weight(eq, i)); + } + } + sums.map(|row| row.map(Sum8::reduce)) + }) + } + + /// Fold every tree's blocks at `r` into its `out`, in one dispatch for the three, + /// returning the next two rounds' grids when given their weights. + pub(super) fn fold( + inputs: [Input; 3], + r: [F192; 2], + eq: Option<&SplitEq>, + outs: [&mut Vec; 3], + ) -> Option<[Grid; 3]> { + let n_out = inputs.each_ref().map(|input| input.len().div_ceil(4)); + let mut outs = outs; + let dst: [parallel::SendPtr; 3] = std::array::from_fn(|t| { + let out = &mut *outs[t]; + if out.capacity() < n_out[t] { + *out = Vec::new(); + // SAFETY: the tasks below write every quad before anything reads it. + *out = unsafe { primitives::uninit_vec(n_out[t]) }; + } else { + // SAFETY: as above; `Quad` is plain data. + unsafe { out.set_len(n_out[t]) }; + } + parallel::SendPtr(out.as_mut_ptr()) + }); + let r = r.map(F192x8::splat); + let ones = inputs.each_ref().map(|input| coarsen(input.ones, 2)); + let identity = [F192x8::splat(F192::ONE); 4]; + let grids = grids(n_out, |t, quads| { + let stream = Stream::new(); + let mut sums = [[Sum8::ZERO; 6]; 6]; + for o in quads { + // Identity blocks fold to identity rows, which add nothing, and their + // quad is left unwritten. + if covered(&ones[t], 32 * o..32 * o + 32) { + continue; + } + let rows = std::array::from_fn(|k| inputs[t].get(4 * o + k).map_or(identity, |q| fold_quad(&q, r))); + let quad = regroup(rows); + if let Some(eq) = eq { + add_quad(&mut sums, &quad, weight(eq, o)); + } + // The next pass reads it, long after this one evicted it. + // SAFETY: tasks own disjoint quads of the outputs. + unsafe { stream.copy(dst[t].slice(o, 1), std::slice::from_ref(&quad)) }; + } + sums.map(|row| row.map(Sum8::reduce)) + }); + eq.map(|_| grids) + } + + /// The first `rows` rows of `quads`, identity quads filled in. + pub(super) fn to_rows(quads: &[Quad], ones: &[Range], rows: usize) -> Vec { + let mut values = vec![F192::ONE; 128 * quads.len()]; + for (i, (quad, out)) in quads.iter().zip(values.as_chunks_mut::<128>().0).enumerate() { + if covered(ones, 32 * i..32 * i + 32) { + continue; + } + for (j, lanes) in quad.iter().enumerate() { + for (p, e) in lanes.to_array().into_iter().enumerate() { + out[16 * p + j] = e; + } + } + } + values.truncate(4 * rows); + values + } +} + +/// Send a round's `[X¹..X⁴]` messages, batched by `lambda`, and draw its challenge. +fn send(messages: [[F192; 4]; 3], lambda: F192, ps: &mut ProverState) -> F192 { + let coeffs: [F192; 4] = + std::array::from_fn(|k| messages[0][k] + lambda * (messages[1][k] + lambda * messages[2][k])); + ps.add_scalars(&coeffs); + ps.sample() +} + /// The result of a batched grand-product proof: the three leaf evaluations, all reduced to one shared point. pub struct ProductTriple { pub point: Vec, @@ -841,23 +1223,42 @@ pub(crate) fn prove_product_triple(leaves: [Leaves; 3], ps: &mut ProverState) -> } } let mut round_point = Vec::with_capacity(round_count); - // Round `j` weights its pairs by `eq(point[1 + j..], ·)`. - let mut messages = if round_count > 0 { - let eq = SplitEq::new(&point[1..]); + // Round `j` weights its pairs by `eq(point[1 + j..], ·)`. Paired, the two rounds + // from `j` weight their blocks by `eq(point[2 + j..], ·)`. + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + if parallel::num_threads() >= PAIRED_THREADS { + let mut grids = (width >= VERTICAL_ROWS).then(|| { + let inputs = trees.each_ref().map(QuaternaryLayerState::paired_input); + paired::grid(inputs, &SplitEq::new(&point[2..])) + }); + while let Some(grid) = grids { + let j = round_point.len(); + let r0 = send(grid.each_ref().map(|g| g.first_round(point[j + 1])), lambda, ps); + let r1 = send(grid.each_ref().map(|g| g.second_round(r0)), lambda, ps); + round_point.extend([r0, r1]); + let next = (width >> (j + 2) >= VERTICAL_ROWS).then(|| SplitEq::new(&point[j + 4..])); + grids = fold_pair(&mut trees, [r0, r1], next.as_ref()); + } + for tree in &mut trees { + tree.leave_quads(); + } + } + let first = round_point.len(); + let mut messages = if first < round_count { + let eq = SplitEq::new(&point[first + 1..]); trees.each_ref().map(|tree| tree.round_message(&eq)) } else { [[F192::ZERO; 4]; 3] }; - for round in 0..round_count { - let mut coeffs = [0, 1, 2, 3].map(|coefficient| { - messages[0][coefficient] + lambda * (messages[1][coefficient] + lambda * messages[2][coefficient]) - }); - // The kernel accumulates `q(0) + q(1)`, the wire carries `c1`. This is + for round in first..round_count { + // The kernels accumulate `q(0) + q(1)`, the wire carries `c1`. This is // `ProverState::add_round_poly(_, true)` bar its dropped `c0`, which the // claim fixes and the prover therefore never forms. - coeffs[0] = coeffs[0] + coeffs[1] + coeffs[2] + coeffs[3]; - ps.add_scalars(&coeffs); - let challenge = ps.sample(); + let challenge = send( + messages.map(|[sum, c2, c3, c4]| [sum + c2 + c3 + c4, c2, c3, c4]), + lambda, + ps, + ); round_point.push(challenge); if round + 1 < round_count { let eq = SplitEq::new(&point[2 + round..]); @@ -978,6 +1379,68 @@ mod tests { } } + /// The paired kernels against the scalar ones round by round, over ragged prefixes + /// and runs of identity rows. + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[test] + fn paired_rounds_match_single_rounds() { + let c1 = |[sum, c2, c3, c4]: [F192; 4]| [sum + c2 + c3 + c4, c2, c3, c4]; + for width in [1usize << 10, 1 << 13] { + let ones = [3..9, 64..200, 256..width - 24]; + let tree = |n: usize| { + let mut values: Vec = (0..n).map(|i| element(i, n)).collect(); + for row in ones.iter().flat_map(Clone::clone).filter(|&row| 4 * row < n) { + values[4 * row..(4 * row + 4).min(n)].fill(F192::ONE); + } + let mut tree = QuaternaryLayerState::new(values, width); + tree.ones = ones.to_vec(); + tree + }; + for lens in [[4 * width, 4 * width - 5, 4 * width / 3 + 7], [33, 4 * width - 1, 1000]] { + let mut reference = lens.map(tree); + let mut paired = lens.map(tree); + let point: Vec = (0..width.ilog2()).map(|i| F192::new(3 + u64::from(i), 1, 2)).collect(); + let eq = |from: usize| SplitEq::new(&point[from..]); + let mut messages = reference.each_ref().map(|tree| tree.round_message_scalar(&eq(1))); + let inputs = paired.each_ref().map(QuaternaryLayerState::paired_input); + let mut grids = Some(paired::grid(inputs, &eq(2))); + let mut j = 0; + while let Some(grid) = grids { + let r = [F192::new(j as u64 + 5, 13, 19), F192::new(j as u64 + 7, 3, 1)]; + for (g, &m) in grid.iter().zip(&messages) { + assert_eq!( + g.first_round(point[j + 1]), + c1(m), + "width={width}, lens={lens:?}, j={j}" + ); + } + messages = reference + .each_mut() + .map(|tree| tree.fold_and_message_scalar(r[0], &eq(j + 2))); + for (g, &m) in grid.iter().zip(&messages) { + assert_eq!(g.second_round(r[0]), c1(m), "width={width}, lens={lens:?}, j={j}"); + } + messages = reference + .each_mut() + .map(|tree| tree.fold_and_message_scalar(r[1], &eq(j + 3))); + let next = (width >> (j + 2) >= 32).then(|| eq(j + 4)); + grids = fold_pair(&mut paired, r, next.as_ref()); + j += 2; + } + for (p, r) in paired.iter_mut().zip(&mut reference) { + p.leave_quads(); + assert_eq!(p.round_message_scalar(&eq(j + 1)), r.round_message_scalar(&eq(j + 1))); + while r.logical_rows > 1 { + let challenge = F192::new(r.logical_rows as u64, 2, 9); + r.fold_scalar(challenge); + p.fold_scalar(challenge); + } + assert_eq!(p.children(), r.children(), "width={width}, lens={lens:?}"); + } + } + } + } + #[test] fn level_products_match_scalar() { for len in [4, 5, 63, 64, 65, 4 << 12, (4 << 12) + 3, (4 << 13) - 1] { @@ -1000,7 +1463,7 @@ mod tests { #[test] fn radix_four_claims_are_true_at_even_and_odd_depths() { - for mu in 0..=10 { + for mu in 0..=14 { let mut leaves: [Vec; 3] = [0, 1, 2].map(|lane| { (0..1usize << mu) .map(|row| F192::new((1 + row + lane * 100_003) as u64, row as u64, lane as u64)) @@ -1019,7 +1482,7 @@ mod tests { #[test] fn implicit_identity_suffix_matches_dense_padding() { - for mu in 3..=10 { + for mu in 3..=14 { let lengths = [(1usize << mu) - 3, (1usize << (mu - 1)) + 1, (1usize << (mu - 2)) + 3]; let mut leaves: [Vec; 3] = std::array::from_fn(|lane| { (0..lengths[lane]) diff --git a/crates/primitives/src/field/x8.rs b/crates/primitives/src/field/x8.rs index 62fad9d8cf..ca800a8b90 100644 --- a/crates/primitives/src/field/x8.rs +++ b/crates/primitives/src/field/x8.rs @@ -164,6 +164,17 @@ impl F192x8 { std::array::from_fn(|j| Self(std::array::from_fn(|k| blocks[(3 * j + k) / 8][(3 * j + k) % 8]))) } + /// Eight runs of sixteen consecutive elements, transposed: `out[j]` lane `p` is `src[16p + j]`. + #[inline(always)] + pub fn load_blocks(src: &[F192; 128]) -> [Self; 16] { + let q = src.as_ptr().cast::(); + // As in `load_runs`, over six blocks of each run's 48 qwords. + // SAFETY: the 128 elements are 384 readable qwords. + let blocks: [[__m512i; 8]; 6] = + std::array::from_fn(|b| transpose8(std::array::from_fn(|p| unsafe { load_u64x8(q.add(48 * p + 8 * b)) }))); + std::array::from_fn(|j| Self(std::array::from_fn(|k| blocks[(3 * j + k) / 8][(3 * j + k) % 8]))) + } + /// The even lanes of `a` then of `b`, and the odd lanes of `a` then of `b`. #[inline(always)] pub fn deinterleave(a: Self, b: Self) -> (Self, Self) { From 137dfd69f33b67954de6fa601cfa896cd8ecd3c3 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Mon, 9 Nov 2026 11:39:52 +0000 Subject: [PATCH 03/12] leanvm_core: form the products a push and a pull block share once An interaction's push and pull blocks over one table's columns flush the same coordinates at the same fingerprint weights but for one (the count's slot), so their leaf formulas share all but a term or two. build_leaves now pairs each block of a shape with the block sharing the most terms with it (`Family`): per chunk of rows the shared mixed products are summed once, unreduced, and each member adds its own terms, reduces and adds its constant. A leaf is the same XOR of the same unreduced products, so the proof is unchanged. At xmss 900 the pairing removes 64M of the 155M mixed products per proof. xmss 900, log-inv-rate 1: PROOFDIGEST c7afa387f98cdb98 unchanged. 7950X3D, loaded host, spancpu.sh median span CPU, lv-head4 -> this: default threads, 4 rounds: Bus leaves 0.500 -> 0.451 s, Bus GKR 1.139 -> 1.116 (untouched), Prove 8.92 -> 8.80 LEANVM_NUM_THREADS=1, 2 rounds: Bus leaves 0.313 -> 0.253 s Rebased onto e5d75f858 (after #724): the digests and measurements above were taken on the older base 8577d1e21. On the new base the series leaves the proofs byte-identical: PROOFDIGEST 7d91b8513bdd4b03 (aggregate --xmss 900 --log-inv-rate 1), 3c41e153f8b2ee37 (aggregate --sphincs 410 --log-inv-rate 1), aa4b2ebd7a6989b2 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). Co-Authored-By: Claude Opus 5.5 --- crates/leanvm_core/src/leaf.rs | 229 +++++++++++++++++++++++---------- 1 file changed, 160 insertions(+), 69 deletions(-) diff --git a/crates/leanvm_core/src/leaf.rs b/crates/leanvm_core/src/leaf.rs index 5c617039d5..3e23a4697c 100644 --- a/crates/leanvm_core/src/leaf.rs +++ b/crates/leanvm_core/src/leaf.rs @@ -184,6 +184,7 @@ pub fn layout(blocks: &[Block]) -> Layout { /// A non-constant coordinate as `(source, coefficient)`: its leaf contribution is /// the mixed product `coeff · source(z)` with `source(z) ∈ K`, `coeff ∈ E`. /// `GCol` folds the `g^k` factor into the coefficient. +#[derive(Clone, Copy)] enum Term<'a> { Col(usize, F192), Prod(usize, usize, F192), @@ -191,6 +192,36 @@ enum Term<'a> { Public(&'a [F64], F192), } +impl Term<'_> { + /// Whether both are one source at one coefficient. + fn same(&self, other: &Self) -> bool { + match (self, other) { + (Term::Col(i, c), Term::Col(j, d)) => (i, c) == (j, d), + (Term::Prod(i, j, c), Term::Prod(k, l, d)) => (i, j, c) == (k, l, d), + (Term::Index(c), Term::Index(d)) => c == d, + (Term::Public(u, c), Term::Public(v, d)) => std::ptr::eq(*u, *v) && c == d, + _ => false, + } + } + + /// `coeff · source(z)`, unreduced, for the index column value `g^z = index`. + #[inline(always)] + fn at(&self, z: usize, cols: &[&[F64]], index: F64) -> F192Unreduced { + match self { + Term::Col(i, c) => c.mul_base_unreduced(cols[*i][z]), + Term::Prod(i, j, c) => c.mul_base_unreduced(cols[*i][z] * cols[*j][z]), + Term::Index(c) => c.mul_base_unreduced(index), + Term::Public(vals, c) => c.mul_base_unreduced(vals[z]), + } + } + + fn coeff(&self) -> F192 { + match self { + Term::Col(_, c) | Term::Prod(_, _, c) | Term::Index(c) | Term::Public(_, c) => *c, + } + } +} + /// Flatten one coordinate into leaf terms at coefficient `w`. A [`Coord::Sum`] /// spreads its children over the SAME `w`: they are one coordinate, so they share /// its `α`-power. @@ -228,65 +259,107 @@ struct LeafBlock<'a> { terms: Vec>, } -impl LeafBlock<'_> { - /// The leaf of real row `z`, whose index column value `g^z` is `index`. - #[inline] - fn leaf(&self, z: usize, cols: &[&[F64]], index: F64) -> F192 { - // The α-weighted coordinate sum defers its reductions: each mixed - // product contributes its three raw limb products (3 PMULL, no - // reduction tail), one combined reduction per row at the end, - // bit-identical to summing reduced `mul_base` terms. - let mut acc = F192Unreduced::ZERO; - for t in &self.terms { - acc ^= match t { - Term::Col(i, c) => c.mul_base_unreduced(cols[*i][z]), - Term::Prod(i, j, c) => c.mul_base_unreduced(cols[*i][z] * cols[*j][z]), - Term::Index(c) => c.mul_base_unreduced(index), - Term::Public(vals, c) => c.mul_base_unreduced(vals[z]), - }; +/// Blocks of one shape whose formulas share terms, as an interaction's push and pull +/// blocks over one table's columns do: member `m`'s leaf is its block's `const_part` +/// plus `shared` plus its own terms, so the shared products are formed once. +struct Family<'a, 'b> { + shared: Vec>, + members: Vec<(&'b LeafBlock<'a>, Vec>)>, +} + +/// The most blocks a [`Family`] holds. +const MEMBERS: usize = 2; + +impl<'a, 'b> Family<'a, 'b> { + fn single(blk: &'b LeafBlock<'a>) -> Self { + Self { + shared: blk.terms.clone(), + members: vec![(blk, Vec::new())], + } + } + + /// `blocks` (of one shape) as families, each block paired with the one sharing the + /// most terms with it, if any. + fn of(blocks: &[&'b LeafBlock<'a>]) -> Vec { + // The terms of `a` and `b` they share, then `a`'s and `b`'s others. + let split = |a: &[Term<'a>], b: &[Term<'a>]| { + let (mut shared, mut own_a, mut own_b) = (Vec::new(), Vec::new(), b.to_vec()); + for t in a { + match own_b.iter().position(|u| t.same(u)) { + Some(i) => shared.push(own_b.swap_remove(i)), + None => own_a.push(*t), + } + } + (shared, own_a, own_b) + }; + let mut left = blocks.to_vec(); + let mut families = Vec::new(); + while !left.is_empty() { + let a = left.remove(0); + let best = (0..left.len()).max_by_key(|&i| split(&a.terms, &left[i].terms).0.len()); + match best { + Some(i) if !split(&a.terms, &left[i].terms).0.is_empty() => { + let b = left.remove(i); + let (shared, own_a, own_b) = split(&a.terms, &b.terms); + families.push(Self { + shared, + members: vec![(a, own_a), (b, own_b)], + }); + } + _ => families.push(Self::single(a)), + } } - self.const_part + acc.reduce() + families } - /// The leaves of rows `z0..z0 + out.len()`. - fn fill(&self, z0: usize, out: &mut [F192], cols: &[&[F64]]) { - let real = self.rows.saturating_sub(z0).min(out.len()); - let (real_out, padding) = out.split_at_mut(real); - // A padding row's leaf is the identity, so it leaves the product alone. - padding.fill(F192::ONE); + /// Every member's leaves of rows `z0..z0 + len`, into `outs[m]`. + fn fill(&self, z0: usize, outs: &mut [&mut [F192]], cols: &[&[F64]]) { + debug_assert_eq!(outs.len(), self.members.len()); + let len = outs[0].len(); + let real = self.members[0].0.rows.saturating_sub(z0).min(len); + for out in outs.iter_mut() { + // A padding row's leaf is the identity, so it leaves the product alone. + out[real..].fill(F192::ONE); + } #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] - let done = self.fill_lanes(z0, real_out, cols); + let done = self.fill_lanes(z0, real, outs, cols); #[cfg(not(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq")))] let done = 0; + // Each mixed product contributes its raw limb products, and a leaf is reduced + // once, bit-identical to summing reduced `mul_base` terms. + let sum = |terms: &[Term], z: usize, index: F64| { + terms + .iter() + .fold(F192Unreduced::ZERO, |acc, t| acc ^ t.at(z, cols, index)) + }; let mut index = g_pow(z0 + done); - for (i, slot) in real_out.iter_mut().enumerate().skip(done) { - *slot = self.leaf(z0 + i, cols, index); + for i in done..real { + let shared = sum(&self.shared, z0 + i, index); + for ((blk, own), out) in self.members.iter().zip(outs.iter_mut()) { + out[i] = blk.const_part + (shared ^ sum(own, z0 + i, index)).reduce(); + } index = mul_by_g(index); } } - /// [`fill`](Self::fill) [`lanes::N`] rows at a time, every term one lane-wise mixed - /// product; returns how many rows it wrote. + /// [`fill`](Self::fill) over the first `real` rows, [`lanes::N`] at a time, every term one + /// lane-wise mixed product; returns how many rows it wrote. #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] - fn fill_lanes(&self, z0: usize, out: &mut [F192], cols: &[&[F64]]) -> usize { + fn fill_lanes(&self, z0: usize, real: usize, outs: &mut [&mut [F192]], cols: &[&[F64]]) -> usize { use lanes::{E, K, Mixed, N}; let load = |source: &[F64], z: usize| K::load(source[z..z + N].try_into().unwrap()); // `g^z` for the `N` rows, stepped by `g^N`. let mut index = K::new(std::array::from_fn(|i| g_pow(z0 + i))); let step = K::splat(g_pow(N)); - let coeffs: Vec = self - .terms + let coeffs = |terms: &[Term]| -> Vec { terms.iter().map(|t| E::splat(t.coeff())).collect() }; + let shared = coeffs(&self.shared); + let own: Vec<(E, Vec)> = self + .members .iter() - .map(|t| match t { - Term::Col(_, c) | Term::Prod(_, _, c) | Term::Index(c) | Term::Public(_, c) => E::splat(*c), - }) + .map(|(blk, terms)| (E::splat(blk.const_part), coeffs(terms))) .collect(); - let constant = E::splat(self.const_part); - let (chunks, _) = out.as_chunks_mut::(); - for (k, slot) in chunks.iter_mut().enumerate() { - let z = z0 + N * k; - let mut acc = Mixed::ZERO; - for (t, &c) in self.terms.iter().zip(&coeffs) { + let add = |acc: &mut Mixed, terms: &[Term], coeffs: &[E], z: usize, index: K| { + for (t, &c) in terms.iter().zip(coeffs) { match t { Term::Col(i, _) => acc.add(c, load(cols[*i], z)), Term::Prod(i, j, _) => acc.add(c, load(cols[*i], z) * load(cols[*j], z)), @@ -294,14 +367,23 @@ impl LeafBlock<'_> { Term::Public(vals, _) => acc.add(c, load(vals, z)), } } + }; + for k in 0..real / N { + let z = z0 + N * k; + let mut acc = Mixed::ZERO; + add(&mut acc, &self.shared, &shared, z, index); + for (((_, terms), (constant, c)), out) in self.members.iter().zip(&own).zip(outs.iter_mut()) { + let mut acc = acc; + add(&mut acc, terms, c, z, index); + (acc.reduce() + *constant).store((&mut out[z - z0..z - z0 + N]).try_into().unwrap()); + } index = index * step; - (acc.reduce() + constant).store(slot); } - N * chunks.len() + N * (real / N) } } -/// The lane-wise kernels [`LeafBlock::fill_lanes`] runs on: eight lanes with AVX-512 +/// The lane-wise kernels [`Family::fill_lanes`] runs on: eight lanes with AVX-512 /// ([`primitives::field::x8`]), four without ([`primitives::field::x4`]). #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] mod lanes { @@ -393,9 +475,12 @@ pub(crate) fn build_leaves(sides: [(&[Block], &Layout, &[F192], F192); 3], cols: .collect(); shapes.sort_unstable(); shapes.dedup(); - let groups: Vec> = shapes + let groups: Vec> = shapes .iter() - .map(|&shape| blocks.iter().filter(|b| (b.kappa, b.rows) == shape).collect()) + .map(|&shape| { + let group: Vec<&LeafBlock> = blocks.iter().filter(|b| (b.kappa, b.rows) == shape).collect(); + Family::of(&group) + }) .collect(); let tasks: Vec<(usize, Range)> = shapes .iter() @@ -404,7 +489,8 @@ pub(crate) fn build_leaves(sides: [(&[Block], &Layout, &[F192], F192); 3], cols: .flat_map(|(g, (&(kappa, _), group))| { let height = 1usize << kappa; // At least one sixteen per task, or its `sixteens` would go unwritten. - let rows = (TASK_LEAVES / group.len()).next_power_of_two().clamp(16, height); + let blocks: usize = group.iter().map(|family| family.members.len()).sum(); + let rows = (TASK_LEAVES / blocks).next_power_of_two().clamp(16, height); (0..height).step_by(rows).map(move |z| (g, z..z + rows)) }) .collect(); @@ -412,34 +498,39 @@ pub(crate) fn build_leaves(sides: [(&[Block], &Layout, &[F192], F192); 3], cols: .map(|level| level.each_mut().map(|v| parallel::SendPtr(v.as_mut_ptr()))); parallel::for_each(tasks.len(), |task| { let (group, ref rows) = tasks[task]; - let mut stage = [F192::ZERO; CHUNK]; + let mut stages = [[F192::ZERO; CHUNK]; MEMBERS]; let mut stage4 = [F192::ZERO; CHUNK / 4]; let mut stage16 = [F192::ZERO; CHUNK / 16]; // The GKR reads all three long after this pass has evicted them. let stream = primitives::stream::Stream::new(); for z0 in rows.clone().step_by(CHUNK) { let len = CHUNK.min(rows.len()); - let (leaves, fours, sixteens) = (&mut stage[..len], &mut stage4[..len / 4], &mut stage16[..len / 16]); - for blk in &groups[group] { - blk.fill(z0, leaves, cols); - // Past the block's real rows every leaf, and so every tuple's product, is the identity. - let live = if gkr::SKIP_IDENTITY { - blk.rows.saturating_sub(z0).next_multiple_of(16).min(len) - } else { - len - }; - gkr::products_into(&leaves[..live], &mut fours[..live / 4]); - gkr::products_into(&fours[..live / 4], &mut sixteens[..live / 16]); - fours[live / 4..].fill(F192::ONE); - sixteens[live / 16..].fill(F192::ONE); - let at = blk.offset + z0; - // SAFETY: a block's chunks are disjoint ranges of its side's leaves, blocks - // are disjoint, and a chunk owns the tuples it starts, `at` being a multiple - // of sixteen. - unsafe { - stream.copy(ptrs[0][blk.side].slice(at, len), leaves); - stream.copy(ptrs[1][blk.side].slice(at / 4, len / 4), fours); - stream.copy(ptrs[2][blk.side].slice(at / 16, len / 16), sixteens); + let (fours, sixteens) = (&mut stage4[..len / 4], &mut stage16[..len / 16]); + for family in &groups[group] { + let [first, second] = &mut stages; + let n = family.members.len(); + family.fill(z0, &mut [&mut first[..len], &mut second[..len]][..n], cols); + for ((blk, _), stage) in family.members.iter().zip(&stages) { + let leaves = &stage[..len]; + // Past the block's real rows every leaf, and so every tuple's product, is the identity. + let live = if gkr::SKIP_IDENTITY { + blk.rows.saturating_sub(z0).next_multiple_of(16).min(len) + } else { + len + }; + gkr::products_into(&leaves[..live], &mut fours[..live / 4]); + gkr::products_into(&fours[..live / 4], &mut sixteens[..live / 16]); + fours[live / 4..].fill(F192::ONE); + sixteens[live / 16..].fill(F192::ONE); + let at = blk.offset + z0; + // SAFETY: a block's chunks are disjoint ranges of its side's leaves, blocks + // are disjoint, and a chunk owns the tuples it starts, `at` being a multiple + // of sixteen. + unsafe { + stream.copy(ptrs[0][blk.side].slice(at, len), leaves); + stream.copy(ptrs[1][blk.side].slice(at / 4, len / 4), fours); + stream.copy(ptrs[2][blk.side].slice(at / 16, len / 16), sixteens); + } } } } @@ -448,7 +539,7 @@ pub(crate) fn build_leaves(sides: [(&[Block], &Layout, &[F192], F192); 3], cols: for (side, v) in values.iter_mut().enumerate() { let mut tail = lens[side]; for blk in blocks.iter().filter(|b| b.side == side && b.kappa < CHUNKED) { - blk.fill(0, &mut v[blk.offset..blk.offset + (1 << blk.kappa)], cols); + Family::single(blk).fill(0, &mut [&mut v[blk.offset..blk.offset + (1 << blk.kappa)]], cols); tail = tail.min(blk.offset); } if sides[side].0.is_empty() { From 48cea407b9f6968a7be8a9e9ba4dfb994bb4c9a9 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Mon, 9 Nov 2026 10:43:25 +0000 Subject: [PATCH 04/12] keccak_gkr: skip sums by pivots and GFNI-weighted packs The AVX-512 SkipBlock weighed every product of a row by its position's weight: per mixed point and eight slices, two carry-less products for b * c and six for the weight, five times a row (40 VPCLMULQDQ). A row's five products now go by three pivots, each a factor its products share: e2 (w0 e1 + w1 e3), e4 (w2 e3 + w3 e0), e1 (w4 e0). A weight times an embedding is linear in the pack's bits, so per-position affine byte maps (SkipWeights, built once a layer, 384 bytes a position, laid out so eight positions' maps are one contiguous register) give the weighted packs already reduced to three words; the extension is a sum of those too, and a mixed point then costs three products per pivot (18 VPCLMULQDQ). A half block runs at a time, the extension is formed a high point at a time from the 16 packs, and each output's three word sums stay in registers over the pivots, so per eight slices only 12 quantities of 16 registers are stored. The sums are a polynomial congruent to the old ones modulo the minimal polynomial, so the prover's evaluation and the proof bytes are unchanged (PROOFDIGEST f090f5e4ee18ffad). The other arms keep the old kernels behind the same SkipWeights/SkipBlock interface. Profile (head): SkipBlock was CLMUL-port bound, 208k zmm VPCLMULQDQ per block at ~2.2 cycles each (measured on this host); the new kernel is 94k per block, plus 38k GF2P8AFFINEQB that overlap the multiplier. SkipBlock::run alone, one thread, perf cycles per block (5 runs each, host loaded): head 707-747k, new 463-555k, about -33%. Per-kernel perf cycle samples (aggregate --xmss 900 rate 1 --repeat 2, two runs each, interleaved): SkipBlock head 19.22G / 19.34G, new 14.66G / 16.00G plus SkipWeights 0.18G / 0.16G, about -20%. hash --log-n 17 --repeat 3: head 18.98G / 19.54G, new 15.87G / 15.86G. Whole pass (ab.sh 3x4, load average 100-370): head cpu min 11.93 med 12.24 s, new min 11.95 med 12.34 s, inside the noise (expected about -2%). Rebased onto e5d75f858 (after #724): the digests and measurements above were taken on the older base 8577d1e21. On the new base the series leaves the proofs byte-identical: PROOFDIGEST 7d91b8513bdd4b03 (aggregate --xmss 900 --log-inv-rate 1), 3c41e153f8b2ee37 (aggregate --sphincs 410 --log-inv-rate 1), aa4b2ebd7a6989b2 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). --- crates/keccak_gkr/src/kernels.rs | 407 ++++++++++++++++++++++++------- crates/keccak_gkr/src/prover.rs | 4 +- 2 files changed, 322 insertions(+), 89 deletions(-) diff --git a/crates/keccak_gkr/src/kernels.rs b/crates/keccak_gkr/src/kernels.rs index 79bf3fdb48..c2c2c5586e 100644 --- a/crates/keccak_gkr/src/kernels.rs +++ b/crates/keccak_gkr/src/kernels.rs @@ -5,9 +5,11 @@ //! (PCLMULQDQ, and VPCLMULQDQ where it is) and a NEON one (PMULL), which agree bit for bit. The //! vector forms are compiled only where their features are. +use primitives::field::F192; use primitives::field::gf2_64::mul_wide; use crate::keccak::row_positions; +use crate::power::{Element, PowerField}; use crate::rmfe::RMFE; use crate::witness::BLOCK_WORDS; @@ -258,9 +260,18 @@ cfg_select! { target_feature = "vpclmulqdq", target_feature = "gfni" ) => { - pub(crate) use x86::SkipBlock; + pub(crate) use x86::{SkipBlock, SkipWeights}; } _ => { + /// The position weights a block's [`SkipBlock::run`] reads, in the power basis. + pub(crate) struct SkipWeights(Vec); + + impl SkipWeights { + pub fn new(field: &PowerField, weights: &[F192]) -> Self { + Self(weights.iter().map(|&w| field.to_poly(w)).collect()) + } + } + /// [`skip_row`] over every row and slice of a block, a position at a time. pub(crate) struct SkipBlock { acc: Box, @@ -276,7 +287,7 @@ cfg_select! { } /// Sum a block, the position weights as elements. - pub fn run(&mut self, block: &[u64], weights: &[[u64; 3]]) { + pub fn run(&mut self, block: &[u64], weights: &SkipWeights) { self.acc.clear(); for r in 0..5 { for z in 0..64 { @@ -284,12 +295,13 @@ cfg_select! { for x in 0..5 { embed_extend(crate::witness::words(block, js[x]), &mut self.extended[x]); } - skip_row(&mut self.acc, &self.extended, std::array::from_fn(|x| &weights[js[x]])); + skip_row(&mut self.acc, &self.extended, std::array::from_fn(|x| &weights.0[js[x]])); } } } - /// The three partial products of slot `i`. + /// The three partial products of slot `i`, a polynomial congruent to the slot's sum + /// modulo the minimal polynomial. pub fn get(&self, i: usize) -> [u128; 3] { self.acc.get(i) } @@ -417,11 +429,10 @@ mod x86 { } } - /// The packs of eight consecutive positions, embedded: `out[q]` holds pack `q` of position - /// `i` in word `i`. The positions' words are transposed first, so a register holds one word - /// of each, and a pack is then a field of every word alike. + /// The words of eight consecutive positions, transposed: register `w` holds word `w` of + /// position `i` in its word `i`, so a pack is a field of every word alike. #[inline(always)] - unsafe fn embed8(positions: *const u64, out: &mut [__m512i; PACKS]) { + unsafe fn transpose_words(positions: *const u64) -> [__m512i; BLOCK_WORDS] { unsafe { let a: [__m512i; 8] = std::array::from_fn(|i| _mm512_loadu_si512(positions.add(BLOCK_WORDS * i).cast())); let b: [__m512i; 8] = std::array::from_fn(|i| { @@ -441,7 +452,7 @@ mod x86 { let (x, y) = (b[4 * m + t % 2], b[4 * m + 2 + t % 2]); if t < 2 { low(x, y) } else { high(x, y) } }); - let words = [ + [ low(c[0], c[4]), low(c[1], c[5]), low(c[2], c[6]), @@ -450,7 +461,15 @@ mod x86 { high(c[1], c[5]), high(c[2], c[6]), high(c[3], c[7]), - ]; + ] + } + } + + /// The packs of some of [`transpose_words`]'s words, embedded: `out[4 w + t]` holds pack + /// `t` of `words[w]`, of position `i` in word `i`. + #[inline(always)] + unsafe fn embed_words(words: &[__m512i], out: &mut [__m512i]) { + unsafe { let zero = _mm512_setzero_si512(); for (w, &v) in words.iter().enumerate() { let low = _mm512_or_si512(embedding_byte(v, 0), _mm512_slli_epi16(embedding_byte(v, 1), 8)); @@ -466,110 +485,321 @@ mod x86 { } } - /// [`extend_group`] on registers, each of eight positions' values. + /// The bytes of the eight packs of two of [`transpose_words`]'s words: `out[h]` holds byte + /// `h` of pack `k` of position `i` in byte `k` of word `i`. #[inline(always)] - unsafe fn extend8(v: &[__m512i], out: &mut [__m512i; MIXED]) { + unsafe fn pack_bytes(words: [__m512i; 2]) -> [__m512i; 2] { unsafe { - let x = |a: [__m512i; 4], b: [__m512i; 4]| -> [__m512i; 4] { - std::array::from_fn(|l| _mm512_xor_si512(a[l], b[l])) - }; - let q = |k: usize| -> [__m512i; 4] { std::array::from_fn(|l| v[4 * k + l]) }; - let (w4, w5) = (x(q(0), q(1)), x(q(2), q(3))); - let w = [q(0), q(1), q(2), q(3), w4, w5, x(q(0), q(2)), x(q(1), q(3)), x(w4, w5)]; - for (k, w) in w[4..].iter().enumerate() { - out[4 * k..][..4].copy_from_slice(w); + // Per word, its four low bytes then its four high ones, and the other way round. + let split = _mm512_broadcast_i32x4(_mm_setr_epi8(0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15)); + let swapped = _mm512_broadcast_i32x4(_mm_setr_epi8(1, 3, 5, 7, 0, 2, 4, 6, 9, 11, 13, 15, 8, 10, 12, 14)); + let a = _mm512_shuffle_epi8(words[0], split); + let b = _mm512_shuffle_epi8(words[1], swapped); + let low = _mm512_mask_blend_epi32(0xaaaa, a, b); + let high = _mm512_mask_blend_epi32(0xaaaa, b, a); + [low, _mm512_shuffle_epi32::<0b10_11_00_01>(high)] + } + } + + /// Per word, an 8x8 transpose of bytes across the registers: `out[k]` byte `u` of word `i` is + /// `r[u]` byte `k` of word `i`. + #[inline(always)] + unsafe fn transpose_bytes(r: [__m512i; 8]) -> [__m512i; 8] { + unsafe { + // A 128-bit lane `m` holds word `2 m + parity` of each: `a[2 u + parity]` 16-bit field + // `k` is bytes `2 u, 2 u + 1` of `k`. + let a: [__m512i; 8] = std::array::from_fn(|i| { + let (x, y) = (r[2 * (i / 2)], r[2 * (i / 2) + 1]); + if i % 2 == 0 { + _mm512_unpacklo_epi8(x, y) + } else { + _mm512_unpackhi_epi8(x, y) + } + }); + // `b[4 v + 2 h + parity]` 32-bit field `k` is bytes `4 v ..` of `4 h + k`. + let b: [__m512i; 8] = std::array::from_fn(|i| { + let (v, h, parity) = (i / 4, i / 2 % 2, i % 2); + let (x, y) = (a[4 * v + parity], a[4 * v + 2 + parity]); + if h == 0 { + _mm512_unpacklo_epi16(x, y) + } else { + _mm512_unpackhi_epi16(x, y) + } + }); + // `c[4 h + 2 p + parity]` 64-bit field `t` is all of `4 h + 2 p + t`. + let c: [__m512i; 8] = std::array::from_fn(|i| { + let (h, p, parity) = (i / 4, i / 2 % 2, i % 2); + let (x, y) = (b[2 * h + parity], b[4 + 2 * h + parity]); + if p == 0 { + _mm512_unpacklo_epi32(x, y) + } else { + _mm512_unpackhi_epi32(x, y) + } + }); + std::array::from_fn(|k| { + let (x, y) = (c[k & !1], c[k | 1]); + if k % 2 == 0 { + _mm512_unpacklo_epi64(x, y) + } else { + _mm512_unpackhi_epi64(x, y) + } + }) + } + } + + /// Bytes of an element. + const ELEMENT_BYTES: usize = 24; + /// The matrices of a position. + const WEIGHT_MATRICES: usize = 2 * ELEMENT_BYTES; + + /// Per position, the affine byte maps sending byte `h` of a pack to byte `b` of the + /// position's weight times the pack's embedding, reduced modulo the minimal polynomial: + /// matrix `m = ELEMENT_BYTES * h + b` of position `j` at + /// `WEIGHT_MATRICES * 8 * (j / 8) + 8 * m + j % 8`, so that eight consecutive positions' are + /// one register, and theirs are contiguous. + pub struct SkipWeights(Vec); + + impl SkipWeights { + pub fn new(field: &PowerField, weights: &[F192]) -> Self { + assert!(weights.len() >= BITS); + let per_position = parallel::map_collect(BITS, |j| { + let w = field.to_poly(weights[j]); + let v: [Element; 16] = std::array::from_fn(|t| field.mul(&w, &[RMFE.emb[t] as u64, 0, 0])); + std::array::from_fn::(|m| { + let (h, b) = (m / ELEMENT_BYTES, m % ELEMENT_BYTES); + // Byte `i` is byte `b` of input bit `i`'s image; an affine byte map reads + // output bit `u` off matrix byte `7 - u`. + let images = (0..8).fold(0u64, |q, i| { + q | (v[8 * h + i][b / 8] >> (8 * (b % 8)) & 0xff) << (8 * i) + }); + transpose_bits8x8(images).swap_bytes() + }) + }); + let mut matrices = vec![0u64; WEIGHT_MATRICES * BITS]; + for (j, m) in per_position.iter().enumerate() { + for (k, &m) in m.iter().enumerate() { + matrices[WEIGHT_MATRICES * 8 * (j / 8) + 8 * k + j % 8] = m; + } } - for (k, w) in w.iter().enumerate() { - let low = x([w[0], w[2], w[0], w[1]], [w[1], w[3], w[2], w[3]]); - out[20 + 4 * k..][..4].copy_from_slice(&low); - out[56 + k] = _mm512_xor_si512(low[0], low[1]); + Self(matrices) + } + } + + /// Bit `8 c + r` of `x` at bit `8 r + c`. + const fn transpose_bits8x8(mut x: u64) -> u64 { + let t = (x ^ (x >> 7)) & 0x00aa_00aa_00aa_00aa; + x ^= t ^ (t << 7); + let t = (x ^ (x >> 14)) & 0x0000_cccc_0000_cccc; + x ^= t ^ (t << 14); + let t = (x ^ (x >> 28)) & 0x0000_0000_f0f0_f0f0; + x ^ t ^ (t << 28) + } + + /// One octet of a pivot's weighted packs into `out[k][8 o ..]`, per word `k`: `sum over terms + /// of weights[x] * embed(pack)`, a term being position `x`'s matrices and the bytes of the + /// other factor's packs ([`pack_bytes`]). + #[inline(always)] + unsafe fn weigh(terms: [(*const u64, [__m512i; 2]); N], out: &mut [[__m512i; GROUP]], o: usize) { + unsafe { + for (k, out) in out[..3].iter_mut().enumerate() { + let image = transpose_bytes(std::array::from_fn(|u| { + terms.iter().fold(_mm512_setzero_si512(), |g, &(m, [low, high])| { + let matrix = |h: usize| _mm512_loadu_si512(m.add(8 * (ELEMENT_BYTES * h + 8 * k + u)).cast()); + _mm512_ternarylogic_epi64( + g, + _mm512_gf2p8affine_epi64_epi8(low, matrix(0), 0), + _mm512_gf2p8affine_epi64_epi8(high, matrix(1), 0), + 0x96, + ) + }) + })); + for (out, image) in out[8 * o..][..8].iter_mut().zip(image) { + *out = image; + } } } } + /// The quantities of a half block, by pivot: its packs embedded, then each word of its + /// weighted packs, over the group's 16 packs. + type Quantities = [[__m512i; GROUP]; 12]; + + /// A quantity's value at high point `K` of [`extend_group`] and low lane `l`. + #[inline(always)] + unsafe fn high(q: &[__m512i; GROUP], l: usize) -> __m512i { + unsafe { + let (g, xor) = (|m: usize| q[4 * m + l], _mm512_xor_si512); + match K { + 0..4 => g(K), + 4 => xor(g(0), g(1)), + 5 => xor(g(2), g(3)), + 6 => xor(g(0), g(2)), + 7 => xor(g(1), g(3)), + _ => _mm512_ternarylogic_epi64(xor(g(0), g(1)), g(2), g(3), 0x96), + } + } + } + + /// Shapes of [`values`]: of a high point's lanes `w`, `w0 + w1, w2 + w3` and their sum; `w0, + /// w1` and theirs; `w2, w3` and theirs; `w0 + w2, w1 + w3` and theirs. + const LOW_SUMS: usize = 0; + const FIRST: usize = 1; + const SECOND: usize = 2; + const CROSS: usize = 3; + + /// A quantity's values at `S` outputs of high point `K`, of the given shape. + #[inline(always)] + unsafe fn values(q: &[__m512i; GROUP]) -> [__m512i; S] { + unsafe { + let w = |l: usize| high::(q, l); + let xor = _mm512_xor_si512; + let (a, b) = match SHAPE { + LOW_SUMS => (xor(w(0), w(1)), xor(w(2), w(3))), + FIRST => (w(0), w(1)), + SECOND => (w(2), w(3)), + _ => (xor(w(0), w(2)), xor(w(1), w(3))), + }; + let v = [a, b, xor(a, b)]; + std::array::from_fn(|s| v[s]) + } + } + /// [`skip_row`] over every row and slice of a block, eight slices at a time: each register /// holds one value of eight positions, so the extension is a sum of registers and each slot's - /// products are a register's lanes, weighted by eight positions' weights at once. + /// products are a register's lanes. + /// + /// A row's five products `weights[x] * e[x + 1] * e[x + 2]` go by three pivots, each a + /// factor its products share: `e[2] * (weights[0] * e[1] + weights[1] * e[3])`, + /// `e[4] * (weights[2] * e[3] + weights[3] * e[0])` and `e[1] * (weights[4] * e[0])`. A + /// weight times an embedding is linear in the pack's bits, so [`SkipWeights`]' affine byte + /// maps give the weighted packs, reduced to three words, and the extension is a sum of those + /// too. A slot then takes three carry-less products per pivot, where weighing each product + /// would take four per row position. + /// + /// A half block at a time, so that its slots' sums stay in cache. The extension is built a + /// high point at a time from the group's values, and each word's sums of a high point's + /// slots stay in registers over the pivots. pub struct SkipBlock { - /// Per slot, the even and odd positions' sums of each partial product. - acc: Vec<[__m512i; 6]>, - packs: [[__m512i; PACKS]; 5], - extended: [[__m512i; MIXED]; 5], - /// The weights by word: word `k` of position `j` at `k * BITS + j`. - weights: Vec, + /// Per slot, the sums of each word's products. + acc: Vec<[__m512i; 3]>, } impl SkipBlock { pub fn new() -> Box { let zero = unsafe { _mm512_setzero_si512() }; Box::new(Self { - acc: vec![[zero; 6]; 2 * MIXED], - packs: [[zero; PACKS]; 5], - extended: [[zero; MIXED]; 5], - weights: vec![0; 3 * BITS], + acc: vec![[zero; 3]; 2 * MIXED], }) } - /// Sum a block, the position weights as elements. - pub fn run(&mut self, block: &[u64], weights: &[[u64; 3]]) { - assert!(block.len() >= BITS * BLOCK_WORDS && weights.len() >= BITS); - for (j, w) in weights[..BITS].iter().enumerate() { - for (k, &w) in w.iter().enumerate() { - self.weights[k * BITS + j] = w; - } - } + /// Sum a block. + pub fn run(&mut self, block: &[u64], weights: &SkipWeights) { + assert!(block.len() >= BITS * BLOCK_WORDS && weights.0.len() >= WEIGHT_MATRICES * BITS); // SAFETY: the features are enabled at compile time, and every position read is in - // the block, every weight in the table. + // the block, every matrix in the table. unsafe { - self.acc.fill([_mm512_setzero_si512(); 6]); - for r in 0..5 { - for z in (0..64).step_by(8) { - let js = row_positions(r, z); - for (packs, &j) in self.packs.iter_mut().zip(&js) { - embed8(block.as_ptr().add(BLOCK_WORDS * j), packs); - } - let w: [[__m512i; 3]; 5] = std::array::from_fn(|x| { - std::array::from_fn(|k| { - _mm512_loadu_si512(self.weights.as_ptr().add(k * BITS + js[x]).cast()) - }) - }); - for half in 0..2 { - for (extended, packs) in self.extended.iter_mut().zip(&self.packs) { - extend8(&packs[GROUP * half..][..GROUP], extended); + let zero = _mm512_setzero_si512(); + self.acc.fill([zero; 3]); + let mut q: Quantities = [[zero; GROUP]; 12]; + for half in 0..2 { + for r in 0..5 { + for z in (0..64).step_by(8) { + let js = row_positions(r, z); + let words: [[__m512i; GROUP / 4]; 5] = std::array::from_fn(|x| { + let words = transpose_words(block.as_ptr().add(BLOCK_WORDS * js[x])); + std::array::from_fn(|w| words[GROUP / 4 * half + w]) + }); + let m = |x: usize| weights.0.as_ptr().add(WEIGHT_MATRICES * js[x]); + for o in 0..2 { + let bytes = |x: usize| pack_bytes([words[x][2 * o], words[x][2 * o + 1]]); + let (b0, b1, b3) = (bytes(0), bytes(1), bytes(3)); + weigh([(m(0), b1), (m(1), b3)], &mut q[1..4], o); + weigh([(m(2), b3), (m(3), b0)], &mut q[5..8], o); + weigh([(m(4), b0)], &mut q[9..12], o); } - for i in 0..MIXED { - let e: [__m512i; 5] = std::array::from_fn(|x| self.extended[x][i]); - let mut sums = [_mm512_setzero_si512(); 6]; - for x in 0..5 { - let (b, c) = (e[(x + 1) % 5], e[(x + 2) % 5]); - let even = _mm512_clmulepi64_epi128(b, c, 0x00); - let odd = _mm512_clmulepi64_epi128(b, c, 0x11); - for k in 0..3 { - let (s, w) = (&mut sums[2 * k..], w[x][k]); - s[0] = _mm512_xor_si512(s[0], _mm512_clmulepi64_epi128(even, w, 0x00)); - s[1] = _mm512_xor_si512(s[1], _mm512_clmulepi64_epi128(odd, w, 0x10)); - } - } - let acc = &mut self.acc[slot(half, i)]; - for (acc, sum) in acc.iter_mut().zip(sums) { - *acc = _mm512_xor_si512(*acc, sum); - } + for (p, pivot) in [2, 4, 1].into_iter().enumerate() { + embed_words(&words[pivot], &mut q[4 * p]); } + self.multiply(half, &q); } } } } } - /// The three partial products of slot `i`. + /// Every output of [`extend_group`] of the quantities: its products into its slot. + #[inline(always)] + unsafe fn multiply(&mut self, half: usize, q: &Quantities) { + unsafe { + self.high_point::<0>(half, q); + self.high_point::<1>(half, q); + self.high_point::<2>(half, q); + self.high_point::<3>(half, q); + self.high_point::<4>(half, q); + self.high_point::<5>(half, q); + self.high_point::<6>(half, q); + self.high_point::<7>(half, q); + self.high_point::<8>(half, q); + } + } + + /// The outputs of high point `K`: the binary low points of a new high point, its low + /// mixed points, then their lane sum. + #[inline(always)] + unsafe fn high_point(&mut self, half: usize, q: &Quantities) { + unsafe { + let (lows, sum) = ([20 + 4 * K, 21 + 4 * K, 22 + 4 * K, 23 + 4 * K], 56 + K); + if K < 4 { + self.add_products::(half, [lows[0], lows[1], sum], q); + self.add_products::(half, [lows[2], lows[3]], q); + } else { + let binary = 4 * (K - 4); + self.add_products::(half, [binary, binary + 1, lows[0]], q); + self.add_products::(half, [binary + 2, binary + 3, lows[1]], q); + self.add_products::(half, [lows[2], lows[3], sum], q); + } + } + } + + /// For `S` outputs of high point `K`, given by `SHAPE`, every pivot's products into + /// their slots. + #[inline(always)] + unsafe fn add_products( + &mut self, + half: usize, + outputs: [usize; S], + q: &Quantities, + ) { + unsafe { + let mut acc: [[__m512i; 3]; S] = std::array::from_fn(|s| self.acc[slot(half, outputs[s])]); + for p in 0..3 { + let e = values::(&q[4 * p]); + for k in 0..3 { + let l = values::(&q[4 * p + 1 + k]); + for s in 0..S { + acc[s][k] = _mm512_ternarylogic_epi64( + acc[s][k], + _mm512_clmulepi64_epi128(l[s], e[s], 0x00), + _mm512_clmulepi64_epi128(l[s], e[s], 0x11), + 0x96, + ); + } + } + } + for (s, acc) in acc.into_iter().enumerate() { + self.acc[slot(half, outputs[s])] = acc; + } + } + } + + /// The three partial products of slot `i`, a polynomial congruent to the slot's sum + /// modulo the minimal polynomial. pub fn get(&self, i: usize) -> [u128; 3] { std::array::from_fn(|k| { - let mut lanes = [[0u128; 4]; 2]; - for (lanes, &acc) in lanes.iter_mut().zip(&self.acc[i][2 * k..2 * k + 2]) { - // SAFETY: four 128-bit lanes to a register. - unsafe { _mm512_storeu_si512(lanes.as_mut_ptr().cast(), acc) }; - } - lanes.as_flattened().iter().fold(0, |a, &b| a ^ b) + let mut lanes = [0u128; 4]; + // SAFETY: four 128-bit lanes to a register. + unsafe { _mm512_storeu_si512(lanes.as_mut_ptr().cast(), self.acc[i][k]) }; + lanes.iter().fold(0, |a, &b| a ^ b) }) } } @@ -1054,12 +1284,15 @@ mod tests { assert_eq!(fold_groups(&e, &fold), fold_groups_portable(&e, &fold)); } let block: Vec = (0..crate::witness::BLOCK_LEN).map(|_| rng.next_u64()).collect(); - let weights: Vec<[u64; 3]> = (0..BITS).map(|_| std::array::from_fn(|_| rng.next_u64())).collect(); + let field = PowerField::new(rng.ext()).unwrap(); + let weights = rng.ext_vec(BITS); let mut skip = SkipBlock::new(); - skip.run(&block, &weights); - let expected = skip_block_portable(&block, &weights); + skip.run(&block, &SkipWeights::new(&field, &weights)); + let polys: Vec = weights.iter().map(|&w| field.to_poly(w)).collect(); + let expected = skip_block_portable(&block, &polys); + let reduce = |sum: &[u128; 3]| field.reduce(&crate::basis::carry::<3, 6>(sum)); for (i, expected) in expected.iter().enumerate() { - assert_eq!(&skip.get(i), expected, "slot {i}"); + assert_eq!(reduce(&skip.get(i)), reduce(expected), "slot {i}"); } } } diff --git a/crates/keccak_gkr/src/prover.rs b/crates/keccak_gkr/src/prover.rs index 41b59a0324..0401f0a589 100644 --- a/crates/keccak_gkr/src/prover.rs +++ b/crates/keccak_gkr/src/prover.rs @@ -15,7 +15,7 @@ use primitives::multilinear::{eq_table, poly_eval}; use crate::basis::{PolyEval, add_mul, carry}; use crate::keccak::{BITS, LOCAL_VARS, along, linear_transpose, row_positions}; -use crate::kernels::{GROUP, MIXED, PACKS, SKIP, SkipBlock, TERNARY, embed, mixed_points, products, slot}; +use crate::kernels::{GROUP, MIXED, PACKS, SKIP, SkipBlock, SkipWeights, TERNARY, embed, mixed_points, products, slot}; use crate::open::fold; use crate::power::{Element, PowerField, get, set}; use crate::protocol::{Claim, chi_weights, mu_at}; @@ -349,7 +349,7 @@ impl Layer<'_> { /// here. A binary one is a pack, whose sum `binary` already holds. The packs of the padding /// block are all equal, so their extension vanishes there. fn skip_sums(&self, claim: &Claim, field: &PowerField, s: F192, binary: &[F192; GROUP]) -> Vec { - let weights: Vec = claim.weights.iter().map(|&w| field.to_poly(w)).collect(); + let weights = SkipWeights::new(field, &claim.weights); let groups: Vec = eq_table(&claim.point[SKIP..]) .iter() .map(|&e| field.to_poly(e)) From 2e33b72058128cf990faadf71ad9479380a98f90 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Mon, 9 Nov 2026 10:54:56 +0000 Subject: [PATCH 05/12] keccak_gkr: round values without reducing the factor B Round::add_pair reduced B[j] = weights[j] * (kappa + T[j + 1]) + kappa * weights[j + 2] for both rows of a pair (Karatsuba's six products plus Barrett's twelve, per row) and then took the two dot products with T[j + 2] (six each). The x86 kernel (PowerField::round_dot) now leaves B as six words and meets T[j + 2] with them as two halves of three: per entry 72 carry-less products where 96 were. B + B' is weights * (T1 + T1'), the constants cancelling, so the value at infinity needs no second B. The sums grow to nine words (RoundSums to twelve after the eq weight), congruent to the old ones modulo the minimal polynomial, so the evaluated round values and the proof bytes are unchanged (PROOFDIGEST f090f5e4ee18ffad). The other arms keep reducing B (round_dot_rows), which is also the kernel's test oracle; the shifted rows of T are built in one pass. Profile: add_pair is CLMUL bound (carry-less products are ~45% of its samples). Per-kernel perf cycle samples, hash --log-n 17 --repeat 3, three runs each interleaved: Round::add_pair 4.34G / 4.41G / 4.45G -> 3.55G / 3.63G / 3.47G, about -19%. add_pair is about 1.8% of an aggregate --xmss 900 proof's cycles, so the whole-pass effect (about -0.35%) is below what ab.sh resolves on this host. Rebased onto e5d75f858 (after #724): the digests and measurements above were taken on the older base 8577d1e21. On the new base the series leaves the proofs byte-identical: PROOFDIGEST 7d91b8513bdd4b03 (aggregate --xmss 900 --log-inv-rate 1), 3c41e153f8b2ee37 (aggregate --sphincs 410 --log-inv-rate 1), aa4b2ebd7a6989b2 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). --- crates/keccak_gkr/src/power.rs | 125 +++++++++++++++++++++++++++++++- crates/keccak_gkr/src/prover.rs | 56 +++++++------- 2 files changed, 155 insertions(+), 26 deletions(-) diff --git a/crates/keccak_gkr/src/power.rs b/crates/keccak_gkr/src/power.rs index 6b108b7c52..8299b6eab9 100644 --- a/crates/keccak_gkr/src/power.rs +++ b/crates/keccak_gkr/src/power.rs @@ -121,6 +121,58 @@ impl PowerField { row_kernel!(width, dot_pair(a, b, width), dot_pair_portable(a, b, width)) } + /// `sum of B[j] * b[j]` and `sum of (B[j] + B'[j]) * (b[j] + b'[j])`, unreduced, where + /// `B[j] = weights[j] * a[j] + constant[j]` and `B'` likewise of `a'`: a round's values over + /// a pair of rows. `scratch` holds two rows where the products are reduced before the sums. + pub fn round_dot( + &self, + weights: &[u64], + constant: &[u64], + a: [&[u64]; 2], + b: [&[u64]; 2], + scratch: &mut [u64], + width: usize, + ) -> ([u64; 9], [u64; 9]) { + assert!( + [weights, constant] + .iter() + .chain(&a) + .chain(&b) + .all(|row| row.len() >= 3 * width) + ); + cfg_select! { + all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq") => { + if width.is_multiple_of(x86::STEP) { + let _ = scratch; + // SAFETY: the features are enabled at compile time, and every row is bounded above. + unsafe { x86::round_dot(weights, constant, a, b, width) } + } else { + self.round_dot_rows(weights, constant, a, b, scratch, width) + } + } + _ => self.round_dot_rows(weights, constant, a, b, scratch, width), + } + } + + /// [`Self::round_dot`] by reducing `B` and `B'` into `scratch` first: the other arms' form, + /// and the x86 kernel's test oracle, whose sums are congruent to these. + pub fn round_dot_rows( + &self, + weights: &[u64], + constant: &[u64], + a: [&[u64]; 2], + b: [&[u64]; 2], + scratch: &mut [u64], + width: usize, + ) -> ([u64; 9], [u64; 9]) { + let (b0, b1) = scratch[..6 * width].split_at_mut(3 * width); + self.mul_row(b0, weights, a[0], constant, width); + self.mul_row(b1, weights, a[1], constant, width); + let (zero, infinity) = self.dot_pair([b0, b1], b, width); + let pad = |x: [u64; 6]| std::array::from_fn(|i| if i < 6 { x[i] } else { 0 }); + (pad(zero), pad(infinity)) + } + /// `dst[j] = lo[j] + challenge * (lo[j] + hi[j])`. pub fn fold_row(&self, dst: &mut [u64], lo: &[u64], hi: &[u64], challenge: &Element, width: usize) { assert!(dst.len() >= 3 * width && lo.len() >= 3 * width && hi.len() >= 3 * width); @@ -342,6 +394,62 @@ mod x86 { (finish(&acc[0]), finish(&acc[1])) } + /// [`PowerField::round_dot`] without reducing `B`: its six words meet `b` as two halves of + /// three, Karatsuba's twelve products where reducing and multiplying take eighteen. `B + B'` + /// is `weights * (a + a')`, the constants cancelling. + pub unsafe fn round_dot( + weights: &[u64], + constant: &[u64], + a: [&[u64]; 2], + b: [&[u64]; 2], + width: usize, + ) -> ([u64; 9], [u64; 9]) { + // Per sum, the 128-bit partial sums by word offset. + let mut acc = [[V::zero(); 8]; 2]; + // The six words of products given by their even and odd entries' partial sums. + let words = |p: [[V; 5]; 2]| -> [V; 6] { + let [even, odd] = p.map(|half| std::array::from_fn::(|w| word(&half, w))); + std::array::from_fn(|w| even[w].unpacklo(odd[w])) + }; + for j in (0..width).step_by(STEP) { + let (w, c, a0, a1, b0, b1) = unsafe { + ( + load3(weights, width, j), + load3(constant, width, j), + load3(a[0], width, j), + load3(a[1], width, j), + load3(b[0], width, j), + load3(b[1], width, j), + ) + }; + let sw = sums(&w); + let mut first = words(karatsuba(&w, &sw, &a0, &sums(&a0))); + for (f, c) in first.iter_mut().zip(c) { + *f = f.xor(c); + } + let da = xor3(&a0, &a1); + let delta = words(karatsuba(&w, &sw, &da, &sums(&da))); + for (acc, (f, b)) in acc.iter_mut().zip([(first, b0), (delta, xor3(&b0, &b1))]) { + for offset in [0, 3] { + let p = partial(&[f[offset], f[offset + 1], f[offset + 2]], &b); + for s in 0..5 { + acc[offset + s] = acc[offset + s].xor(p[0][s].xor(p[1][s])); + } + } + } + } + let finish = |acc: &[V; 8]| { + let mut out = [0u64; 9]; + for (s, sum) in acc.iter().enumerate() { + let sum = sum.sum(); + out[s] ^= sum as u64; + out[s + 1] ^= (sum >> 64) as u64; + } + out + }; + (finish(&acc[0]), finish(&acc[1])) + } + pub unsafe fn reduce_row(field: &PowerField, dst: &mut [u64], wide: &[u64], width: usize) { for j in (0..width).step_by(STEP) { let w: [V; 4] = std::array::from_fn(|k| unsafe { V::load(wide.as_ptr().add(k * width + j)) }); @@ -541,7 +649,8 @@ mod tests { #[test] fn arithmetic_matches_the_standard_basis() { let mut rng = Rng::new(41); - let field = PowerField::new(rng.ext()).unwrap(); + let s = rng.ext(); + let field = PowerField::new(s).unwrap(); let width = 16; let elements = |rng: &mut Rng| -> (Vec, Vec) { let values = rng.ext_vec(width); @@ -590,5 +699,19 @@ mod tests { for j in 0..width { assert_eq!(field.to_field(&get(&fast, width, j)), a0[j] * b0[j] + b1[j]); } + + // The sums are congruent, not equal: compare their values at `s`. + let eval = PolyEval::new(s, 72); + let mut scratch = vec![0u64; 6 * width]; + let (zero, infinity) = field.round_dot(&ra0, &rb0, [&ra1, &rb1], [&rb0, &ra0], &mut scratch, width); + let (zero_rows, infinity_rows) = + field.round_dot_rows(&ra0, &rb0, [&ra1, &rb1], [&rb0, &ra0], &mut scratch, width); + assert_eq!(eval.eval(&zero), eval.eval(&zero_rows)); + assert_eq!(eval.eval(&infinity), eval.eval(&infinity_rows)); + let b = |j: usize, a: &[F192]| a0[j] * a[j] + b0[j]; + let expected: F192 = (0..width).map(|j| b(j, &a1) * b0[j]).sum(); + assert_eq!(eval.eval(&zero), expected); + let expected: F192 = (0..width).map(|j| (b(j, &a1) + b(j, &b1)) * (b0[j] + a0[j])).sum(); + assert_eq!(eval.eval(&infinity), expected); } } diff --git a/crates/keccak_gkr/src/prover.rs b/crates/keccak_gkr/src/prover.rs index 0401f0a589..85720a3231 100644 --- a/crates/keccak_gkr/src/prover.rs +++ b/crates/keccak_gkr/src/prover.rs @@ -193,6 +193,8 @@ struct Layer<'a> { /// A row of a table in the power basis: three words per position, laid out by word. const ROW: usize = 3 * BITS; +/// The scratch [`Round::add_pair`] takes: four shifted rows, and two for `round_dot`. +const PAIR_SCRATCH: usize = 6 * ROW; impl Layer<'_> { /// The blocks a segment's tasks walk: those written, then the padding block once, standing @@ -431,7 +433,7 @@ impl Layer<'_> { target_feature = "gfni" )); ( - vec![0u64; 4 * ROW], + vec![0u64; PAIR_SCRATCH], vec![0u64; if gfni { 0 } else { 8 * BITS }], vec![0u64; 2 * ROW], ) @@ -483,11 +485,11 @@ impl Layer<'_> { /// The sums a round accumulates, unreduced: its values at `0` and at infinity. #[derive(Clone, Copy, Default)] -struct RoundSums([u64; 9], [u64; 9]); +struct RoundSums([u64; 12], [u64; 12]); impl RoundSums { fn merge(mut self, other: Self) -> Self { - for i in 0..9 { + for i in 0..12 { self.0[i] ^= other.0[i]; self.1[i] ^= other.1[i]; } @@ -519,45 +521,49 @@ impl<'a> Round<'a> { } Self { field, - eval: PolyEval::new(s, 72), + eval: PolyEval::new(s, 96), kappa: field.to_poly(kappa), weights, constant, } } - /// `B[j] = weights[j] * (kappa + T[j + 1]) + kappa * weights[j + 2]` and `T[j + 2]`, for a - /// row `T`. - fn factors(&self, t: &[u64], b: &mut [u64], shifted: &mut [u64]) { - // `T[j + 1] + kappa`, then `T[j + 2]`: a lane is 64 positions, five lanes to a row. + /// `kappa + T[j + 1]` and `T[j + 2]`, for a row `T`: a lane is 64 positions, five lanes to a + /// row. + fn shifted(&self, t: &[u64], next: &mut [u64], after: &mut [u64]) { for k in 0..3 { - let (src, next) = (&t[k * BITS..][..BITS], &mut shifted[k * BITS..][..BITS]); + let (src, next, after) = ( + &t[k * BITS..][..BITS], + &mut next[k * BITS..][..BITS], + &mut after[k * BITS..][..BITS], + ); for lane in 0..BITS / 64 { let at = along(64 * lane, 1); for z in 0..64 { next[64 * lane + z] = src[at + z] ^ self.kappa[k]; } - } - } - self.field.mul_row(b, &self.weights, shifted, &self.constant, BITS); - for k in 0..3 { - let (src, dst) = (&t[k * BITS..][..BITS], &mut shifted[k * BITS..][..BITS]); - for lane in 0..BITS / 64 { - dst[64 * lane..][..64].copy_from_slice(&src[along(64 * lane, 2)..][..64]); + after[64 * lane..][..64].copy_from_slice(&src[along(64 * lane, 2)..][..64]); } } } /// Add one pair of rows to a round's sums: the values at `0` and at infinity of - /// `sum over positions of B[j] * T[j + 2]`, weighted by `eq`. + /// `sum over positions of B[j] * T[j + 2]`, weighted by `eq`, where + /// `B[j] = weights[j] * (kappa + T[j + 1]) + kappa * weights[j + 2]`. fn add_pair(&self, sums: &mut RoundSums, scratch: &mut [u64], low: &[u64], high: &[u64], eq: F192) { - let (b, shifted) = scratch.split_at_mut(2 * ROW); + let (shifted, rest) = scratch.split_at_mut(4 * ROW); + let (next, after) = shifted.split_at_mut(2 * ROW); for (half, t) in [low, high].into_iter().enumerate() { - self.factors(t, &mut b[half * ROW..][..ROW], &mut shifted[half * ROW..][..ROW]); + self.shifted(t, &mut next[half * ROW..][..ROW], &mut after[half * ROW..][..ROW]); } - let (zero, infinity) = self - .field - .dot_pair([&b[..ROW], &b[ROW..]], [&shifted[..ROW], &shifted[ROW..]], BITS); + let (zero, infinity) = self.field.round_dot( + &self.weights, + &self.constant, + [&next[..ROW], &next[ROW..]], + [&after[..ROW], &after[ROW..]], + rest, + BITS, + ); let weight = self.field.to_poly(eq); add_mul(&mut sums.0, &zero, &weight); add_mul(&mut sums.1, &infinity, &weight); @@ -600,7 +606,7 @@ impl<'a> Round<'a> { fold(out, segment, 0); } let mut sums = RoundSums::default(); - let mut scratch = vec![0u64; 4 * ROW]; + let mut scratch = vec![0u64; PAIR_SCRATCH]; for (pair, &eq) in dst[..to.len() * ROW].as_chunks::<{ 2 * ROW }>().0.iter().zip(eq) { self.add_pair(&mut sums, &mut scratch, &pair[..ROW], &pair[ROW..], eq); } @@ -610,7 +616,7 @@ impl<'a> Round<'a> { let (pairs, half) = (to.live.div_ceil(2), to.m / 2); let mut sums = parallel::map_reduce_with_state( to.segments * pairs, - || (vec![0u64; 4 * ROW], vec![0u64; 2 * ROW]), + || (vec![0u64; PAIR_SCRATCH], vec![0u64; 2 * ROW]), RoundSums::default, |(scratch, rows), sums, task| { let (segment, t) = (task / pairs, task % pairs); @@ -643,7 +649,7 @@ impl<'a> Round<'a> { if first == half { return; } - let mut scratch = vec![0u64; 4 * ROW]; + let mut scratch = vec![0u64; PAIR_SCRATCH]; for (pad, eq) in pads.as_chunks::().0.iter().zip(eq.chunks_exact(half)) { self.add_pair(sums, &mut scratch, pad, pad, eq[first..].iter().sum()); } From ca23a5f5d9237068acdd73224f59e4282b5a1fd5 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Mon, 9 Nov 2026 12:00:14 +0000 Subject: [PATCH 06/12] pcs: skip the empty vector of an AVX-512 sweep's short last chunk The AVX-512 register kernels take two eight-lane vectors of every row at a time. A row of 56 words, the L0 base encode's 51 lanes padded to a vector, is three such chunks and a last chunk of one vector, and the kernel ran both vectors of that chunk, the second all zeros: every sweep of the deep pass and of the openings' row rebuilds did 64 lanes of work for 56. The last chunk now runs the kernel on the vectors that hold lanes only, each count an inlined call of its own. Rows of whole chunks (the gathered pass's 64-word stage rows, the extension encodes' 48 words) are unchanged. Byte-identical: PROOFDIGEST c7afa387f98cdb98 (aggregate --xmss 900 --log-inv-rate 1). Tests: pcs. Measured on the loaded host against lv-head4, BENCH_SPAN_CPU process CPU per span, 7 interleaved runs: Prove > Commit (base encode and its leaf hashing) median 1.456 -> 1.322 s, min 1.369 -> 1.204 s; level commits unchanged (0.785 -> 0.769 s). Kernel alone, one thread, 11 layers over 2^11 rows of 56 words: 0.31-0.37 -> 0.26 ns per butterfly, the rate of a 64-word row. Rebased onto e5d75f858 (after #724): the digests and measurements above were taken on the older base 8577d1e21. On the new base the series leaves the proofs byte-identical: PROOFDIGEST 7d91b8513bdd4b03 (aggregate --xmss 900 --log-inv-rate 1), 3c41e153f8b2ee37 (aggregate --sphincs 410 --log-inv-rate 1), aa4b2ebd7a6989b2 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). --- crates/pcs/src/ntt/kernels.rs | 39 +++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/crates/pcs/src/ntt/kernels.rs b/crates/pcs/src/ntt/kernels.rs index 301eb21582..5f27b10461 100644 --- a/crates/pcs/src/ntt/kernels.rs +++ b/crates/pcs/src/ntt/kernels.rs @@ -116,10 +116,10 @@ pub(super) fn butterfly_interleaved_fused_3layer( num_ntts, t, #[inline(always)] - |v, t| { + |v, t, vectors| { radix8_schedule( #[inline(always)] - |top, bot, k| butterfly_regs(v, top, bot, t[k]), + |top, bot, k| butterfly_regs(v, top, bot, t[k], vectors), ); }, ); @@ -180,7 +180,7 @@ fn fused_sweep_avx512( stride_rows: usize, num_ntts: usize, twiddles: &[F64; T], - kernel: impl Fn(&mut RowVectors, &[Twiddle; T]), + kernel: impl Fn(&mut RowVectors, &[Twiddle; T], usize), ) { fused_rows::(block, src, stride_rows, num_ntts, |src, rows| { fused_lanes_avx512( @@ -188,7 +188,7 @@ fn fused_sweep_avx512( rows, twiddles, #[inline(always)] - |v, t| kernel(v, t), + |v, t, vectors| kernel(v, t, vectors), ) }); } @@ -265,10 +265,10 @@ pub(super) fn butterfly_interleaved_fused_2layer( num_ntts, t, #[inline(always)] - |v, t| { + |v, t, vectors| { radix4_schedule( #[inline(always)] - |top, bot, k| butterfly_regs(v, top, bot, t[k]), + |top, bot, k| butterfly_regs(v, top, bot, t[k], vectors), ); }, ); @@ -641,12 +641,12 @@ type Twiddle = (core::arch::x86_64::__m512i, bool); #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] type RowVectors = [[core::arch::x86_64::__m512i; LANE_VECTORS]; N]; -/// Butterfly rows `top` and `bot` of every vector in flight. +/// Butterfly rows `top` and `bot` of the first `vectors` vectors in flight. #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] #[inline] #[target_feature(enable = "vpclmulqdq", enable = "avx512f", enable = "avx512bw")] -fn butterfly_regs(v: &mut RowVectors, top: usize, bot: usize, (tw, zero): Twiddle) { - for k in 0..LANE_VECTORS { +fn butterfly_regs(v: &mut RowVectors, top: usize, bot: usize, (tw, zero): Twiddle, vectors: usize) { + for k in 0..vectors { let (u, w) = (v[top][k], v[bot][k]); let new_u = if zero { u } else { mul_xor_avx512(u, w, tw) }; v[top][k] = new_u; @@ -659,7 +659,8 @@ fn butterfly_regs(v: &mut RowVectors, top: usize, bot: usize, /// - The vectors load from `src`, or from `rows` when there is none, and store to `rows`. /// - A vector is loaded and stored once, however many butterflies `kernel` applies to it. /// - The kernel orders its butterflies breadth-first, so the vectors in flight hide each other's latency. -/// - The last chunk may be short: its vectors past the row's end load as zeros and are not stored. +/// - The kernel runs on the first `vectors` vectors it is handed: all of them, but in the last chunk only those +/// that hold lanes. Their lanes past the row's end load as zeros and are not stored. #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] #[inline] #[target_feature(enable = "vpclmulqdq", enable = "avx512f", enable = "avx512bw")] @@ -667,7 +668,7 @@ fn fused_lanes_avx512( src: Option<[&[F64]; N]>, rows: &mut [&mut [F64]; N], twiddles: &[F64; T], - kernel: impl Fn(&mut RowVectors, &[Twiddle; T]), + kernel: impl Fn(&mut RowVectors, &[Twiddle; T], usize), ) { use core::arch::x86_64::*; use primitives::simd::{load, load_prefix, store, store_prefix}; @@ -690,7 +691,7 @@ fn fused_lanes_avx512( }), None => std::array::from_fn(|i| std::array::from_fn(|k| load(&*chunks[i], k))), }; - kernel(&mut v, &tw); + kernel(&mut v, &tw, LANE_VECTORS); for i in 0..N { for k in 0..LANE_VECTORS { store(chunks[i], k, v[i][k]); @@ -700,17 +701,23 @@ fn fused_lanes_avx512( } if lane < len { let reg = lane / 8; - // The lanes of vector `k` of the short chunk: all eight, the row's last few, or none. - let n = |k: usize| len.saturating_sub(lane + 8 * k).min(8); + // The lanes of vector `k` of the short chunk: all eight, or the row's last few. + let n = |k: usize| (len - lane - 8 * k).min(8); + let vectors = (len - lane).div_ceil(8); let mut v: RowVectors = std::array::from_fn(|i| { std::array::from_fn(|k| match &src { + _ if k >= vectors => _mm512_setzero_si512(), Some(src) => load_prefix(src[i], reg + k, n(k)), None => load_prefix(&*rows[i], reg + k, n(k)), }) }); - kernel(&mut v, &tw); + // Each count is its own call, so the kernel's loops over the vectors unroll. + match vectors { + 1 => kernel(&mut v, &tw, 1), + _ => kernel(&mut v, &tw, LANE_VECTORS), + } for i in 0..N { - for k in 0..LANE_VECTORS { + for k in 0..vectors { store_prefix(rows[i], reg + k, n(k), v[i][k]); } } From e5f837c28f94f4bafd1d0b7f8c9440a378cc8bc9 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Mon, 9 Nov 2026 12:00:22 +0000 Subject: [PATCH 07/12] pcs: run the NTT's layers depth first past L1 run_layers swept each fused group of layers over the whole buffer before the next. A deep task is an L2 budget of rows or more (2^11 rows of 56 words at L0, 2^12 rows of 48 words at the level-2 encode, 1.5 MiB), so each of its sweeps streamed it through L2 and spilled to L3. Now a buffer larger than L1 (4096 words) runs depth first: after a sweep, each block of the sweep's last layer runs its remaining layers alone, while it is in cache. Same butterflies and twiddles, in another order. With `live` (the openings' pruned rebuilds) the order stays breadth first. Byte-identical: PROOFDIGEST c7afa387f98cdb98. Tests: pcs. Kernel alone, one thread, min of 20: 12 layers over 2^12 rows of 48 words 0.341-0.347 -> 0.309-0.310 ns per butterfly, 11 layers over 2^11 rows of 48 words 0.325-0.336 -> 0.303-0.308, 2^11 rows of 56 words 0.40 -> 0.35 (before the previous commit). In the prover the deep pass waits on its DRAM reads, and the stage moves within noise (BENCH_SPAN_CPU, 7 interleaved runs, on top of the previous commit): Prove > Commit median 1.322 -> 1.294 s, min 1.204 -> 1.279 s; level commits 0.769 -> 0.772 s. Rebased onto e5d75f858 (after #724): the digests and measurements above were taken on the older base 8577d1e21. On the new base the series leaves the proofs byte-identical: PROOFDIGEST 7d91b8513bdd4b03 (aggregate --xmss 900 --log-inv-rate 1), 3c41e153f8b2ee37 (aggregate --sphincs 410 --log-inv-rate 1), aa4b2ebd7a6989b2 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). --- crates/pcs/src/ntt/layers.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/crates/pcs/src/ntt/layers.rs b/crates/pcs/src/ntt/layers.rs index e517e708ca..9ba2adbdad 100644 --- a/crates/pcs/src/ntt/layers.rs +++ b/crates/pcs/src/ntt/layers.rs @@ -13,6 +13,8 @@ impl AdditiveNttF64 { /// - The domain has `2^log_d` rows and splits into `2^outer_log` equal sub-blocks. /// - The buffer is one of them; its index fixes the global block index, and so the twiddle, of each block. /// - Layers fuse into radix-8 sweeps, with radix-4 sweeps where a lone radix-2 would be left over. + /// - A buffer larger than L1 runs depth first: after a sweep, each of its blocks finishes alone, while in cache. + /// With `live`, it stays breadth first. /// /// With `live` (sorted), blocks that hold none of those buffer rows are left alone. /// @@ -101,10 +103,22 @@ impl AdditiveNttF64 { layer += 1; } src = None; + // A buffer larger than L1 goes depth first: each block of the sweep finishes alone, while it is in cache. + if live.is_none() && layer < log_d && buf.len() > DEPTH_FIRST_WORDS { + let block_elems = num_ntts << (log_d - layer); + let first = sub_idx << (layer - outer_log); + for (b, block) in buf.chunks_exact_mut(block_elems).enumerate() { + self.run_layers(block, None, log_d, num_ntts, layer, layer, first + b, None); + } + return; + } } } } +/// Words of a buffer whose layers run breadth first: one L1 data cache. +const DEPTH_FIRST_WORDS: usize = 1 << 12; + /// Call `f` with each block of `1 << log_rows` rows that holds a row of `live` (sorted). /// With no `live`, call it with every block of `num_blocks`. fn for_each_block(num_blocks: usize, log_rows: usize, live: Option<&[usize]>, mut f: impl FnMut(usize)) { From f35ec0b61342d0c25328895051be358da9d157a1 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Mon, 9 Nov 2026 12:00:34 +0000 Subject: [PATCH 08/12] pcs: keep only the message of an extension encode whose layers are all deep WHIR levels 2 to 4 (rates 1/128 to 1/8192, 2^12 down to 2^4 message rows of 16 F192 lanes) plan a deep pass only: each task builds its cosets in scratch from the message, hands them to the leaf hasher, and then streamed them out as the finished codeword, 200, 100 and 50 MiB. On x86 (REBUILD_OVER_STORE) the commit keeps a codeword before its deep layers, and for these plans that is the message's replicas. So the encode now writes no buffer: it keeps one replica, the message itself (1.5 MiB at level 2), as Codeword::BeforeDeep, and a query's row runs the deep layers of its sub-block from that replica, pruned to the queried rows, as rows read from any codeword kept before its deep layers already do. Level 1 (one gathered pass, then the deep pass) and the L0 base encode are unchanged, as is the Apple path that keeps the whole codeword. Byte-identical: PROOFDIGEST c7afa387f98cdb98 (aggregate --xmss 900 --log-inv-rate 1), 729f46975ab551c1 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). Tests: pcs; the deferred-deep test adds a shape whose tasks span several replicas. Measured on the loaded host against lv-head4 (this and the two commits before), BENCH_SPAN_CPU process CPU per span, 7 interleaved runs: Prove > PCS open > Commit (levels 1 to 4) median 0.785 -> 0.737 s, min 0.734 -> 0.654 s; Prove > PCS open > Open 0.078 -> 0.094 s (the rows of levels 2 to 4 are now finished on demand); Prove > PCS open 1.290 -> 1.271 s; Prove > Commit 1.456 -> 1.344 s. Whole pass (ab.sh 4x4, load average 500 to 850): CPU min 8.45 -> 8.03 s, median 8.90 -> 8.89 s; wall min 1.451 -> 1.763 s, median 2.596 -> 2.288 s. The 350 MiB of level 2 to 4 codewords are no longer allocated. PCS throughput bench (PCS_LOG_N=26, rate 1/2, 5 passes, two interleaved rounds) is noise-bound on this host: commit 3029 / 325 ms -> 287 / 401 ms, open 7097 / 394 ms -> 387 / 486 ms. Rebased onto e5d75f858 (after #724): the digests and measurements above were taken on the older base 8577d1e21. On the new base the series leaves the proofs byte-identical: PROOFDIGEST 7d91b8513bdd4b03 (aggregate --xmss 900 --log-inv-rate 1), 3c41e153f8b2ee37 (aggregate --sphincs 410 --log-inv-rate 1), aa4b2ebd7a6989b2 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). --- crates/pcs/src/ntt/deep.rs | 19 ++++++++++++------- crates/pcs/src/ntt/encode.rs | 9 +++++++++ crates/pcs/src/ntt/openings.rs | 5 ++++- crates/pcs/src/ntt/tests.rs | 11 +++++++++-- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/crates/pcs/src/ntt/deep.rs b/crates/pcs/src/ntt/deep.rs index df056b3dfd..7558c39fb0 100644 --- a/crates/pcs/src/ntt/deep.rs +++ b/crates/pcs/src/ntt/deep.rs @@ -43,29 +43,33 @@ impl<'s> DeepPass<'s> { pub(super) fn run(&self, ntt: &AdditiveNttF64, win: &mut Window, src: Source<'_>) { match src { - Source::Rows(msg) => self.with_message(ntt, win, msg), + Source::Rows(msg) => self.with_message(ntt, Some(win), msg), _ => self.in_place(ntt, win), } } - /// A separate message: build each task in scratch, hash it, then write it out once. + /// A separate message and no buffer: each task is built in scratch and goes to the sink only. + pub(super) fn unkept(&self, ntt: &AdditiveNttF64, msg: &[F64]) { + self.with_message(ntt, None, msg); + } + + /// A separate message: build each task in scratch, hash it, then write it out once into `out`, if given. /// /// Streamed out whole, the codeword is written without ever being read. - fn with_message(&self, ntt: &AdditiveNttF64, win: &mut Window, msg: &[F64]) { + fn with_message(&self, ntt: &AdditiveNttF64, out: Option<&mut Window>, msg: &[F64]) { let Plan { log_d, num_ntts, - start, deep_start, .. } = self.plan; let (task_len, outer, log_task, sink) = (self.task_len, self.outer, self.log_task, self.sink); - let (base, total) = (win.base(), win.len()); - let block_len = total >> start; + let out = out.map(|win| (win.base(), win.stream())); + let block_len = msg.len(); parallel::for_each_chunk(1 << outer, |lo, hi| { with_scratch(task_len, |scratch| { // One fence at the end of the task covers all its streaming stores. - let stream = win.stream().then(Stream::new); + let stream = out.is_some_and(|(_, stream)| stream).then(Stream::new); for task in lo..hi { // A task sits at the same offset in every replica, or covers whole replicas. // @@ -81,6 +85,7 @@ impl<'s> DeepPass<'s> { if let Some(f) = sink { f(task << log_task, scratch); } + let Some((base, _)) = out else { continue }; // SAFETY: the task's rows lie inside the codeword, and this task alone owns them. let dst = unsafe { std::slice::from_raw_parts_mut(base.add(task * task_len), task_len) }; match &stream { diff --git a/crates/pcs/src/ntt/encode.rs b/crates/pcs/src/ntt/encode.rs index 6479ae85e6..5308707577 100644 --- a/crates/pcs/src/ntt/encode.rs +++ b/crates/pcs/src/ntt/encode.rs @@ -37,6 +37,7 @@ pub(crate) enum Codeword { /// All of it. Whole(Vec, Plan), /// The buffer as the gathered passes left it: the deep layers run again on each row read. + /// With no gathered pass, one replica of the message: every replica is the same. BeforeDeep(Vec, Plan), /// No buffer: each row is recomputed from the message of this many lane-major lanes. Nothing(Plan, usize), @@ -124,6 +125,8 @@ impl AdditiveNttF64 { /// - The codeword equals the in-place encode's, and every word is written before it is read. /// - With `defer_deep`, the deep pass is not written back: each deep sub-block is finished in scratch /// and handed to `on_rows` from there, and the codeword is kept [`Codeword::BeforeDeep`]. + /// - When every layer is deep, that is the message's replicas: only the message is kept, and no buffer + /// is written at all. pub(crate) fn encode_interleaved_with( &self, msg: &[F64], @@ -132,6 +135,12 @@ impl AdditiveNttF64 { defer_deep: bool, on_rows: &RowSink<'_>, ) -> Codeword { + let log_d = log2_strict_usize(msg.len() / num_ntts) + log_inv_rate; + let plan = Plan::new(log_d, num_ntts, log_inv_rate); + if defer_deep && plan.deep_start == plan.start && plan.fuses_rows() { + DeepPass::new(&plan, Some(on_rows), true).unkept(self, msg); + return Codeword::BeforeDeep(msg.to_vec(), plan); + } let mut done = None; // SAFETY: the fill writes every element before it reads any. let data = unsafe { diff --git a/crates/pcs/src/ntt/openings.rs b/crates/pcs/src/ntt/openings.rs index dd38b64d1b..3f142d00b0 100644 --- a/crates/pcs/src/ntt/openings.rs +++ b/crates/pcs/src/ntt/openings.rs @@ -68,7 +68,10 @@ impl AdditiveNttF64 { Codeword::Whole(ref words, plan) => gather_rows(words, plan.num_ntts, positions), Codeword::BeforeDeep(ref words, plan) => { let sub_len = plan.sub_len(); - self.finished_rows(&plan, positions, |_, sub| &words[sub * sub_len..][..sub_len]) + // A kept message is one replica, which every sub-block of a coset is a piece of. + self.finished_rows(&plan, positions, |_, sub| { + &words[(sub * sub_len) % words.len()..][..sub_len] + }) } Codeword::Nothing(plan, n_lanes) => { let log_sub = plan.log_sub(); diff --git a/crates/pcs/src/ntt/tests.rs b/crates/pcs/src/ntt/tests.rs index 4a84088230..78a6652151 100644 --- a/crates/pcs/src/ntt/tests.rs +++ b/crates/pcs/src/ntt/tests.rs @@ -292,13 +292,20 @@ fn lane_major_encode_matches_transpose_then_in_place() { /// must equal the full encode's, wherever the plan splits off a deep pass. Whether a shape /// splits depends on the cache budget and the pool, so only some shape must. /// -/// A deep-only plan with a separate message builds the whole codeword and returns `None`. +/// A deep-only plan with a separate message keeps only the message, and finishes each row from one replica of it: +/// `(9, 8, 1)` and, with tasks of several replicas, `(10, 48, 7)`. /// The in-place, deep-only, deferred branch runs in `lane_major_encode_matches_transpose_then_in_place`, /// at shape `(5, 2, 7)`. #[test] fn deferred_deep_rows_match_the_full_encode() { let mut rng = Rng::new(0xDEE9); - let shapes = [(14usize, 51usize, 1usize), (15, 64, 2), (9, 8, 1), (4, 8, 4)]; + let shapes = [ + (14usize, 51usize, 1usize), + (15, 64, 2), + (9, 8, 1), + (10, 48, 7), + (4, 8, 4), + ]; let mut splits = 0; for (log_d, lanes, log_inv_rate) in shapes { let ntt = AdditiveNttF64::standard(log_d); From bbf8c4c330a87fcf4a2634d0d3a2fbb474b0d960 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Mon, 9 Nov 2026 11:06:40 +0000 Subject: [PATCH 09/12] execute: advance access counts in the access that reads or writes the cell An instruction's read and store now return the cell's access count and advance it in the same touch (Mem::read, Mem::put_counted), instead of a second pass of bump_access_count over the row's cells after its reads and stores. Bumps keep flush order, so aliased operands still pair each access with its own count; DEREF's store side keeps its separate bumps. A cell is indexed past its covering check without bounds checks (SAFETY: ensure grew the pages past it). cells_of_state is specialized on the layout like state_of_cells. Byte-identical: PROOFDIGEST c7afa387f98cdb98 (aggregate xmss 900 rate 1), 729f46975ab551c1 (recursion n 2 xmss-per-leaf 900 rate 2); witness checksum of q and the virtual columns unchanged. Measured (solo execute, one thread, perf stat over 20 executes, xmss 900): user instructions per execute 778M -> 627M (-19%). Cycles per execute in execute_with itself (5 interleaved rounds, min) 420M -> 420M: the loop is bound by cache misses on this host, so the instructions saved do not show as time here. Since #724 a KECCAK row writes only the output cells its bytecode requests: put_counted runs on those, and a cell it does not write keeps count 0, as on main. Rebased onto e5d75f858 (after #724): the digests and measurements above were taken on the older base 8577d1e21. On the new base the series leaves the proofs byte-identical: PROOFDIGEST 7d91b8513bdd4b03 (aggregate --xmss 900 --log-inv-rate 1), 3c41e153f8b2ee37 (aggregate --sphincs 410 --log-inv-rate 1), aa4b2ebd7a6989b2 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). --- crates/leanvm_core/src/cpu/execute.rs | 105 +++++++++++++++----------- crates/leanvm_core/src/keccak.rs | 13 ++-- 2 files changed, 71 insertions(+), 47 deletions(-) diff --git a/crates/leanvm_core/src/cpu/execute.rs b/crates/leanvm_core/src/cpu/execute.rs index 83dcdd6a63..b87f743d86 100644 --- a/crates/leanvm_core/src/cpu/execute.rs +++ b/crates/leanvm_core/src/cpu/execute.rs @@ -390,16 +390,24 @@ impl Program { .push(unsafe { Box::<[Cell]>::new_zeroed_slice(PAGE).assume_init() }); } } - /// Cell `c`, which `ensure` has covered. + /// Cell `c`, covered first. #[inline(always)] fn at(&mut self, c: u32) -> &mut Cell { let c = c as usize; - &mut self.pages[c >> PAGE_LOG][c & (PAGE - 1)] + self.ensure(c); + // SAFETY: `ensure` grew the pages past cell `c`. + unsafe { + self.pages + .get_unchecked_mut(c >> PAGE_LOG) + .get_unchecked_mut(c & (PAGE - 1)) + } } #[inline(always)] fn covered(&self, c: u32) -> Option<&Cell> { let c = c as usize; - (c < self.used).then(|| &self.pages[c >> PAGE_LOG][c & (PAGE - 1)]) + // SAFETY: every cell below `used` is in a page. + (c < self.used) + .then(|| unsafe { self.pages.get_unchecked(c >> PAGE_LOG).get_unchecked(c & (PAGE - 1)) }) } #[inline(always)] fn written(&self, cell: u32) -> bool { @@ -411,18 +419,21 @@ impl Program { fn get(&self, cell: u32) -> F192 { self.covered(cell).map_or(F192::ZERO, |c| c.word) } - // An instruction's read. An unwritten cell reads as ZERO, which is then written - // there: the row is filled from the final image, which has to agree. + // An instruction's read, which advances the cell's access count as + // `bump_access_count` does: the word and the count before. An unwritten cell + // reads as ZERO, which is then written there: the row is filled from the final + // image, which has to agree. #[inline(always)] - fn read(&mut self, cell: u32) -> F192 { - self.ensure(cell as usize); + fn read(&mut self, cell: u32) -> (F192, u32) { let c = self.at(cell); - let word = c.word; - if c.state != State::Written { - c.state = State::Written; + let (word, count, fresh) = (c.word, c.count, c.state != State::Written); + c.count = count + 1; + c.state = State::Written; + self.counts = self.counts.max(count + 1); + if fresh { self.unwritten_reads.push(cell); } - word + (word, count) } // Write-once store, carried to every cell linked to this one. #[inline(always)] @@ -432,10 +443,27 @@ impl Program { } Ok(()) } + // An instruction's `put`, which advances the cell's access count as + // `bump_access_count` does: the count before. + #[inline(always)] + fn put_counted(&mut self, cell: u32, v: F192) -> Result { + let c = self.at(cell); + let (s, count) = (c.state, c.count); + if s == State::Written && c.word != v { + return Err(self.conflict(cell, v)); + } + c.word = v; + c.state = State::Written; + c.count = count + 1; + self.counts = self.counts.max(count + 1); + if s == State::Linked { + self.spread(cell, v)?; + } + Ok(count) + } // `put` on one cell; whether it was linked. #[inline(always)] fn store(&mut self, cell: u32, v: F192) -> Result { - self.ensure(cell as usize); let c = self.at(cell); let s = c.state; if s == State::Written && c.word != v { @@ -459,7 +487,6 @@ impl Program { } fn link(&mut self, a: u32, b: u32) { for (x, y) in [(a, b), (b, a)] { - self.ensure(x as usize); let c = self.at(x); if c.state == State::Unwritten { c.state = State::Linked; @@ -481,7 +508,6 @@ impl Program { // exponent. A run makes fewer than `2^32` accesses in all, so it cannot wrap. #[inline(always)] fn bump_access_count(&mut self, cell: u32) -> u32 { - self.ensure(cell as usize); let c = self.at(cell); let count = c.count; c.count = count + 1; @@ -830,13 +856,12 @@ impl Program { m.put(if ha { ab } else { aa }, m.get(ac) * vk.inv())?; } } - let va = m.read(aa); - let vb = m.read(ab); + // Each access advances its cell's count, in flush order, so a cell two + // operands alias still pairs each access with its own count. + let (va, ra) = m.read(aa); + let (vb, rb) = m.read(ab); let vc = if is_xor { va + vb } else { va * vb }; - m.put(ac, vc)?; - let ra = m.bump_access_count(aa); - let rb = m.bump_access_count(ab); - let rc = m.bump_access_count(ac); + let rc = m.put_counted(ac, vc)?; let row = Xrow { pc, fp, @@ -853,9 +878,7 @@ impl Program { pc += 1; } Op::Set { o, k } => { - let a = fp + o; - m.put(a, k)?; - let r = m.bump_access_count(a); + let r = m.put_counted(fp + o, k)?; set.push(Srow { pc, fp, @@ -866,7 +889,7 @@ impl Program { } Op::Deref { o1, o2, o3, mode } => { let a1 = fp + o1; - let p = m.read(a1); + let (p, r1) = m.read(a1); let p_addr = p.as_base().ok_or_else(|| fail(Fault::WildPointer { value: p }))?; let base = g.log(p_addr).ok_or_else(|| fail(Fault::WildPointer { value: p }))?; let a2 = base + o2; @@ -898,7 +921,6 @@ impl Program { m.put(a2, v)?; } } - let r1 = m.bump_access_count(a1); let r2 = m.bump_access_count(a2); let r3 = m.bump_access_count(a3); deref.push(Drow { @@ -919,16 +941,16 @@ impl Program { // otherwise not balance. A guest branches on g-powers; the one // idiom that once branched on a word, `assert a != b`, takes an // inverse hint instead (§sec:prog-div-ne). - let c = in_k("JUMP condition", m.read(ac)).map_err(fail)?; - let d = in_k("JUMP target", m.read(ad)).map_err(fail)?; - let f = in_k("JUMP fp", m.read(af)).map_err(fail)?; + let (c, rc) = m.read(ac); + let c = in_k("JUMP condition", c).map_err(fail)?; + let (d, rd) = m.read(ad); + let d = in_k("JUMP target", d).map_err(fail)?; + let (f, rf) = m.read(af); + let f = in_k("JUMP fp", f).map_err(fail)?; // The is-nonzero witness `w = c⁻¹` is never used for control // flow, only recorded as a witness column, so it is not // computed here at all: `JumpTable::fill` batch-inverts every // row's condition at once (§the trace rows in `cpu::trace`). - let rc = m.bump_access_count(ac); - let rd = m.bump_access_count(ad); - let rf = m.bump_access_count(af); let taken = !c.is_zero(); jump.push(Jrow { pc, @@ -958,8 +980,13 @@ impl Program { Op::Keccak { .. } => { use crate::keccak::{ROW_CELLS, STATE_CELLS}; let k = crate::tables::KeccakOp::of(&op); - let cells: [u32; ROW_CELLS] = std::array::from_fn(|c| k.cell(fp, c)); - let input: [F192; STATE_CELLS] = std::array::from_fn(|c| m.read(cells[c])); + // The counts in flush order, so a cell two operands alias still pairs + // each access with its own count. + let mut r = [0; ROW_CELLS]; + let mut input = [F192::ZERO; STATE_CELLS]; + for c in 0..STATE_CELLS { + (input[c], r[c]) = m.read(k.cell(fp, c)); + } // The table reads every cell with a literal zero top limb, so anything // else could not balance the bus. for (cell, &value) in input.iter().enumerate() { @@ -973,16 +1000,10 @@ impl Program { let mut lanes = crate::keccak::state_of_cells(k.in_odd, |c| input[c]); keccak_step(&lanes); primitives::hash::keccak_f(&mut lanes); - let written = STATE_CELLS + k.out_cells.cells(); let output = crate::keccak::cells_of_state(&lanes, k.out_odd); - for (&cell, &v) in cells[STATE_CELLS..written].iter().zip(&output) { - m.put(cell, v)?; - } - // In flush order, so a cell two operands alias still pairs each - // read with its own count. A cell not written keeps count 0. - let mut r = [0; ROW_CELLS]; - for (count, &cell) in r.iter_mut().zip(&cells[..written]) { - *count = m.bump_access_count(cell); + // A cell not written keeps count 0. + for (c, &v) in output[..k.out_cells.cells()].iter().enumerate() { + r[STATE_CELLS + c] = m.put_counted(k.cell(fp, STATE_CELLS + c), v)?; } keccak.push(Krow { pc, diff --git a/crates/leanvm_core/src/keccak.rs b/crates/leanvm_core/src/keccak.rs index 085d3d12c6..4a293bbba2 100644 --- a/crates/leanvm_core/src/keccak.rs +++ b/crates/leanvm_core/src/keccak.rs @@ -102,11 +102,14 @@ pub(crate) fn state_of_cells(odd: bool, cell: impl Fn(usize) -> F192) -> [u64; S /// The thirteen canonical state cells holding 25 lanes in a layout, a free lane zero. pub(crate) fn cells_of_state(s: &[u64; STATE_LANES], odd: bool) -> [F192; STATE_CELLS] { - let lanes = cell_lanes(odd); - std::array::from_fn(|c| { - let (lo, hi) = lanes[c]; - F192::new(lo.map_or(0, |l| s[l]), hi.map_or(0, |h| s[h]), 0) - }) + // The layout is a constant, so the loop compiles to straight-line moves. + fn write(s: &[u64; STATE_LANES]) -> [F192; STATE_CELLS] { + std::array::from_fn(|c| { + let (lo, hi) = const { cell_lanes(ODD) }[c]; + F192::new(lo.map_or(0, |l| s[l]), hi.map_or(0, |h| s[h]), 0) + }) + } + if odd { write::(s) } else { write::(s) } } /// What a `KECCAK` instruction writes, given the cells it reads. From 76c2a226e80d50d82f08cce03960f9e502581574 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Mon, 9 Nov 2026 11:06:40 +0000 Subject: [PATCH 10/12] hints: g-power index slots carry their keys, behind per-slot tags GPow's reverse index (announced exponents) and its coarse index (every 64th exponent) held only the exponent and confirmed a candidate slot against the forward table, a random read of a table as large as the address space for every probe. A slot now holds (key, exponent), and a byte per slot holds eight tag bits of the keys whose home slot it is, so a lookup of an absent key, all but the last of the up to 64 probes a resolve makes, reads one byte of an array a sixteenth the slots' size. Resolve inserts the exponent it found directly. Byte-identical: PROOFDIGEST c7afa387f98cdb98, recursion 729f46975ab551c1. Measured (solo execute, one thread, xmss 900): GPow::resolve's share of the execute loop's L2 misses 23% -> 12% (in-run sample shares, so the host's load cancels). Cycles per execute in GPow, Index, hint streams and field inverses (perf, 3 to 5 interleaved rounds, min) 75M -> 56M. Rebased onto e5d75f858 (after #724): the digests and measurements above were taken on the older base 8577d1e21. On the new base the series leaves the proofs byte-identical: PROOFDIGEST 7d91b8513bdd4b03 (aggregate --xmss 900 --log-inv-rate 1), 3c41e153f8b2ee37 (aggregate --sphincs 410 --log-inv-rate 1), aa4b2ebd7a6989b2 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). --- crates/leanvm_core/src/cpu/hints.rs | 170 +++++++++++++++++----------- 1 file changed, 104 insertions(+), 66 deletions(-) diff --git a/crates/leanvm_core/src/cpu/hints.rs b/crates/leanvm_core/src/cpu/hints.rs index ebd97026b8..a964f52246 100644 --- a/crates/leanvm_core/src/cpu/hints.rs +++ b/crates/leanvm_core/src/cpu/hints.rs @@ -17,11 +17,12 @@ pub(crate) const MAX_CELLS: u32 = 1 << 28; /// and a pointer word read back out of memory must be inverted to the index it /// addresses. Three things keep this cheap. /// -/// The reverse index holds nothing but the exponent, confirming a candidate slot -/// against the forward table, so a slot is four bytes where a `HashMap` bucket is -/// sixteen plus a control byte. Keys are field elements, hence effectively -/// uniform, which is all the placement asks for: one multiplicative mix, then -/// linear probing. The table stays at most half full to keep probing short. +/// A reverse-index slot holds the key beside its exponent, so a probe reads the +/// slot alone: confirming a candidate against the forward table, as large as the +/// address space, would cost a cache miss per probe. Keys are field elements, +/// hence effectively uniform, which is all the placement asks for: one +/// multiplicative mix, then linear probing. The table stays at most half full to +/// keep probing short. /// /// It indexes only the exponents [`Self::note`] announces, or a lookup resolved: /// a run addresses millions of cells but stores only the few thousand bases of @@ -34,26 +35,106 @@ pub(crate) const MAX_CELLS: u32 = 1 << 28; /// most `STRIDE` probes once. pub struct GPow { pow: Vec, - /// `k + 1` at the slot `g^k` mixes to, for the announced `k`; 0 marks an empty - /// slot. Power-of-two length. - index: Vec, - announced: usize, - /// The same for every multiple `k` of `STRIDE` that `pow` covers. - coarse: Vec, + /// The announced exponents. + index: Index, + /// Every multiple of `STRIDE` that `pow` covers. + coarse: Index, } /// Spacing of the coarse index, and so the most probes a missed lookup costs. const STRIDE: usize = 64; +/// Exponents by their g-power, open-addressed. A zero key marks an empty slot: no +/// g-power is zero. +/// +/// Bit `t` of `tags[s]` is set when a key mixing to home slot `s` and tag `t`, three +/// more bits of its mix, is in. A lookup of an absent key almost always misses its +/// tag and reads no slot, which is what a [`GPow::log`] that resolves does on all +/// but one of its probes: the tags are a sixteenth of the slots' size, so they stay +/// in cache where the slots do not. +struct Index { + slots: Vec<(u64, u32)>, + tags: Vec, + len: usize, +} + +impl Index { + fn new(slots: usize) -> Self { + Self { + slots: vec![(0, 0); slots], + tags: vec![0; slots], + len: 0, + } + } + + /// The home slot of `key` and its tag bit. + #[inline(always)] + fn home(&self, key: u64) -> (usize, u8) { + // Bits 32..64 of the mix are the well-distributed ones. + let mix = (key.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 32) as usize; + let slots = self.slots.len(); + (mix & (slots - 1), 1 << ((mix >> slots.trailing_zeros()) & 7)) + } + + #[inline] + fn find(&self, x: F64) -> Option { + let (mut i, tag) = self.home(x.0); + if self.tags[i] & tag == 0 { + return None; + } + let mask = self.slots.len() - 1; + loop { + let (key, k) = self.slots[i]; + if key == 0 { + return None; + } + if key == x.0 { + return Some(k); + } + i = (i + 1) & mask; + } + } + + /// Add `x = g^k`, which is not in the index yet. + fn insert(&mut self, x: F64, k: usize) { + self.reserve(self.len + 1); + self.place(x.0, k as u32); + self.len += 1; + } + + /// Make room for `n` entries, staying at most half full. + fn reserve(&mut self, n: usize) { + if 2 * n > self.slots.len() { + let slots = (2 * n).next_power_of_two(); + self.tags = vec![0; slots]; + for (key, k) in std::mem::replace(&mut self.slots, vec![(0, 0); slots]) { + if key != 0 { + self.place(key, k); + } + } + } + } + + fn place(&mut self, key: u64, k: u32) { + let (mut i, tag) = self.home(key); + self.tags[i] |= tag; + let mask = self.slots.len() - 1; + while self.slots[i].0 != 0 { + i = (i + 1) & mask; + } + self.slots[i] = (key, k); + } +} + impl GPow { /// Seeded with `g^0 = 1`, grown to cover index `upto`, nothing announced. pub fn new(upto: usize) -> Self { let mut g = Self { pow: vec![F64::ONE], - index: vec![0; 1 << 13], - announced: 0, - coarse: Vec::new(), + index: Index::new(1 << 13), + coarse: Index::new(1 << 4), }; + g.coarse.insert(F64::ONE, 0); g.extend(upto); g } @@ -106,13 +187,10 @@ impl GPow { p = mul_by_g(p); p })); - let (mut from, to) = (n.div_ceil(STRIDE), self.pow.len().div_ceil(STRIDE)); - if 2 * to > self.coarse.len() { - self.coarse = vec![0; (2 * to).next_power_of_two()]; - from = 0; - } - for j in from..to { - insert(&mut self.coarse, self.pow[j * STRIDE], j * STRIDE); + let to = self.pow.len().div_ceil(STRIDE); + self.coarse.reserve(to); + for j in n.div_ceil(STRIDE)..to { + self.coarse.insert(self.pow[j * STRIDE], j * STRIDE); } } @@ -120,18 +198,10 @@ impl GPow { /// is about to store it in memory as an address. Covers `k` first. Idempotent. pub fn note(&mut self, k: usize) { self.grow_to(k); - if find(&self.index, &self.pow, self.pow[k]).is_some() { - return; + let x = self.pow[k]; + if self.index.find(x).is_none() { + self.index.insert(x, k); } - if 2 * (self.announced + 1) > self.index.len() { - let wider = vec![0; 2 * self.index.len()]; - let old = std::mem::replace(&mut self.index, wider); - for e in old.into_iter().filter(|&e| e != 0) { - insert(&mut self.index, self.pow[e as usize - 1], e as usize - 1); - } - } - insert(&mut self.index, self.pow[k], k); - self.announced += 1; } /// The discrete log of `x`, if it is a covered g-power. @@ -143,7 +213,7 @@ impl GPow { /// The exponent of `x` if announced: any covered one, for a table built by [`Self::dense`]. #[inline] pub(crate) fn lookup(&self, x: F64) -> Option { - find(&self.index, &self.pow, x) + self.index.find(x) } /// `log` for an address that is not announced: divide by `g` down to a @@ -152,7 +222,7 @@ impl GPow { fn resolve(&mut self, x: F64) -> Option { let mut y = x; for j in 0..STRIDE { - if let Some(k) = find(&self.coarse, &self.pow, y) { + if let Some(k) = self.coarse.find(y) { let k = k as usize + j; return (k < self.pow.len()).then(|| { self.note(k); @@ -165,31 +235,6 @@ impl GPow { } } -/// The `k` of a slot of `table` that holds `x = pow[k]`. -#[inline] -fn find(table: &[u32], pow: &[F64], x: F64) -> Option { - let mask = table.len() - 1; - let mut i = slot_of(x.0, mask); - while table[i] != 0 { - let k = table[i] - 1; - if pow[k as usize] == x { - return Some(k); - } - i = (i + 1) & mask; - } - None -} - -/// Put `k` in the first free slot of `table` from where `g^k = key` mixes to. -fn insert(table: &mut [u32], key: F64, k: usize) { - let mask = table.len() - 1; - let mut i = slot_of(key.0, mask); - while table[i] != 0 { - i = (i + 1) & mask; - } - table[i] = k as u32 + 1; -} - /// The inverse of [`mul_by_g`]: bit 0 of its output says whether it folded. #[inline(always)] fn div_by_g(a: F64) -> F64 { @@ -200,13 +245,6 @@ fn div_by_g(a: F64) -> F64 { }) } -/// Bits 32..64 of the mix are the well-distributed ones; the mask keeps as many -/// of them as the table is wide. -#[inline(always)] -fn slot_of(key: u64, mask: usize) -> usize { - (key.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 32) as usize & mask -} - /// Where a computed-advice bit buffer lives. Frame cells are ordinary memory, so /// a run of them serves as well as a heap region and costs no `DEREF` to index at /// a compile-time offset; the heap form stays for a buffer indexed at run time. From 46bba9b9548cf56f594aa51b98518c398c50a6b0 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Mon, 9 Nov 2026 11:06:40 +0000 Subject: [PATCH 11/12] tables: fill each table's columns in one pass over its rows Every table filled its columns in four to six parallel passes over its trace rows (PC, FP, operands and words, counts, bytecode count), each re-reading the rows and re-decoding the instruction; KECCAK took six. FillCtx::cols now takes the list of local columns it writes, so each table fills all its columns in one dispatch and one decode per row; KECCAK lists every column but the input lanes, which are q_keccak's. JUMP's is-nonzero witness w = 1/c is a Montgomery batch inversion per line of eight rows inside the same pass, where it was a serial pass over all rows building three temporary arrays. Rows::iter is gone (no user left). Byte-identical: PROOFDIGEST c7afa387f98cdb98, recursion 729f46975ab551c1. Measured against lv-head4 (spancpu.sh 5 rounds, median process CPU, xmss 900 rate 1): Fill columns 0.266 s (parent commit) -> 0.235 s (-12%). Since #724 the KECCAK row also carries its OUT_TWO and OUT_ALL flags, and its output lanes are the input lanes permuted again (a row may not write its output cells); the one pass forms both per row, as main's separate passes did. Rebased onto e5d75f858 (after #724): the digests and measurements above were taken on the older base 8577d1e21. On the new base the series leaves the proofs byte-identical: PROOFDIGEST 7d91b8513bdd4b03 (aggregate --xmss 900 --log-inv-rate 1), 3c41e153f8b2ee37 (aggregate --sphincs 410 --log-inv-rate 1), aa4b2ebd7a6989b2 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). --- crates/leanvm_core/src/cpu/trace.rs | 4 - crates/leanvm_core/src/tables.rs | 303 ++++++++++++++-------------- 2 files changed, 155 insertions(+), 152 deletions(-) diff --git a/crates/leanvm_core/src/cpu/trace.rs b/crates/leanvm_core/src/cpu/trace.rs index 5508fe2c76..9c63a53020 100644 --- a/crates/leanvm_core/src/cpu/trace.rs +++ b/crates/leanvm_core/src/cpu/trace.rs @@ -91,10 +91,6 @@ impl Rows { pub(crate) fn get(&self, i: usize) -> &T { &self.chunks[i >> CHUNK_LOG][i & (CHUNK - 1)] } - - pub(crate) fn iter(&self) -> impl DoubleEndedIterator { - self.chunks.iter().flatten() - } } pub(crate) struct Trace { diff --git a/crates/leanvm_core/src/tables.rs b/crates/leanvm_core/src/tables.rs index 00c648947b..68e51edde3 100644 --- a/crates/leanvm_core/src/tables.rs +++ b/crates/leanvm_core/src/tables.rs @@ -14,7 +14,7 @@ //! so a relation on machine words is written out lane by lane. A table hands its //! identities to the table sumcheck as a degree-2 form ([`Table::constraint`]). -use crate::cpu::{Drow, Jrow, Krow, Memory, Op, Rows, Srow, Trace}; +use crate::cpu::{Jrow, Memory, Op, Rows, Trace}; use crate::leaf::BusForm; use crate::leaf::Coord::{self, Col, Const, GCol, Prod}; use primitives::field::{F64, F192, mul_by_g}; @@ -183,6 +183,9 @@ pub struct FillCtx<'a> { /// buffer if the column is virtual. pub(crate) type ColumnOut<'a> = &'a mut [F64]; +/// Rows a fill gathers before streaming them out: a cache line of each column. +const LINE: usize = 8; + impl<'a> FillCtx<'a> { pub(crate) fn new( trace: &'a Trace, @@ -218,59 +221,58 @@ impl<'a> FillCtx<'a> { } } - /// Write local column `at`: `f` over the trace rows, then its padding value to the - /// end of the window. - fn col(&self, out: &mut [ColumnOut], rows: &Rows, at: usize, f: impl Fn(&R) -> F64 + Sync) { - self.cols(out, rows, at, |r| [f(r)]); - } - - /// Write the `N` local columns at `at..at + N` from one closure per row. - /// Columns fed by the same read fill together: the three lanes of a 192-bit - /// memory word are one random access, and splitting them across `N` passes - /// pays for it `N` times. + /// Write the local columns `cols` from one closure per row, then each column's + /// padding value to the end of its window. A table fills all its columns in one + /// pass: the row, its instruction and the memory words it names are read once, + /// and the three lanes of a 192-bit word are one random access. fn cols( &self, out: &mut [ColumnOut], rows: &Rows, - at: usize, + cols: [usize; N], f: impl Fn(&R) -> [F64; N] + Sync, ) { - self.cols_at(out, rows.len(), at, |i| f(rows.get(i))); + self.lines(out, rows.len(), cols, |start, len, buf| { + for j in 0..len { + let v = f(rows.get(start + j)); + for k in 0..N { + buf[k][j] = v[k]; + } + } + }); } - /// [`Self::cols`] over row indices, for values held in a side buffer rather - /// than read off the row. - fn cols_at( + /// [`Self::cols`] a line of rows at a time: `f(start, len, buf)` writes trace rows + /// `start..start + len` of column `cols[k]` to `buf[k][..len]`. + fn lines( &self, out: &mut [ColumnOut], n_rows: usize, - at: usize, - f: impl Fn(usize) -> [F64; N] + Sync, + cols: [usize; N], + f: impl Fn(usize, usize, &mut [[F64; LINE]; N]) + Sync, ) { let n = self.height; - let dst: [parallel::SendPtr; N] = std::array::from_fn(|k| { - assert_eq!(out[at + k].len(), n, "column {} has the wrong window length", at + k); - let col = at + k; + let dst: [parallel::SendPtr; N] = cols.map(|col| { + assert_eq!(out[col].len(), n, "column {col} has the wrong window length"); self.written[col / 64].fetch_or(1 << (col % 64), std::sync::atomic::Ordering::Relaxed); - parallel::SendPtr(out[at + k].as_mut_ptr()) + parallel::SendPtr(out[col].as_mut_ptr()) }); assert!(n_rows <= n, "a table's rows must fit its cube"); - let padding: [F64; N] = std::array::from_fn(|k| self.padding[at + k]); + let padding: [F64; N] = cols.map(|col| self.padding[col]); // A cache line of rows is gathered column by column, then streamed out: the // stack is read by a later pass, so its lines need not be fetched. - const LINE: usize = 8; parallel::for_each_chunk(n.div_ceil(LINE), |first, end| { let stream = primitives::stream::Stream::new(); for b in first..end { let start = b * LINE; let len = LINE.min(n - start); let mut buf = [[F64::ZERO; LINE]; N]; - for j in 0..len { - let i = start + j; - let v = if i < n_rows { f(i) } else { padding }; - for k in 0..N { - buf[k][j] = v[k]; - } + let rows = len.min(n_rows.saturating_sub(start)); + if rows > 0 { + f(start, rows, &mut buf); + } + for (column, &pad) in buf.iter_mut().zip(&padding) { + column[rows..len].fill(pad); } for (p, column) in dst.iter().zip(&buf) { // SAFETY: block `b` writes rows `start..start + len` of each of the @@ -521,14 +523,15 @@ impl Table for Arith { fn fill(&self, ctx: &FillCtx, out: &mut [ColumnOut]) { use arith::*; let rows = if self.is_xor { &ctx.trace.xor } else { &ctx.trace.mul }; - ctx.col(out, rows, PC, |r| ctx.g_at(r.pc)); - ctx.col(out, rows, FP, |r| ctx.g_at(r.fp)); - // The offsets and both operand words come out of ONE bytecode decode: split - // across passes, the row's instruction is fetched once per pass. - ctx.cols(out, rows, OA, |r| { + let cols = [ + PC, FP, OA, OB, OC, VA_LO, VA_HI, VA_TOP, VB_LO, VB_HI, VB_TOP, RA, RB, RC, RBC, + ]; + ctx.cols(out, rows, cols, |r| { let (a, b, c) = ctx.ternary_operands(r.pc); let (va, vb) = (ctx.limbs(r.fp + a), ctx.limbs(r.fp + b)); [ + ctx.g_at(r.pc), + ctx.g_at(r.fp), ctx.g_at(a), ctx.g_at(b), ctx.g_at(c), @@ -538,10 +541,12 @@ impl Table for Arith { vb[0], vb[1], vb[2], + ctx.g_at(r.ra), + ctx.g_at(r.rb), + ctx.g_at(r.rc), + ctx.g_at(r.bytecode_read), ] }); - ctx.cols(out, rows, RA, |r| [r.ra, r.rb, r.rc].map(|k| ctx.g_at(k))); - ctx.col(out, rows, RBC, |r| ctx.g_at(r.bytecode_read)); } } @@ -588,19 +593,23 @@ impl Table for SetTable { use set::*; let rows = &ctx.trace.set; // The offset and the stored immediate are the instruction's. - let imm = |r: &Srow| match ctx.prog[r.pc as usize] { - Op::Set { o, k } => (o, k), - op => unreachable!("a SET row's pc {} holds {op:?}", r.pc), - }; - ctx.col(out, rows, PC, |r| ctx.g_at(r.pc)); - ctx.col(out, rows, FP, |r| ctx.g_at(r.fp)); - ctx.col(out, rows, O, |r| ctx.g_at(imm(r).0)); - ctx.cols(out, rows, K_LO, |r| { - let k = imm(r).1; - k.to_limbs().map(F64) + ctx.cols(out, rows, [PC, FP, O, K_LO, K_HI, K_TOP, R, RBC], |r| { + let (o, k) = match ctx.prog[r.pc as usize] { + Op::Set { o, k } => (o, k), + op => unreachable!("a SET row's pc {} holds {op:?}", r.pc), + }; + let k = k.to_limbs().map(F64); + [ + ctx.g_at(r.pc), + ctx.g_at(r.fp), + ctx.g_at(o), + k[0], + k[1], + k[2], + ctx.g_at(r.r), + ctx.g_at(r.bytecode_read), + ] }); - ctx.col(out, rows, R, |r| ctx.g_at(r.r)); - ctx.col(out, rows, RBC, |r| ctx.g_at(r.bytecode_read)); } } @@ -671,36 +680,33 @@ impl Table for DerefTable { let rows = &ctx.trace.deref; // Offsets and store mode are the instruction's. Neither the store target's // address nor its value is committed. - let ins = |r: &Drow| match ctx.prog[r.pc as usize] { - Op::Deref { o1, o2, o3, mode } => (o1, o2, o3, mode), - op => unreachable!("a DEREF row's pc {} holds {op:?}", r.pc), - }; - ctx.col(out, rows, PC, |r| ctx.g_at(r.pc)); - ctx.col(out, rows, FP, |r| ctx.g_at(r.fp)); - debug_assert!( - rows.iter() - .all(|r| ctx.mem.word((r.fp + ins(r).0) as usize).as_base().is_some()), - "deref pointer must be K-valued" - ); - // The three offsets, the two mode flags, the pointer lane and the local - // word all follow from ONE bytecode decode. - ctx.cols(out, rows, O1, |r| { - let (o1, o2, o3, mode) = ins(r); + let cols = [PC, FP, O1, O2, O3, FPC, FFP, P, V3_LO, V3_HI, V3_TOP, R1, R2, R3, RBC]; + ctx.cols(out, rows, cols, |r| { + let (o1, o2, o3, mode) = match ctx.prog[r.pc as usize] { + Op::Deref { o1, o2, o3, mode } => (o1, o2, o3, mode), + op => unreachable!("a DEREF row's pc {} holds {op:?}", r.pc), + }; + let p = ctx.mem.word((r.fp + o1) as usize); + debug_assert!(p.as_base().is_some(), "deref pointer must be K-valued"); let v3 = ctx.limbs(r.fp + o3); [ + ctx.g_at(r.pc), + ctx.g_at(r.fp), ctx.g_at(o1), ctx.g_at(o2), ctx.g_at(o3), mode.f_pc(), mode.f_fp(), - F64(ctx.mem.word((r.fp + o1) as usize).c0), + F64(p.c0), v3[0], v3[1], v3[2], + ctx.g_at(r.r1), + ctx.g_at(r.r2), + ctx.g_at(r.r3), + ctx.g_at(r.bytecode_read), ] }); - ctx.cols(out, rows, R1, |r| [r.r1, r.r2, r.r3].map(|k| ctx.g_at(k))); - ctx.col(out, rows, RBC, |r| ctx.g_at(r.bytecode_read)); } } @@ -792,60 +798,62 @@ impl Table for JumpTable { op => unreachable!("a JUMP row's pc {} holds {op:?}", r.pc), }; let cell = |r: &Jrow, o: u32| ctx.mem.word((r.fp + o) as usize); - let cond = |r: &Jrow| { - let c = cell(r, ins(r).0); - debug_assert!(c.as_base().is_some(), "a JUMP condition outside K at pc {}", r.pc); - F64(c.c0) - }; - ctx.col(out, rows, PC, |r| ctx.g_at(r.pc)); - ctx.col(out, rows, FP, |r| ctx.g_at(r.fp)); - // The three offsets and the three cells they name come out of ONE decode. - // Those cells are K-valued on every row, taken or not (`cpu::execute` - // rejects anything else), so each is one lane and the memory flush carries - // literal zeros above it. - ctx.cols(out, rows, OC, |r| { - let (oc, od, of) = ins(r); - [ - ctx.g_at(oc), - ctx.g_at(od), - ctx.g_at(of), - F64(cell(r, oc).c0), - F64(cell(r, od).c0), - F64(cell(r, of).c0), - ] - }); - // The is-nonzero witness `w = c⁻¹` (0 where c = 0) for every row, in one - // batched Montgomery inversion. `prefix[i]` is the - // running product of the nonzero conditions before row `i`, so `acc` ends - // as their full product (nonzero, hence invertible). The taken indicator - // `b = [c ≠ 0]` falls out of the same pass, so it costs no extra decode. - // The conditions lie in K, a subfield, so the inversion runs in K. - let (w, b) = { + // Every column, in local order, so `buf[c]` is column `c`. + let cols: [usize; N] = std::array::from_fn(|c| c); + ctx.lines(out, rows.len(), cols, |start, len, buf| { + // The is-nonzero witness `w = c⁻¹` (0 where c = 0) by one batched Montgomery + // inversion per line: `prefix[j]` is the running product of the nonzero + // conditions before row `j`, so `acc` ends as their full product (nonzero, + // hence invertible). The conditions lie in K, a subfield, so the inversion + // runs in K. + let mut prefix = [F64::ONE; LINE]; let mut acc = F64::ONE; - let mut prefix: Vec = Vec::with_capacity(rows.len()); - let mut b = vec![F64::ZERO; rows.len()]; - for (i, r) in rows.iter().enumerate() { - prefix.push(acc); - let c = cond(r); - if !c.is_zero() { + for j in 0..len { + let r = rows.get(start + j); + let (oc, od, of) = ins(r); + // The three cells are K-valued on every row, taken or not + // (`cpu::execute` rejects anything else), so each is one lane and the + // memory flush carries literal zeros above it. + let c = cell(r, oc); + debug_assert!(c.as_base().is_some(), "a JUMP condition outside K at pc {}", r.pc); + let c = F64(c.c0); + let row = [ + (PC, ctx.g_at(r.pc)), + (FP, ctx.g_at(r.fp)), + (OC, ctx.g_at(oc)), + (OD, ctx.g_at(od)), + (OF, ctx.g_at(of)), + (V_COND, c), + (V_PC, F64(cell(r, od).c0)), + (V_FP, F64(cell(r, of).c0)), + (RC, ctx.g_at(r.rc)), + (RD, ctx.g_at(r.rd)), + (RF, ctx.g_at(r.rf)), + (RBC, ctx.g_at(r.bytecode_read)), + ]; + for (col, v) in row { + buf[col][j] = v; + } + prefix[j] = acc; + // The taken indicator `b = [c ≠ 0]`. + let taken = !c.is_zero(); + buf[B][j] = F64(u64::from(taken)); + if taken { acc *= c; - b[i] = F64::ONE; } } let mut inv = acc.inv(); - let mut w = vec![F64::ZERO; rows.len()]; - for i in (0..rows.len()).rev() { - let c = cond(rows.get(i)); - if !c.is_zero() { - w[i] = inv * prefix[i]; + for j in (0..len).rev() { + let c = buf[V_COND][j]; + buf[W][j] = if c.is_zero() { + F64::ZERO + } else { + let w = inv * prefix[j]; inv *= c; - } + w + }; } - (w, b) - }; - ctx.cols_at(out, rows.len(), W, |i| [w[i], b[i]]); - ctx.cols(out, rows, RC, |r| [r.rc, r.rd, r.rf].map(|k| ctx.g_at(k))); - ctx.col(out, rows, RBC, |r| ctx.g_at(r.bytecode_read)); + }); } } @@ -962,44 +970,43 @@ impl Table for KeccakTable { /// Every column but the input lanes, which are `q_keccak`'s and have no window here. The output lanes are the /// input permuted again, since a row writes only the output cells its bytecode requests. fn fill(&self, ctx: &FillCtx, out: &mut [ColumnOut]) { - use crate::keccak::state_of_cells; + use crate::keccak::{ROW_CELLS, state_of_cells}; use keccakt::*; + use primitives::hash::STATE_LANES; let rows = &ctx.trace.keccak; - let op = |r: &Krow| KeccakOp::of(&ctx.prog[r.pc as usize]); let flag = |b: bool| F64(u64::from(b)); - ctx.col(out, rows, PC, |r| ctx.g_at(r.pc)); - ctx.col(out, rows, FP, |r| ctx.g_at(r.fp)); - ctx.cols(out, rows, O_M0, |r| { - let k = op(r); - let [m0, m1, m2, m3, tail, b8, cap, o] = k.offsets.map(|o| ctx.g_at(o)); - let (cell, high) = FREE_LIMB[usize::from(k.in_odd)]; - let free = ctx.mem.word(k.cell(r.fp, cell) as usize); - let free = F64(if high { free.c1 } else { free.c0 }); - let (two, all) = k.out_cells.flags(); - [ - m0, - m1, - m2, - m3, - tail, - b8, - cap, - o, - flag(k.in_odd), - flag(k.out_odd), - flag(two), - flag(all), - free, - ] - }); - ctx.cols(out, rows, V_OUT, |r| { - let k = op(r); - let mut lanes = state_of_cells(k.in_odd, |c| ctx.mem.word(k.cell(r.fp, c) as usize)); - primitives::hash::keccak_f(&mut lanes); - lanes.map(F64) + // Every column but the input lanes, in local order: `buf[c]` is column `c` below + // `V_IN`, and column `c + STATE_LANES` from there. + const COLS: usize = N - STATE_LANES; + let cols: [usize; COLS] = std::array::from_fn(|c| if c < V_IN { c } else { c + STATE_LANES }); + ctx.lines(out, rows.len(), cols, |start, len, buf| { + for j in 0..len { + let r = rows.get(start + j); + let k = KeccakOp::of(&ctx.prog[r.pc as usize]); + buf[PC][j] = ctx.g_at(r.pc); + buf[FP][j] = ctx.g_at(r.fp); + for (i, &o) in k.offsets.iter().enumerate() { + buf[O_M0 + i][j] = ctx.g_at(o); + } + let (two, all) = k.out_cells.flags(); + buf[IN_ODD][j] = flag(k.in_odd); + buf[OUT_ODD][j] = flag(k.out_odd); + buf[OUT_TWO][j] = flag(two); + buf[OUT_ALL][j] = flag(all); + let (cell, high) = FREE_LIMB[usize::from(k.in_odd)]; + let free = ctx.mem.word(k.cell(r.fp, cell) as usize); + buf[FREE][j] = F64(if high { free.c1 } else { free.c0 }); + let mut lanes = state_of_cells(k.in_odd, |c| ctx.mem.word(k.cell(r.fp, c) as usize)); + primitives::hash::keccak_f(&mut lanes); + for (l, &lane) in lanes.iter().enumerate() { + buf[V_OUT - STATE_LANES + l][j] = F64(lane); + } + for c in 0..ROW_CELLS { + buf[R0 - STATE_LANES + c][j] = ctx.g_at(r.r[c]); + } + buf[RBC - STATE_LANES][j] = ctx.g_at(r.bytecode_read); + } }); - ctx.cols(out, rows, R0, |r| r.r.map(|k| ctx.g_at(k))); - ctx.col(out, rows, RBC, |r| ctx.g_at(r.bytecode_read)); } } From 0fc4679d7d28fb3dc760daccabd81f1a22870e71 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Mon, 9 Nov 2026 11:14:17 +0000 Subject: [PATCH 12/12] hints: forward g-powers as strides times a first stride; the fill builds the table The interpreter grew a table of every power g^0..g^k as far as the run addressed (about 4M entries, 32 MiB, plus the copies of its doublings), serially and on the critical path, and handed it to the fill. GPow now keeps g^(64j) for the covered strides and g^0..g^63, so g^k is one product; the coarse index is built from the strides as before. Program::build makes the fill's table with primitives::field::g_powers, across the pool. Execution::gpow is gone. Byte-identical: PROOFDIGEST c7afa387f98cdb98, recursion 729f46975ab551c1; cargo test --release -p leanvm_core -p lean_compiler pass. Measured (solo execute, one thread, xmss 900, interleaved, min): cycles per execute in GPow, hints and inverses 56M -> 47M; GPow::extend is gone from the profile. Whole series against lv-head4, xmss 900 rate 1 (loaded host, load average 80 to 800 during the runs): - traced --repeat 1, 6 interleaved runs: Execute program min 142 -> 124 ms, median 204 -> 160 ms; Build witness min 40 -> 33 ms, median 54 -> 36 ms; Fill columns min 34 -> 25 ms, median 46 -> 29 ms. - spancpu.sh 6 rounds (median process CPU): Fill columns 0.274 -> 0.245 s, Build witness 0.302 -> 0.296 s, Execute program 0.280 -> 0.301 s (the span's CPU includes the Keccak block workers; within noise), Prove 8.98 -> 8.69 s. - ab.sh 3x4, CPU per pass: min 8.37 -> 8.25 s, median 8.58 -> 8.70 s (no whole-pass CPU change beyond noise); wall min 1.195 -> 0.929 s, median 1.383 -> 1.172 s. Rebased onto e5d75f858 (after #724): the digests and measurements above were taken on the older base 8577d1e21. On the new base the series leaves the proofs byte-identical: PROOFDIGEST 7d91b8513bdd4b03 (aggregate --xmss 900 --log-inv-rate 1), 3c41e153f8b2ee37 (aggregate --sphincs 410 --log-inv-rate 1), aa4b2ebd7a6989b2 (recursion --n 2 --xmss-per-leaf 900 --log-inv-rate 2). --- crates/leanvm_core/src/cpu/execute.rs | 3 - crates/leanvm_core/src/cpu/hints.rs | 82 ++++++++++++++------------- crates/leanvm_core/src/cpu/layout.rs | 7 +-- 3 files changed, 44 insertions(+), 48 deletions(-) diff --git a/crates/leanvm_core/src/cpu/execute.rs b/crates/leanvm_core/src/cpu/execute.rs index b87f743d86..e7f1a1569b 100644 --- a/crates/leanvm_core/src/cpu/execute.rs +++ b/crates/leanvm_core/src/cpu/execute.rs @@ -28,8 +28,6 @@ pub struct Execution { /// and an arithmetic back-solve writes its operand before reading it. pub unconstrained_reads: Vec, pub(crate) trace: Trace, // rows + final access-count columns, emitted in the same walk - /// `g^0, g^1, ..` as far as the run indexed them: every pc, frame base and cell it reached. - pub(crate) gpow: Vec, } impl Execution { @@ -1091,7 +1089,6 @@ impl Program { mem_used, unconstrained_reads: m.unwritten_reads, trace, - gpow: g.into_powers(), }) } } diff --git a/crates/leanvm_core/src/cpu/hints.rs b/crates/leanvm_core/src/cpu/hints.rs index a964f52246..e27a10249c 100644 --- a/crates/leanvm_core/src/cpu/hints.rs +++ b/crates/leanvm_core/src/cpu/hints.rs @@ -10,19 +10,22 @@ pub type Off = u32; /// Cells a program can address: [`GPow`] covers no more. pub(crate) const MAX_CELLS: u32 = 1 << 28; -/// The `g^k` table paired with a reverse index `g^k ↦ k`, both grown on demand +/// The powers `g^k` paired with a reverse index `g^k ↦ k`, both grown on demand /// (recursion depth, and so the address range, is unbounded). /// /// The interpreter needs both directions: a cell index becomes the address `g^k`, /// and a pointer word read back out of memory must be inverted to the index it -/// addresses. Three things keep this cheap. +/// addresses. Four things keep this cheap. +/// +/// The forward powers are every `STRIDE`th one and the first `STRIDE`, so `g^k` is +/// one product: a table of every power would be as large as the address space, +/// built by the interpreter's one thread and read at random. /// /// A reverse-index slot holds the key beside its exponent, so a probe reads the -/// slot alone: confirming a candidate against the forward table, as large as the -/// address space, would cost a cache miss per probe. Keys are field elements, -/// hence effectively uniform, which is all the placement asks for: one -/// multiplicative mix, then linear probing. The table stays at most half full to -/// keep probing short. +/// slot alone rather than computing the power a candidate exponent names. Keys +/// are field elements, hence effectively uniform, which is all the placement asks +/// for: one multiplicative mix, then linear probing. The table stays at most half +/// full to keep probing short. /// /// It indexes only the exponents [`Self::note`] announces, or a lookup resolved: /// a run addresses millions of cells but stores only the few thousand bases of @@ -34,10 +37,15 @@ pub(crate) const MAX_CELLS: u32 = 1 << 28; /// interpreter could not label, a jump target or a pointer into a buffer, costs at /// most `STRIDE` probes once. pub struct GPow { - pow: Vec, + /// `g^{STRIDE·j}` for every multiple of `STRIDE` covered. + strides: Vec, + /// `g^0` to `g^{STRIDE - 1}`. + first: [F64; STRIDE], + /// Exponents below this are covered. + covered: usize, /// The announced exponents. index: Index, - /// Every multiple of `STRIDE` that `pow` covers. + /// Every multiple of `STRIDE` covered. coarse: Index, } @@ -129,8 +137,16 @@ impl Index { impl GPow { /// Seeded with `g^0 = 1`, grown to cover index `upto`, nothing announced. pub fn new(upto: usize) -> Self { + let mut p = F64::ONE; + let first = std::array::from_fn(|_| { + let q = p; + p = mul_by_g(p); + q + }); let mut g = Self { - pow: vec![F64::ONE], + strides: vec![F64::ONE], + first, + covered: 1, index: Index::new(1 << 13), coarse: Index::new(1 << 4), }; @@ -153,18 +169,14 @@ impl GPow { /// index the interpreter reads has been covered by a [`Self::grow_to`]. #[inline(always)] pub fn pow(&self, k: usize) -> F64 { - self.pow[k] - } - - /// The forward table, `g^0` to `g^{covered - 1}`. - pub(crate) fn into_powers(self) -> Vec { - self.pow + assert!(k < self.covered, "g^{k} is past the covered range"); + self.strides[k / STRIDE] * self.first[k % STRIDE] } - /// Extend the forward table to cover index `upto`. + /// Cover index `upto`. #[inline] pub fn grow_to(&mut self, upto: usize) { - if upto >= self.pow.len() { + if upto >= self.covered { self.extend(upto); } } @@ -172,33 +184,22 @@ impl GPow { #[cold] fn extend(&mut self, upto: usize) { assert!(upto < MAX_CELLS as usize, "address space overflow (program too large)"); - // At least double, and a quarter more than a large jump asks: the small - // extends that follow fill spare capacity instead of copying the table. - let n = self.pow.len(); - if upto >= self.pow.capacity() { - let target = (2 * self.pow.capacity()) - .max(upto + 1 + (upto + 1) / 4) - .min(MAX_CELLS as usize); - self.pow.reserve_exact(target - n); - } - // ×g is ×x = `mul_by_g` (shift+fold), not a PMULL. - let mut p = *self.pow.last().unwrap(); - self.pow.extend((n..=upto).map(|_| { - p = mul_by_g(p); - p - })); - let to = self.pow.len().div_ceil(STRIDE); + let step = mul_by_g(self.first[STRIDE - 1]); + let to = upto / STRIDE + 1; self.coarse.reserve(to); - for j in n.div_ceil(STRIDE)..to { - self.coarse.insert(self.pow[j * STRIDE], j * STRIDE); + for j in self.strides.len()..to { + let p = self.strides[j - 1] * step; + self.strides.push(p); + self.coarse.insert(p, j * STRIDE); } + self.covered = upto + 1; } /// Announce that `g^k` may come back for inversion, because the interpreter /// is about to store it in memory as an address. Covers `k` first. Idempotent. pub fn note(&mut self, k: usize) { self.grow_to(k); - let x = self.pow[k]; + let x = self.pow(k); if self.index.find(x).is_none() { self.index.insert(x, k); } @@ -217,15 +218,16 @@ impl GPow { } /// `log` for an address that is not announced: divide by `g` down to a - /// multiple of `STRIDE`, then announce the exponent. + /// multiple of `STRIDE`, then announce the exponent. Called on a lookup's miss, + /// so `x` is not in the index. #[cold] fn resolve(&mut self, x: F64) -> Option { let mut y = x; for j in 0..STRIDE { if let Some(k) = self.coarse.find(y) { let k = k as usize + j; - return (k < self.pow.len()).then(|| { - self.note(k); + return (k < self.covered).then(|| { + self.index.insert(x, k); k as u32 }); } diff --git a/crates/leanvm_core/src/cpu/layout.rs b/crates/leanvm_core/src/cpu/layout.rs index ab2e81f1d7..985ac9b79b 100644 --- a/crates/leanvm_core/src/cpu/layout.rs +++ b/crates/leanvm_core/src/cpu/layout.rs @@ -525,12 +525,9 @@ impl Program { let sch = schema(); // g^0..g^{span-1}, so every pc/frame/operand/count fill is an O(1) lookup instead // of an O(log) power: a frame base or an operand is below the cells the run used. - // The run's own table covers them unless a count outgrew it. + // Built here across the pool rather than by the run's one thread. let span = exec.mem_used.max(bytecode_size).max(tr.counts); - let gpow = match exec.gpow.get(..span) { - Some(gpow) => std::borrow::Cow::Borrowed(gpow), - None => std::borrow::Cow::Owned(primitives::field::g_powers(span)), - }; + let gpow = primitives::field::g_powers(span); // The public layout (flush/count blocks, placements, boundary, taus) is a pure // function of the program + announced sizes + public input, with no committed