diff --git a/crates/keccak_gkr/src/kernels.rs b/crates/keccak_gkr/src/kernels.rs index 79bf3fdb48..c2c2c5586e 100644 --- a/crates/keccak_gkr/src/kernels.rs +++ b/crates/keccak_gkr/src/kernels.rs @@ -5,9 +5,11 @@ //! (PCLMULQDQ, and VPCLMULQDQ where it is) and a NEON one (PMULL), which agree bit for bit. The //! vector forms are compiled only where their features are. +use primitives::field::F192; use primitives::field::gf2_64::mul_wide; use crate::keccak::row_positions; +use crate::power::{Element, PowerField}; use crate::rmfe::RMFE; use crate::witness::BLOCK_WORDS; @@ -258,9 +260,18 @@ cfg_select! { target_feature = "vpclmulqdq", target_feature = "gfni" ) => { - pub(crate) use x86::SkipBlock; + pub(crate) use x86::{SkipBlock, SkipWeights}; } _ => { + /// The position weights a block's [`SkipBlock::run`] reads, in the power basis. + pub(crate) struct SkipWeights(Vec); + + impl SkipWeights { + pub fn new(field: &PowerField, weights: &[F192]) -> Self { + Self(weights.iter().map(|&w| field.to_poly(w)).collect()) + } + } + /// [`skip_row`] over every row and slice of a block, a position at a time. pub(crate) struct SkipBlock { acc: Box, @@ -276,7 +287,7 @@ cfg_select! { } /// Sum a block, the position weights as elements. - pub fn run(&mut self, block: &[u64], weights: &[[u64; 3]]) { + pub fn run(&mut self, block: &[u64], weights: &SkipWeights) { self.acc.clear(); for r in 0..5 { for z in 0..64 { @@ -284,12 +295,13 @@ cfg_select! { for x in 0..5 { embed_extend(crate::witness::words(block, js[x]), &mut self.extended[x]); } - skip_row(&mut self.acc, &self.extended, std::array::from_fn(|x| &weights[js[x]])); + skip_row(&mut self.acc, &self.extended, std::array::from_fn(|x| &weights.0[js[x]])); } } } - /// The three partial products of slot `i`. + /// The three partial products of slot `i`, a polynomial congruent to the slot's sum + /// modulo the minimal polynomial. pub fn get(&self, i: usize) -> [u128; 3] { self.acc.get(i) } @@ -417,11 +429,10 @@ mod x86 { } } - /// The packs of eight consecutive positions, embedded: `out[q]` holds pack `q` of position - /// `i` in word `i`. The positions' words are transposed first, so a register holds one word - /// of each, and a pack is then a field of every word alike. + /// The words of eight consecutive positions, transposed: register `w` holds word `w` of + /// position `i` in its word `i`, so a pack is a field of every word alike. #[inline(always)] - unsafe fn embed8(positions: *const u64, out: &mut [__m512i; PACKS]) { + unsafe fn transpose_words(positions: *const u64) -> [__m512i; BLOCK_WORDS] { unsafe { let a: [__m512i; 8] = std::array::from_fn(|i| _mm512_loadu_si512(positions.add(BLOCK_WORDS * i).cast())); let b: [__m512i; 8] = std::array::from_fn(|i| { @@ -441,7 +452,7 @@ mod x86 { let (x, y) = (b[4 * m + t % 2], b[4 * m + 2 + t % 2]); if t < 2 { low(x, y) } else { high(x, y) } }); - let words = [ + [ low(c[0], c[4]), low(c[1], c[5]), low(c[2], c[6]), @@ -450,7 +461,15 @@ mod x86 { high(c[1], c[5]), high(c[2], c[6]), high(c[3], c[7]), - ]; + ] + } + } + + /// The packs of some of [`transpose_words`]'s words, embedded: `out[4 w + t]` holds pack + /// `t` of `words[w]`, of position `i` in word `i`. + #[inline(always)] + unsafe fn embed_words(words: &[__m512i], out: &mut [__m512i]) { + unsafe { let zero = _mm512_setzero_si512(); for (w, &v) in words.iter().enumerate() { let low = _mm512_or_si512(embedding_byte(v, 0), _mm512_slli_epi16(embedding_byte(v, 1), 8)); @@ -466,110 +485,321 @@ mod x86 { } } - /// [`extend_group`] on registers, each of eight positions' values. + /// The bytes of the eight packs of two of [`transpose_words`]'s words: `out[h]` holds byte + /// `h` of pack `k` of position `i` in byte `k` of word `i`. #[inline(always)] - unsafe fn extend8(v: &[__m512i], out: &mut [__m512i; MIXED]) { + unsafe fn pack_bytes(words: [__m512i; 2]) -> [__m512i; 2] { unsafe { - let x = |a: [__m512i; 4], b: [__m512i; 4]| -> [__m512i; 4] { - std::array::from_fn(|l| _mm512_xor_si512(a[l], b[l])) - }; - let q = |k: usize| -> [__m512i; 4] { std::array::from_fn(|l| v[4 * k + l]) }; - let (w4, w5) = (x(q(0), q(1)), x(q(2), q(3))); - let w = [q(0), q(1), q(2), q(3), w4, w5, x(q(0), q(2)), x(q(1), q(3)), x(w4, w5)]; - for (k, w) in w[4..].iter().enumerate() { - out[4 * k..][..4].copy_from_slice(w); + // Per word, its four low bytes then its four high ones, and the other way round. + let split = _mm512_broadcast_i32x4(_mm_setr_epi8(0, 2, 4, 6, 1, 3, 5, 7, 8, 10, 12, 14, 9, 11, 13, 15)); + let swapped = _mm512_broadcast_i32x4(_mm_setr_epi8(1, 3, 5, 7, 0, 2, 4, 6, 9, 11, 13, 15, 8, 10, 12, 14)); + let a = _mm512_shuffle_epi8(words[0], split); + let b = _mm512_shuffle_epi8(words[1], swapped); + let low = _mm512_mask_blend_epi32(0xaaaa, a, b); + let high = _mm512_mask_blend_epi32(0xaaaa, b, a); + [low, _mm512_shuffle_epi32::<0b10_11_00_01>(high)] + } + } + + /// Per word, an 8x8 transpose of bytes across the registers: `out[k]` byte `u` of word `i` is + /// `r[u]` byte `k` of word `i`. + #[inline(always)] + unsafe fn transpose_bytes(r: [__m512i; 8]) -> [__m512i; 8] { + unsafe { + // A 128-bit lane `m` holds word `2 m + parity` of each: `a[2 u + parity]` 16-bit field + // `k` is bytes `2 u, 2 u + 1` of `k`. + let a: [__m512i; 8] = std::array::from_fn(|i| { + let (x, y) = (r[2 * (i / 2)], r[2 * (i / 2) + 1]); + if i % 2 == 0 { + _mm512_unpacklo_epi8(x, y) + } else { + _mm512_unpackhi_epi8(x, y) + } + }); + // `b[4 v + 2 h + parity]` 32-bit field `k` is bytes `4 v ..` of `4 h + k`. + let b: [__m512i; 8] = std::array::from_fn(|i| { + let (v, h, parity) = (i / 4, i / 2 % 2, i % 2); + let (x, y) = (a[4 * v + parity], a[4 * v + 2 + parity]); + if h == 0 { + _mm512_unpacklo_epi16(x, y) + } else { + _mm512_unpackhi_epi16(x, y) + } + }); + // `c[4 h + 2 p + parity]` 64-bit field `t` is all of `4 h + 2 p + t`. + let c: [__m512i; 8] = std::array::from_fn(|i| { + let (h, p, parity) = (i / 4, i / 2 % 2, i % 2); + let (x, y) = (b[2 * h + parity], b[4 + 2 * h + parity]); + if p == 0 { + _mm512_unpacklo_epi32(x, y) + } else { + _mm512_unpackhi_epi32(x, y) + } + }); + std::array::from_fn(|k| { + let (x, y) = (c[k & !1], c[k | 1]); + if k % 2 == 0 { + _mm512_unpacklo_epi64(x, y) + } else { + _mm512_unpackhi_epi64(x, y) + } + }) + } + } + + /// Bytes of an element. + const ELEMENT_BYTES: usize = 24; + /// The matrices of a position. + const WEIGHT_MATRICES: usize = 2 * ELEMENT_BYTES; + + /// Per position, the affine byte maps sending byte `h` of a pack to byte `b` of the + /// position's weight times the pack's embedding, reduced modulo the minimal polynomial: + /// matrix `m = ELEMENT_BYTES * h + b` of position `j` at + /// `WEIGHT_MATRICES * 8 * (j / 8) + 8 * m + j % 8`, so that eight consecutive positions' are + /// one register, and theirs are contiguous. + pub struct SkipWeights(Vec); + + impl SkipWeights { + pub fn new(field: &PowerField, weights: &[F192]) -> Self { + assert!(weights.len() >= BITS); + let per_position = parallel::map_collect(BITS, |j| { + let w = field.to_poly(weights[j]); + let v: [Element; 16] = std::array::from_fn(|t| field.mul(&w, &[RMFE.emb[t] as u64, 0, 0])); + std::array::from_fn::(|m| { + let (h, b) = (m / ELEMENT_BYTES, m % ELEMENT_BYTES); + // Byte `i` is byte `b` of input bit `i`'s image; an affine byte map reads + // output bit `u` off matrix byte `7 - u`. + let images = (0..8).fold(0u64, |q, i| { + q | (v[8 * h + i][b / 8] >> (8 * (b % 8)) & 0xff) << (8 * i) + }); + transpose_bits8x8(images).swap_bytes() + }) + }); + let mut matrices = vec![0u64; WEIGHT_MATRICES * BITS]; + for (j, m) in per_position.iter().enumerate() { + for (k, &m) in m.iter().enumerate() { + matrices[WEIGHT_MATRICES * 8 * (j / 8) + 8 * k + j % 8] = m; + } } - for (k, w) in w.iter().enumerate() { - let low = x([w[0], w[2], w[0], w[1]], [w[1], w[3], w[2], w[3]]); - out[20 + 4 * k..][..4].copy_from_slice(&low); - out[56 + k] = _mm512_xor_si512(low[0], low[1]); + Self(matrices) + } + } + + /// Bit `8 c + r` of `x` at bit `8 r + c`. + const fn transpose_bits8x8(mut x: u64) -> u64 { + let t = (x ^ (x >> 7)) & 0x00aa_00aa_00aa_00aa; + x ^= t ^ (t << 7); + let t = (x ^ (x >> 14)) & 0x0000_cccc_0000_cccc; + x ^= t ^ (t << 14); + let t = (x ^ (x >> 28)) & 0x0000_0000_f0f0_f0f0; + x ^ t ^ (t << 28) + } + + /// One octet of a pivot's weighted packs into `out[k][8 o ..]`, per word `k`: `sum over terms + /// of weights[x] * embed(pack)`, a term being position `x`'s matrices and the bytes of the + /// other factor's packs ([`pack_bytes`]). + #[inline(always)] + unsafe fn weigh(terms: [(*const u64, [__m512i; 2]); N], out: &mut [[__m512i; GROUP]], o: usize) { + unsafe { + for (k, out) in out[..3].iter_mut().enumerate() { + let image = transpose_bytes(std::array::from_fn(|u| { + terms.iter().fold(_mm512_setzero_si512(), |g, &(m, [low, high])| { + let matrix = |h: usize| _mm512_loadu_si512(m.add(8 * (ELEMENT_BYTES * h + 8 * k + u)).cast()); + _mm512_ternarylogic_epi64( + g, + _mm512_gf2p8affine_epi64_epi8(low, matrix(0), 0), + _mm512_gf2p8affine_epi64_epi8(high, matrix(1), 0), + 0x96, + ) + }) + })); + for (out, image) in out[8 * o..][..8].iter_mut().zip(image) { + *out = image; + } } } } + /// The quantities of a half block, by pivot: its packs embedded, then each word of its + /// weighted packs, over the group's 16 packs. + type Quantities = [[__m512i; GROUP]; 12]; + + /// A quantity's value at high point `K` of [`extend_group`] and low lane `l`. + #[inline(always)] + unsafe fn high(q: &[__m512i; GROUP], l: usize) -> __m512i { + unsafe { + let (g, xor) = (|m: usize| q[4 * m + l], _mm512_xor_si512); + match K { + 0..4 => g(K), + 4 => xor(g(0), g(1)), + 5 => xor(g(2), g(3)), + 6 => xor(g(0), g(2)), + 7 => xor(g(1), g(3)), + _ => _mm512_ternarylogic_epi64(xor(g(0), g(1)), g(2), g(3), 0x96), + } + } + } + + /// Shapes of [`values`]: of a high point's lanes `w`, `w0 + w1, w2 + w3` and their sum; `w0, + /// w1` and theirs; `w2, w3` and theirs; `w0 + w2, w1 + w3` and theirs. + const LOW_SUMS: usize = 0; + const FIRST: usize = 1; + const SECOND: usize = 2; + const CROSS: usize = 3; + + /// A quantity's values at `S` outputs of high point `K`, of the given shape. + #[inline(always)] + unsafe fn values(q: &[__m512i; GROUP]) -> [__m512i; S] { + unsafe { + let w = |l: usize| high::(q, l); + let xor = _mm512_xor_si512; + let (a, b) = match SHAPE { + LOW_SUMS => (xor(w(0), w(1)), xor(w(2), w(3))), + FIRST => (w(0), w(1)), + SECOND => (w(2), w(3)), + _ => (xor(w(0), w(2)), xor(w(1), w(3))), + }; + let v = [a, b, xor(a, b)]; + std::array::from_fn(|s| v[s]) + } + } + /// [`skip_row`] over every row and slice of a block, eight slices at a time: each register /// holds one value of eight positions, so the extension is a sum of registers and each slot's - /// products are a register's lanes, weighted by eight positions' weights at once. + /// products are a register's lanes. + /// + /// A row's five products `weights[x] * e[x + 1] * e[x + 2]` go by three pivots, each a + /// factor its products share: `e[2] * (weights[0] * e[1] + weights[1] * e[3])`, + /// `e[4] * (weights[2] * e[3] + weights[3] * e[0])` and `e[1] * (weights[4] * e[0])`. A + /// weight times an embedding is linear in the pack's bits, so [`SkipWeights`]' affine byte + /// maps give the weighted packs, reduced to three words, and the extension is a sum of those + /// too. A slot then takes three carry-less products per pivot, where weighing each product + /// would take four per row position. + /// + /// A half block at a time, so that its slots' sums stay in cache. The extension is built a + /// high point at a time from the group's values, and each word's sums of a high point's + /// slots stay in registers over the pivots. pub struct SkipBlock { - /// Per slot, the even and odd positions' sums of each partial product. - acc: Vec<[__m512i; 6]>, - packs: [[__m512i; PACKS]; 5], - extended: [[__m512i; MIXED]; 5], - /// The weights by word: word `k` of position `j` at `k * BITS + j`. - weights: Vec, + /// Per slot, the sums of each word's products. + acc: Vec<[__m512i; 3]>, } impl SkipBlock { pub fn new() -> Box { let zero = unsafe { _mm512_setzero_si512() }; Box::new(Self { - acc: vec![[zero; 6]; 2 * MIXED], - packs: [[zero; PACKS]; 5], - extended: [[zero; MIXED]; 5], - weights: vec![0; 3 * BITS], + acc: vec![[zero; 3]; 2 * MIXED], }) } - /// Sum a block, the position weights as elements. - pub fn run(&mut self, block: &[u64], weights: &[[u64; 3]]) { - assert!(block.len() >= BITS * BLOCK_WORDS && weights.len() >= BITS); - for (j, w) in weights[..BITS].iter().enumerate() { - for (k, &w) in w.iter().enumerate() { - self.weights[k * BITS + j] = w; - } - } + /// Sum a block. + pub fn run(&mut self, block: &[u64], weights: &SkipWeights) { + assert!(block.len() >= BITS * BLOCK_WORDS && weights.0.len() >= WEIGHT_MATRICES * BITS); // SAFETY: the features are enabled at compile time, and every position read is in - // the block, every weight in the table. + // the block, every matrix in the table. unsafe { - self.acc.fill([_mm512_setzero_si512(); 6]); - for r in 0..5 { - for z in (0..64).step_by(8) { - let js = row_positions(r, z); - for (packs, &j) in self.packs.iter_mut().zip(&js) { - embed8(block.as_ptr().add(BLOCK_WORDS * j), packs); - } - let w: [[__m512i; 3]; 5] = std::array::from_fn(|x| { - std::array::from_fn(|k| { - _mm512_loadu_si512(self.weights.as_ptr().add(k * BITS + js[x]).cast()) - }) - }); - for half in 0..2 { - for (extended, packs) in self.extended.iter_mut().zip(&self.packs) { - extend8(&packs[GROUP * half..][..GROUP], extended); + let zero = _mm512_setzero_si512(); + self.acc.fill([zero; 3]); + let mut q: Quantities = [[zero; GROUP]; 12]; + for half in 0..2 { + for r in 0..5 { + for z in (0..64).step_by(8) { + let js = row_positions(r, z); + let words: [[__m512i; GROUP / 4]; 5] = std::array::from_fn(|x| { + let words = transpose_words(block.as_ptr().add(BLOCK_WORDS * js[x])); + std::array::from_fn(|w| words[GROUP / 4 * half + w]) + }); + let m = |x: usize| weights.0.as_ptr().add(WEIGHT_MATRICES * js[x]); + for o in 0..2 { + let bytes = |x: usize| pack_bytes([words[x][2 * o], words[x][2 * o + 1]]); + let (b0, b1, b3) = (bytes(0), bytes(1), bytes(3)); + weigh([(m(0), b1), (m(1), b3)], &mut q[1..4], o); + weigh([(m(2), b3), (m(3), b0)], &mut q[5..8], o); + weigh([(m(4), b0)], &mut q[9..12], o); } - for i in 0..MIXED { - let e: [__m512i; 5] = std::array::from_fn(|x| self.extended[x][i]); - let mut sums = [_mm512_setzero_si512(); 6]; - for x in 0..5 { - let (b, c) = (e[(x + 1) % 5], e[(x + 2) % 5]); - let even = _mm512_clmulepi64_epi128(b, c, 0x00); - let odd = _mm512_clmulepi64_epi128(b, c, 0x11); - for k in 0..3 { - let (s, w) = (&mut sums[2 * k..], w[x][k]); - s[0] = _mm512_xor_si512(s[0], _mm512_clmulepi64_epi128(even, w, 0x00)); - s[1] = _mm512_xor_si512(s[1], _mm512_clmulepi64_epi128(odd, w, 0x10)); - } - } - let acc = &mut self.acc[slot(half, i)]; - for (acc, sum) in acc.iter_mut().zip(sums) { - *acc = _mm512_xor_si512(*acc, sum); - } + for (p, pivot) in [2, 4, 1].into_iter().enumerate() { + embed_words(&words[pivot], &mut q[4 * p]); } + self.multiply(half, &q); } } } } } - /// The three partial products of slot `i`. + /// Every output of [`extend_group`] of the quantities: its products into its slot. + #[inline(always)] + unsafe fn multiply(&mut self, half: usize, q: &Quantities) { + unsafe { + self.high_point::<0>(half, q); + self.high_point::<1>(half, q); + self.high_point::<2>(half, q); + self.high_point::<3>(half, q); + self.high_point::<4>(half, q); + self.high_point::<5>(half, q); + self.high_point::<6>(half, q); + self.high_point::<7>(half, q); + self.high_point::<8>(half, q); + } + } + + /// The outputs of high point `K`: the binary low points of a new high point, its low + /// mixed points, then their lane sum. + #[inline(always)] + unsafe fn high_point(&mut self, half: usize, q: &Quantities) { + unsafe { + let (lows, sum) = ([20 + 4 * K, 21 + 4 * K, 22 + 4 * K, 23 + 4 * K], 56 + K); + if K < 4 { + self.add_products::(half, [lows[0], lows[1], sum], q); + self.add_products::(half, [lows[2], lows[3]], q); + } else { + let binary = 4 * (K - 4); + self.add_products::(half, [binary, binary + 1, lows[0]], q); + self.add_products::(half, [binary + 2, binary + 3, lows[1]], q); + self.add_products::(half, [lows[2], lows[3], sum], q); + } + } + } + + /// For `S` outputs of high point `K`, given by `SHAPE`, every pivot's products into + /// their slots. + #[inline(always)] + unsafe fn add_products( + &mut self, + half: usize, + outputs: [usize; S], + q: &Quantities, + ) { + unsafe { + let mut acc: [[__m512i; 3]; S] = std::array::from_fn(|s| self.acc[slot(half, outputs[s])]); + for p in 0..3 { + let e = values::(&q[4 * p]); + for k in 0..3 { + let l = values::(&q[4 * p + 1 + k]); + for s in 0..S { + acc[s][k] = _mm512_ternarylogic_epi64( + acc[s][k], + _mm512_clmulepi64_epi128(l[s], e[s], 0x00), + _mm512_clmulepi64_epi128(l[s], e[s], 0x11), + 0x96, + ); + } + } + } + for (s, acc) in acc.into_iter().enumerate() { + self.acc[slot(half, outputs[s])] = acc; + } + } + } + + /// The three partial products of slot `i`, a polynomial congruent to the slot's sum + /// modulo the minimal polynomial. pub fn get(&self, i: usize) -> [u128; 3] { std::array::from_fn(|k| { - let mut lanes = [[0u128; 4]; 2]; - for (lanes, &acc) in lanes.iter_mut().zip(&self.acc[i][2 * k..2 * k + 2]) { - // SAFETY: four 128-bit lanes to a register. - unsafe { _mm512_storeu_si512(lanes.as_mut_ptr().cast(), acc) }; - } - lanes.as_flattened().iter().fold(0, |a, &b| a ^ b) + let mut lanes = [0u128; 4]; + // SAFETY: four 128-bit lanes to a register. + unsafe { _mm512_storeu_si512(lanes.as_mut_ptr().cast(), self.acc[i][k]) }; + lanes.iter().fold(0, |a, &b| a ^ b) }) } } @@ -1054,12 +1284,15 @@ mod tests { assert_eq!(fold_groups(&e, &fold), fold_groups_portable(&e, &fold)); } let block: Vec = (0..crate::witness::BLOCK_LEN).map(|_| rng.next_u64()).collect(); - let weights: Vec<[u64; 3]> = (0..BITS).map(|_| std::array::from_fn(|_| rng.next_u64())).collect(); + let field = PowerField::new(rng.ext()).unwrap(); + let weights = rng.ext_vec(BITS); let mut skip = SkipBlock::new(); - skip.run(&block, &weights); - let expected = skip_block_portable(&block, &weights); + skip.run(&block, &SkipWeights::new(&field, &weights)); + let polys: Vec = weights.iter().map(|&w| field.to_poly(w)).collect(); + let expected = skip_block_portable(&block, &polys); + let reduce = |sum: &[u128; 3]| field.reduce(&crate::basis::carry::<3, 6>(sum)); for (i, expected) in expected.iter().enumerate() { - assert_eq!(&skip.get(i), expected, "slot {i}"); + assert_eq!(reduce(&skip.get(i)), reduce(expected), "slot {i}"); } } } diff --git a/crates/keccak_gkr/src/power.rs b/crates/keccak_gkr/src/power.rs index 6b108b7c52..8299b6eab9 100644 --- a/crates/keccak_gkr/src/power.rs +++ b/crates/keccak_gkr/src/power.rs @@ -121,6 +121,58 @@ impl PowerField { row_kernel!(width, dot_pair(a, b, width), dot_pair_portable(a, b, width)) } + /// `sum of B[j] * b[j]` and `sum of (B[j] + B'[j]) * (b[j] + b'[j])`, unreduced, where + /// `B[j] = weights[j] * a[j] + constant[j]` and `B'` likewise of `a'`: a round's values over + /// a pair of rows. `scratch` holds two rows where the products are reduced before the sums. + pub fn round_dot( + &self, + weights: &[u64], + constant: &[u64], + a: [&[u64]; 2], + b: [&[u64]; 2], + scratch: &mut [u64], + width: usize, + ) -> ([u64; 9], [u64; 9]) { + assert!( + [weights, constant] + .iter() + .chain(&a) + .chain(&b) + .all(|row| row.len() >= 3 * width) + ); + cfg_select! { + all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq") => { + if width.is_multiple_of(x86::STEP) { + let _ = scratch; + // SAFETY: the features are enabled at compile time, and every row is bounded above. + unsafe { x86::round_dot(weights, constant, a, b, width) } + } else { + self.round_dot_rows(weights, constant, a, b, scratch, width) + } + } + _ => self.round_dot_rows(weights, constant, a, b, scratch, width), + } + } + + /// [`Self::round_dot`] by reducing `B` and `B'` into `scratch` first: the other arms' form, + /// and the x86 kernel's test oracle, whose sums are congruent to these. + pub fn round_dot_rows( + &self, + weights: &[u64], + constant: &[u64], + a: [&[u64]; 2], + b: [&[u64]; 2], + scratch: &mut [u64], + width: usize, + ) -> ([u64; 9], [u64; 9]) { + let (b0, b1) = scratch[..6 * width].split_at_mut(3 * width); + self.mul_row(b0, weights, a[0], constant, width); + self.mul_row(b1, weights, a[1], constant, width); + let (zero, infinity) = self.dot_pair([b0, b1], b, width); + let pad = |x: [u64; 6]| std::array::from_fn(|i| if i < 6 { x[i] } else { 0 }); + (pad(zero), pad(infinity)) + } + /// `dst[j] = lo[j] + challenge * (lo[j] + hi[j])`. pub fn fold_row(&self, dst: &mut [u64], lo: &[u64], hi: &[u64], challenge: &Element, width: usize) { assert!(dst.len() >= 3 * width && lo.len() >= 3 * width && hi.len() >= 3 * width); @@ -342,6 +394,62 @@ mod x86 { (finish(&acc[0]), finish(&acc[1])) } + /// [`PowerField::round_dot`] without reducing `B`: its six words meet `b` as two halves of + /// three, Karatsuba's twelve products where reducing and multiplying take eighteen. `B + B'` + /// is `weights * (a + a')`, the constants cancelling. + pub unsafe fn round_dot( + weights: &[u64], + constant: &[u64], + a: [&[u64]; 2], + b: [&[u64]; 2], + width: usize, + ) -> ([u64; 9], [u64; 9]) { + // Per sum, the 128-bit partial sums by word offset. + let mut acc = [[V::zero(); 8]; 2]; + // The six words of products given by their even and odd entries' partial sums. + let words = |p: [[V; 5]; 2]| -> [V; 6] { + let [even, odd] = p.map(|half| std::array::from_fn::(|w| word(&half, w))); + std::array::from_fn(|w| even[w].unpacklo(odd[w])) + }; + for j in (0..width).step_by(STEP) { + let (w, c, a0, a1, b0, b1) = unsafe { + ( + load3(weights, width, j), + load3(constant, width, j), + load3(a[0], width, j), + load3(a[1], width, j), + load3(b[0], width, j), + load3(b[1], width, j), + ) + }; + let sw = sums(&w); + let mut first = words(karatsuba(&w, &sw, &a0, &sums(&a0))); + for (f, c) in first.iter_mut().zip(c) { + *f = f.xor(c); + } + let da = xor3(&a0, &a1); + let delta = words(karatsuba(&w, &sw, &da, &sums(&da))); + for (acc, (f, b)) in acc.iter_mut().zip([(first, b0), (delta, xor3(&b0, &b1))]) { + for offset in [0, 3] { + let p = partial(&[f[offset], f[offset + 1], f[offset + 2]], &b); + for s in 0..5 { + acc[offset + s] = acc[offset + s].xor(p[0][s].xor(p[1][s])); + } + } + } + } + let finish = |acc: &[V; 8]| { + let mut out = [0u64; 9]; + for (s, sum) in acc.iter().enumerate() { + let sum = sum.sum(); + out[s] ^= sum as u64; + out[s + 1] ^= (sum >> 64) as u64; + } + out + }; + (finish(&acc[0]), finish(&acc[1])) + } + pub unsafe fn reduce_row(field: &PowerField, dst: &mut [u64], wide: &[u64], width: usize) { for j in (0..width).step_by(STEP) { let w: [V; 4] = std::array::from_fn(|k| unsafe { V::load(wide.as_ptr().add(k * width + j)) }); @@ -541,7 +649,8 @@ mod tests { #[test] fn arithmetic_matches_the_standard_basis() { let mut rng = Rng::new(41); - let field = PowerField::new(rng.ext()).unwrap(); + let s = rng.ext(); + let field = PowerField::new(s).unwrap(); let width = 16; let elements = |rng: &mut Rng| -> (Vec, Vec) { let values = rng.ext_vec(width); @@ -590,5 +699,19 @@ mod tests { for j in 0..width { assert_eq!(field.to_field(&get(&fast, width, j)), a0[j] * b0[j] + b1[j]); } + + // The sums are congruent, not equal: compare their values at `s`. + let eval = PolyEval::new(s, 72); + let mut scratch = vec![0u64; 6 * width]; + let (zero, infinity) = field.round_dot(&ra0, &rb0, [&ra1, &rb1], [&rb0, &ra0], &mut scratch, width); + let (zero_rows, infinity_rows) = + field.round_dot_rows(&ra0, &rb0, [&ra1, &rb1], [&rb0, &ra0], &mut scratch, width); + assert_eq!(eval.eval(&zero), eval.eval(&zero_rows)); + assert_eq!(eval.eval(&infinity), eval.eval(&infinity_rows)); + let b = |j: usize, a: &[F192]| a0[j] * a[j] + b0[j]; + let expected: F192 = (0..width).map(|j| b(j, &a1) * b0[j]).sum(); + assert_eq!(eval.eval(&zero), expected); + let expected: F192 = (0..width).map(|j| (b(j, &a1) + b(j, &b1)) * (b0[j] + a0[j])).sum(); + assert_eq!(eval.eval(&infinity), expected); } } diff --git a/crates/keccak_gkr/src/prover.rs b/crates/keccak_gkr/src/prover.rs index 41b59a0324..85720a3231 100644 --- a/crates/keccak_gkr/src/prover.rs +++ b/crates/keccak_gkr/src/prover.rs @@ -15,7 +15,7 @@ use primitives::multilinear::{eq_table, poly_eval}; use crate::basis::{PolyEval, add_mul, carry}; use crate::keccak::{BITS, LOCAL_VARS, along, linear_transpose, row_positions}; -use crate::kernels::{GROUP, MIXED, PACKS, SKIP, SkipBlock, TERNARY, embed, mixed_points, products, slot}; +use crate::kernels::{GROUP, MIXED, PACKS, SKIP, SkipBlock, SkipWeights, TERNARY, embed, mixed_points, products, slot}; use crate::open::fold; use crate::power::{Element, PowerField, get, set}; use crate::protocol::{Claim, chi_weights, mu_at}; @@ -193,6 +193,8 @@ struct Layer<'a> { /// A row of a table in the power basis: three words per position, laid out by word. const ROW: usize = 3 * BITS; +/// The scratch [`Round::add_pair`] takes: four shifted rows, and two for `round_dot`. +const PAIR_SCRATCH: usize = 6 * ROW; impl Layer<'_> { /// The blocks a segment's tasks walk: those written, then the padding block once, standing @@ -349,7 +351,7 @@ impl Layer<'_> { /// here. A binary one is a pack, whose sum `binary` already holds. The packs of the padding /// block are all equal, so their extension vanishes there. fn skip_sums(&self, claim: &Claim, field: &PowerField, s: F192, binary: &[F192; GROUP]) -> Vec { - let weights: Vec = claim.weights.iter().map(|&w| field.to_poly(w)).collect(); + let weights = SkipWeights::new(field, &claim.weights); let groups: Vec = eq_table(&claim.point[SKIP..]) .iter() .map(|&e| field.to_poly(e)) @@ -431,7 +433,7 @@ impl Layer<'_> { target_feature = "gfni" )); ( - vec![0u64; 4 * ROW], + vec![0u64; PAIR_SCRATCH], vec![0u64; if gfni { 0 } else { 8 * BITS }], vec![0u64; 2 * ROW], ) @@ -483,11 +485,11 @@ impl Layer<'_> { /// The sums a round accumulates, unreduced: its values at `0` and at infinity. #[derive(Clone, Copy, Default)] -struct RoundSums([u64; 9], [u64; 9]); +struct RoundSums([u64; 12], [u64; 12]); impl RoundSums { fn merge(mut self, other: Self) -> Self { - for i in 0..9 { + for i in 0..12 { self.0[i] ^= other.0[i]; self.1[i] ^= other.1[i]; } @@ -519,45 +521,49 @@ impl<'a> Round<'a> { } Self { field, - eval: PolyEval::new(s, 72), + eval: PolyEval::new(s, 96), kappa: field.to_poly(kappa), weights, constant, } } - /// `B[j] = weights[j] * (kappa + T[j + 1]) + kappa * weights[j + 2]` and `T[j + 2]`, for a - /// row `T`. - fn factors(&self, t: &[u64], b: &mut [u64], shifted: &mut [u64]) { - // `T[j + 1] + kappa`, then `T[j + 2]`: a lane is 64 positions, five lanes to a row. + /// `kappa + T[j + 1]` and `T[j + 2]`, for a row `T`: a lane is 64 positions, five lanes to a + /// row. + fn shifted(&self, t: &[u64], next: &mut [u64], after: &mut [u64]) { for k in 0..3 { - let (src, next) = (&t[k * BITS..][..BITS], &mut shifted[k * BITS..][..BITS]); + let (src, next, after) = ( + &t[k * BITS..][..BITS], + &mut next[k * BITS..][..BITS], + &mut after[k * BITS..][..BITS], + ); for lane in 0..BITS / 64 { let at = along(64 * lane, 1); for z in 0..64 { next[64 * lane + z] = src[at + z] ^ self.kappa[k]; } - } - } - self.field.mul_row(b, &self.weights, shifted, &self.constant, BITS); - for k in 0..3 { - let (src, dst) = (&t[k * BITS..][..BITS], &mut shifted[k * BITS..][..BITS]); - for lane in 0..BITS / 64 { - dst[64 * lane..][..64].copy_from_slice(&src[along(64 * lane, 2)..][..64]); + after[64 * lane..][..64].copy_from_slice(&src[along(64 * lane, 2)..][..64]); } } } /// Add one pair of rows to a round's sums: the values at `0` and at infinity of - /// `sum over positions of B[j] * T[j + 2]`, weighted by `eq`. + /// `sum over positions of B[j] * T[j + 2]`, weighted by `eq`, where + /// `B[j] = weights[j] * (kappa + T[j + 1]) + kappa * weights[j + 2]`. fn add_pair(&self, sums: &mut RoundSums, scratch: &mut [u64], low: &[u64], high: &[u64], eq: F192) { - let (b, shifted) = scratch.split_at_mut(2 * ROW); + let (shifted, rest) = scratch.split_at_mut(4 * ROW); + let (next, after) = shifted.split_at_mut(2 * ROW); for (half, t) in [low, high].into_iter().enumerate() { - self.factors(t, &mut b[half * ROW..][..ROW], &mut shifted[half * ROW..][..ROW]); + self.shifted(t, &mut next[half * ROW..][..ROW], &mut after[half * ROW..][..ROW]); } - let (zero, infinity) = self - .field - .dot_pair([&b[..ROW], &b[ROW..]], [&shifted[..ROW], &shifted[ROW..]], BITS); + let (zero, infinity) = self.field.round_dot( + &self.weights, + &self.constant, + [&next[..ROW], &next[ROW..]], + [&after[..ROW], &after[ROW..]], + rest, + BITS, + ); let weight = self.field.to_poly(eq); add_mul(&mut sums.0, &zero, &weight); add_mul(&mut sums.1, &infinity, &weight); @@ -600,7 +606,7 @@ impl<'a> Round<'a> { fold(out, segment, 0); } let mut sums = RoundSums::default(); - let mut scratch = vec![0u64; 4 * ROW]; + let mut scratch = vec![0u64; PAIR_SCRATCH]; for (pair, &eq) in dst[..to.len() * ROW].as_chunks::<{ 2 * ROW }>().0.iter().zip(eq) { self.add_pair(&mut sums, &mut scratch, &pair[..ROW], &pair[ROW..], eq); } @@ -610,7 +616,7 @@ impl<'a> Round<'a> { let (pairs, half) = (to.live.div_ceil(2), to.m / 2); let mut sums = parallel::map_reduce_with_state( to.segments * pairs, - || (vec![0u64; 4 * ROW], vec![0u64; 2 * ROW]), + || (vec![0u64; PAIR_SCRATCH], vec![0u64; 2 * ROW]), RoundSums::default, |(scratch, rows), sums, task| { let (segment, t) = (task / pairs, task % pairs); @@ -643,7 +649,7 @@ impl<'a> Round<'a> { if first == half { return; } - let mut scratch = vec![0u64; 4 * ROW]; + let mut scratch = vec![0u64; PAIR_SCRATCH]; for (pad, eq) in pads.as_chunks::().0.iter().zip(eq.chunks_exact(half)) { self.add_pair(sums, &mut scratch, pad, pad, eq[first..].iter().sum()); } diff --git a/crates/leanvm_core/src/constraints.rs b/crates/leanvm_core/src/constraints.rs index a95ae5fe43..4adbb89bf1 100644 --- a/crates/leanvm_core/src/constraints.rs +++ b/crates/leanvm_core/src/constraints.rs @@ -79,7 +79,7 @@ const FOLD_CHUNK: usize = 1 << 10; /// A table's summand `P(v) = constant + lin(v) + Σ_g v[left_g] · right_g(v)`, its products grouped /// by a shared column. Its homogeneous quadratic part is `Q(v) = Σ_g v[left_g] · right_g(v)`. /// -/// The `right_g` and `lin` are linear, so a joined table folds them as single `E` columns +/// The `right_g` and `lin` are linear, so an active table folds them as single `E` columns /// instead of folding every column: its state is `[lin, left_0, right_0, left_1, right_1, ..]`. struct Form { constant: F192, @@ -144,7 +144,7 @@ impl Form { } } - /// Columns of the folded state: `lin`, then `left_g` and `right_g` per group. + /// Columns of the state: `lin`, then `left_g` and `right_g` per group. fn width(&self) -> usize { 1 + 2 * self.groups.len() } @@ -176,31 +176,9 @@ fn splat(terms: &Lin) -> Vec<(usize, F192x8)> { terms.iter().map(|&(c, w)| (c, F192x8::splat(w))).collect() } -/// A worker's scratch for [`load_rows`] over `n` columns: `end` and `sum`. -fn rows_scratch(n: usize) -> (Vec, Vec) { - let zero = F64x8::from_array(&[F64::ZERO; 8]); - (vec![zero; n], vec![zero; n]) -} - -/// Loads the vectors of rows `i` and `i + half` of every column in `read`: `end` takes the upper vector -/// with `at_one` and the lower without, `sum` their sum. -#[inline(always)] -fn load_rows( - cols: &[&[F64]], - read: &[usize], - i: usize, - half: usize, - at_one: bool, - (end, sum): &mut (Vec, Vec), -) { - for &c in read { - let (lo, hi) = (load_k(cols[c], i), load_k(cols[c], i + half)); - (end[c], sum[c]) = (if at_one { hi } else { lo }, lo + hi); - } -} - -/// A joined table's folded state: per vector of eight rows, its `width` columns side by side, -/// `state[b * width + c]`. Below eight rows, one vector per column holds them all. +/// A table's state: per vector of eight rows, its `width` columns side by side, `state[b * width + c]`, +/// the blocks of the rows below the round's `half` first, then those above. Below eight rows, one vector +/// per column holds them all. type State = Vec; /// The blocks task `t` covers out of `n`. @@ -216,12 +194,14 @@ fn finish([endpoint, slope]: [Prod8; 2], constant: F192) -> [F192; 2] { [endpoint.reduce() + constant, slope.reduce()] } -/// An active round of a table that has not joined: its message from the `K` columns, in place. -/// Per vector of rows, the linear forms are `E x K` mixed dots, and every product of the -/// summand is one mixed product against a reduced `right_g`. -fn message_k(form: &Form, cols: &[&[F64]], eqr: &[F192], half: usize, at_one: bool) -> [F192; 2] { +/// The first round a table is active: its message, and its state over all its rows, unfolded, in one +/// pass over the `K` columns, which no later round reads. The linear forms are `E x K` mixed dots, and +/// every product of the summand is one mixed product against a reduced `right_g`. +fn expand(form: &Form, cols: &[&[F64]], eqr: &[F192], half: usize, at_one: bool) -> (State, [F192; 2]) { if half < 8 { - return message_e(form, &form.small_state(cols, 2 * half), eqr, half, at_one); + let state = form.small_state(cols, 2 * half); + let message = message_e(form, &state, eqr, half, at_one); + return (state, message); } let lin = splat(&form.lin); let rights: Vec<_> = form.groups.iter().map(|(_, right)| splat(right)).collect(); @@ -229,33 +209,55 @@ fn message_k(form: &Form, cols: &[&[F64]], eqr: &[F192], half: usize, at_one: bo terms .iter() .fold(Prod8::ZERO, |acc, &(c, w)| acc ^ w.mul_base_unreduced(vals[c])) + .reduce_lanes() }; - let blocks = half / 8; - let acc = parallel::map_reduce_with_state( - blocks.div_ceil(TASK_BLOCKS), - || rows_scratch(cols.len()), - || [Prod8::ZERO; 2], - |rows, acc, t| { - for b in task_blocks(t, blocks) { - let i = 8 * b; - load_rows(cols, &form.used, i, half, at_one, rows); - let (end, sum) = &*rows; - let (mut endpoint, mut slope) = (dot(&lin, end), Prod8::ZERO); - for ((left, _), right) in form.groups.iter().zip(&rights) { - endpoint ^= dot(right, end).reduce_lanes().mul_base_unreduced(end[*left]); - slope ^= dot(right, sum).reduce_lanes().mul_base_unreduced(sum[*left]); - } - let eq = load_e(eqr, i); - acc[0] ^= endpoint.reduce_lanes().mul_unreduced(eq); - acc[1] ^= slope.reduce_lanes().mul_unreduced(eq); - } - }, - xor2, - ); - finish(acc, form.constant) + let entries = |vals: &[F64x8], out: &mut [F192x8]| { + out[0] = dot(&lin, vals); + for (g, ((left, _), right)) in form.groups.iter().zip(&rights).enumerate() { + out[1 + 2 * g] = F192x8::from(vals[*left]); + out[2 + 2 * g] = dot(right, vals); + } + }; + let (blocks, width) = (half / 8, form.width()); + let zero = F64x8::from_array(&[F64::ZERO; 8]); + let mut acc = [Prod8::ZERO; 2]; + // SAFETY: every task writes all `width` entries of each of its blocks in both halves. + let state = unsafe { + primitives::filled_vec(2 * blocks * width, |state| { + let (lows, highs) = state.split_at_mut(blocks * width); + acc = parallel::map_reduce_chunks_mut( + [lows, highs], + [TASK_BLOCKS * width; 2], + || (vec![zero; cols.len()], vec![zero; cols.len()]), + || [Prod8::ZERO; 2], + |(lo, hi), acc, t, [lows, highs]| { + let outs = lows.chunks_exact_mut(width).zip(highs.chunks_exact_mut(width)); + for (b, (lo_out, hi_out)) in task_blocks(t, blocks).zip(outs) { + let i = 8 * b; + for &c in &form.used { + (lo[c], hi[c]) = (load_k(cols[c], i), load_k(cols[c], i + half)); + } + entries(lo, lo_out); + entries(hi, hi_out); + let (end, end_k) = if at_one { (&*hi_out, &*hi) } else { (&*lo_out, &*lo) }; + let (mut endpoint, mut slope) = (Prod8::ZERO, Prod8::ZERO); + for (g, (left, _)) in form.groups.iter().enumerate() { + endpoint ^= end[2 + 2 * g].mul_base_unreduced(end_k[*left]); + slope ^= (lo_out[2 + 2 * g] + hi_out[2 + 2 * g]).mul_base_unreduced(lo[*left] + hi[*left]); + } + let eq = load_e(eqr, i); + acc[0] ^= (endpoint.reduce_lanes() + end[0]).mul_unreduced(eq); + acc[1] ^= slope.reduce_lanes().mul_unreduced(eq); + } + }, + xor2, + ); + }) + }; + (state, finish(acc, form.constant)) } -/// An active round of a joined table, from its folded state. +/// An active round of a table, from its state. fn message_e(form: &Form, state: &State, eqr: &[F192], half: usize, at_one: bool) -> [F192; 2] { if half < 8 { // Fewer row pairs than a vector: `[Σ_i eqr[i]·end_i, Σ_i eqr[i]·Q(lo_i + hi_i)]` by lanes. @@ -280,15 +282,7 @@ fn message_e(form: &Form, state: &State, eqr: &[F192], half: usize, at_one: bool |acc, t| { for b in task_blocks(t, blocks) { let (lo, hi) = (&lows[b * width..][..width], &highs[b * width..][..width]); - let end = if at_one { hi } else { lo }; - let (mut endpoint, mut slope) = (Prod8::ZERO, Prod8::ZERO); - for g in (1..width).step_by(2) { - endpoint ^= end[g].mul_unreduced(end[g + 1]); - slope ^= (lo[g] + hi[g]).mul_unreduced(lo[g + 1] + hi[g + 1]); - } - let eq = load_e(eqr, 8 * b); - acc[0] ^= (endpoint.reduce_lanes() + end[0]).mul_unreduced(eq); - acc[1] ^= slope.reduce_lanes().mul_unreduced(eq); + block_message(lo, hi, load_e(eqr, 8 * b), at_one, acc); } }, xor2, @@ -296,50 +290,59 @@ fn message_e(form: &Form, state: &State, eqr: &[F192], half: usize, at_one: bool finish(acc, form.constant) } -/// The round a table joins, after its challenge `r`: the state `lo + r·(lo + hi)` of every -/// entry, built straight from the `K` columns. By linearity `lin` and `right_g` take the -/// terms `w·lo + (w·r)·(lo + hi)`, so no row pays an `E x E` product. -fn join(form: &Form, cols: &[&[F64]], r: F192, half: usize) -> State { - if half < 8 { - let mut state = form.small_state(cols, 2 * half); - fold_state(&mut state, r, half); - return state; +/// Adds one block's terms to a table's message: `lo` and `hi` are its state entries in the +/// two halves, `eq` its rows' eq weights. +#[inline(always)] +fn block_message(lo: &[F192x8], hi: &[F192x8], eq: F192x8, at_one: bool, acc: &mut [Prod8; 2]) { + let end = if at_one { hi } else { lo }; + let (mut endpoint, mut slope) = (Prod8::ZERO, Prod8::ZERO); + for g in (1..lo.len()).step_by(2) { + endpoint ^= end[g].mul_unreduced(end[g + 1]); + slope ^= (lo[g] + hi[g]).mul_unreduced(lo[g + 1] + hi[g + 1]); } - let (blocks, width) = (half / 8, form.width()); - let pair = |terms: &Lin| -> Vec<_> { - terms - .iter() - .map(|&(c, w)| (c, F192x8::splat(w), F192x8::splat(w * r))) - .collect() - }; - let lin = pair(&form.lin); - let rights: Vec<_> = form.groups.iter().map(|(_, right)| pair(right)).collect(); + acc[0] ^= (endpoint.reduce_lanes() + end[0]).mul_unreduced(eq); + acc[1] ^= slope.reduce_lanes().mul_unreduced(eq); +} + +/// [`fold_state`], and the next round's message from the folded state, in one pass: a task folds +/// block `k` of every quarter into block `k` of both next halves, the pair that message reads. +/// `eqr` and `at_one` are the next round's; `None` below sixteen rows per half. +fn fold_and_message( + form: &Form, + state: &mut State, + r: F192, + half: usize, + eqr: &[F192], + at_one: bool, +) -> Option<[F192; 2]> { + if half < 16 { + fold_state(state, r, half); + return None; + } + let (pairs, width) = (half / 16, form.width()); let r8 = F192x8::splat(r); - // SAFETY: every task writes all `width` entries of each of its blocks. - unsafe { - primitives::filled_vec(blocks * width, |state| { - parallel::chunks_mut(state, TASK_BLOCKS * width, |t, out| { - let mut rows = rows_scratch(cols.len()); - for (b, out) in task_blocks(t, blocks).zip(out.chunks_exact_mut(width)) { - load_rows(cols, &form.used, 8 * b, half, false, &mut rows); - let (lo, sum) = &rows; - let dot = |terms: &[(usize, F192x8, F192x8)]| { - terms - .iter() - .fold(Prod8::ZERO, |acc, &(c, w, wr)| { - acc ^ w.mul_base_unreduced(lo[c]) ^ wr.mul_base_unreduced(sum[c]) - }) - .reduce_lanes() - }; - out[0] = dot(&lin); - for (g, ((left, _), right)) in form.groups.iter().zip(&rights).enumerate() { - out[1 + 2 * g] = F192x8::from(lo[*left]) + r8.mul_base_unreduced(sum[*left]).reduce_lanes(); - out[2 + 2 * g] = dot(right); - } + let (left, right) = state.split_at_mut(2 * pairs * width); + let (q0, q1) = left.split_at_mut(pairs * width); + let (q2, q3) = right.split_at_mut(pairs * width); + let acc = parallel::map_reduce_chunks_mut( + [q0, q1, q2, q3], + [TASK_BLOCKS * width; 4], + || (), + || [Prod8::ZERO; 2], + |(), acc, t, [lo, hi, upper_lo, upper_hi]| { + let quarters = lo.chunks_exact_mut(width).zip(hi.chunks_exact_mut(width)); + let uppers = upper_lo.chunks_exact(width).zip(upper_hi.chunks_exact(width)); + for (k, ((lo, hi), (upper_lo, upper_hi))) in task_blocks(t, pairs).zip(quarters.zip(uppers)) { + for (v, &u) in lo.iter_mut().zip(upper_lo).chain(hi.iter_mut().zip(upper_hi)) { + *v = *v + (*v + u) * r8; } - }); - }) - } + block_message(lo, hi, load_e(eqr, 8 * k), at_one, acc); + } + }, + xor2, + ); + state.truncate(2 * pairs * width); + Some(finish(acc, form.constant)) } /// The state folded by `r`, `v[i] = v[i] + r·(v[i] + v[i + half])` in every column, the upper half dropped. @@ -372,26 +375,35 @@ struct Table<'a> { weight: F192, form: Form, cols: &'a [&'a [F64]], - /// The folded state, once the table has joined: a few `E` columns, the batch's largest transients. + /// The state, from the table's first active round: a few `E` columns, the batch's largest transients. state: Option, + /// The next round's message, when the last fold built it. + next: Option<[F192; 2]>, } impl Table<'_> { - /// The message of the round that binds variable `m`, while the table is active. - fn message(&self, eqr: &[F192], m: usize, at_one: bool) -> [F192; 2] { + /// The message of the round that binds variable `m`, while the table is active. The first such + /// round expands the table's state. + fn message(&mut self, eqr: &[F192], m: usize, at_one: bool) -> [F192; 2] { + if let Some(p) = self.next.take() { + return p; + } let (form, half) = (&self.form, 1 << m); match &self.state { Some(state) => message_e(form, state, eqr, half, at_one), - None => message_k(form, self.cols, eqr, half, at_one), + None => { + let (state, p) = expand(form, self.cols, eqr, half, at_one); + self.state = Some(state); + p + } } } - /// Binds variable `m` to `r`: the join of a table that has not joined, the fold of one that has. - fn bind(&mut self, r: F192, m: usize) { - match &mut self.state { - Some(state) => fold_state(state, r, 1 << m), - None => self.state = Some(join(&self.form, self.cols, r, 1 << m)), - } + /// Binds variable `m` to `r`, folding the state, and builds the next round's message on the way + /// against its `eqr` and `at_one`. + fn bind(&mut self, r: F192, m: usize, eqr: &[F192], at_one: bool) { + let state = self.state.as_mut().expect("an active table has its state"); + self.next = fold_and_message(&self.form, state, r, 1 << m, eqr, at_one); } } @@ -422,6 +434,7 @@ pub fn prove(airs: &[Air<'_>], zeta: &[F192], ps: &mut ProverState) -> Vec], zeta: &[F192], ps: &mut ProverState) -> Vec], zeta: &[F192], ps: &mut ProverState) -> Vec 0 && zeta[m - 1].is_zero(); for t in &mut tables { if t.tau > m { t.weight *= eq_k; - t.bind(rk, m); + t.bind(rk, m, &eqr, at_one_next); } else { t.weight *= rk; } @@ -554,8 +569,9 @@ mod tests { }) } - /// Every round of a table's life, at row counts on both sides of one vector: the join's - /// message and state, then each fold and joined message, against the naive formulas. + /// Every round of a table's life, at row counts on both sides of one vector: the expanded + /// state's message, then each fold, its state's message and the next round's message the fold + /// builds, against the naive formulas. #[test] fn round_messages_match_naive_evaluation() { let mut s = 7; @@ -569,26 +585,28 @@ mod tests { .map(|c| c.iter().map(|&x| F192::from(x)).collect()) .collect(); let mut eqr = eq_table(&(0..m).map(|_| random_e(&mut s)).collect::>()); - let mut state: Option = None; + let mut state = State::new(); + let mut next: Option<([F192; 2], bool)> = None; let mut half = 1usize << m; while half >= 1 { for at_one in [false, true] { - let got = match &state { - None => message_k(&form, &views, &eqr, half, at_one), - Some(st) => message_e(&form, st, &eqr, half, at_one), - }; - assert_eq!( - got, - naive_message(&pows, &naive_cols, &eqr, half, at_one), - "m {m} half {half}" - ); + let want = naive_message(&pows, &naive_cols, &eqr, half, at_one); + if half == 1 << m { + let (st, p) = expand(&form, &views, &eqr, half, at_one); + assert_eq!(p, want, "expand, m {m}"); + state = st; + } + assert_eq!(message_e(&form, &state, &eqr, half, at_one), want, "m {m} half {half}"); + if let Some((fused, _)) = next.filter(|n| n.1 == at_one) { + assert_eq!(fused, want, "fused, m {m} half {half}"); + } } let r = random_e(&mut s); shrink_eq_high(&mut eqr); - match &mut state { - None => state = Some(join(&form, &views, r, half)), - Some(st) => fold_state(st, r, half), - } + let at_one = half.is_multiple_of(32); + let fused = fold_and_message(&form, &mut state, r, half, &eqr, at_one); + assert_eq!(fused.is_some(), half >= 16, "m {m} half {half}"); + next = fused.map(|p| (p, at_one)); for c in &mut naive_cols { *c = (0..half).map(|i| interp(c[i], c[i + half], r)).collect(); } diff --git a/crates/leanvm_core/src/cpu/execute.rs b/crates/leanvm_core/src/cpu/execute.rs index 83dcdd6a63..e7f1a1569b 100644 --- a/crates/leanvm_core/src/cpu/execute.rs +++ b/crates/leanvm_core/src/cpu/execute.rs @@ -28,8 +28,6 @@ pub struct Execution { /// and an arithmetic back-solve writes its operand before reading it. pub unconstrained_reads: Vec, pub(crate) trace: Trace, // rows + final access-count columns, emitted in the same walk - /// `g^0, g^1, ..` as far as the run indexed them: every pc, frame base and cell it reached. - pub(crate) gpow: Vec, } impl Execution { @@ -390,16 +388,24 @@ impl Program { .push(unsafe { Box::<[Cell]>::new_zeroed_slice(PAGE).assume_init() }); } } - /// Cell `c`, which `ensure` has covered. + /// Cell `c`, covered first. #[inline(always)] fn at(&mut self, c: u32) -> &mut Cell { let c = c as usize; - &mut self.pages[c >> PAGE_LOG][c & (PAGE - 1)] + self.ensure(c); + // SAFETY: `ensure` grew the pages past cell `c`. + unsafe { + self.pages + .get_unchecked_mut(c >> PAGE_LOG) + .get_unchecked_mut(c & (PAGE - 1)) + } } #[inline(always)] fn covered(&self, c: u32) -> Option<&Cell> { let c = c as usize; - (c < self.used).then(|| &self.pages[c >> PAGE_LOG][c & (PAGE - 1)]) + // SAFETY: every cell below `used` is in a page. + (c < self.used) + .then(|| unsafe { self.pages.get_unchecked(c >> PAGE_LOG).get_unchecked(c & (PAGE - 1)) }) } #[inline(always)] fn written(&self, cell: u32) -> bool { @@ -411,18 +417,21 @@ impl Program { fn get(&self, cell: u32) -> F192 { self.covered(cell).map_or(F192::ZERO, |c| c.word) } - // An instruction's read. An unwritten cell reads as ZERO, which is then written - // there: the row is filled from the final image, which has to agree. + // An instruction's read, which advances the cell's access count as + // `bump_access_count` does: the word and the count before. An unwritten cell + // reads as ZERO, which is then written there: the row is filled from the final + // image, which has to agree. #[inline(always)] - fn read(&mut self, cell: u32) -> F192 { - self.ensure(cell as usize); + fn read(&mut self, cell: u32) -> (F192, u32) { let c = self.at(cell); - let word = c.word; - if c.state != State::Written { - c.state = State::Written; + let (word, count, fresh) = (c.word, c.count, c.state != State::Written); + c.count = count + 1; + c.state = State::Written; + self.counts = self.counts.max(count + 1); + if fresh { self.unwritten_reads.push(cell); } - word + (word, count) } // Write-once store, carried to every cell linked to this one. #[inline(always)] @@ -432,10 +441,27 @@ impl Program { } Ok(()) } + // An instruction's `put`, which advances the cell's access count as + // `bump_access_count` does: the count before. + #[inline(always)] + fn put_counted(&mut self, cell: u32, v: F192) -> Result { + let c = self.at(cell); + let (s, count) = (c.state, c.count); + if s == State::Written && c.word != v { + return Err(self.conflict(cell, v)); + } + c.word = v; + c.state = State::Written; + c.count = count + 1; + self.counts = self.counts.max(count + 1); + if s == State::Linked { + self.spread(cell, v)?; + } + Ok(count) + } // `put` on one cell; whether it was linked. #[inline(always)] fn store(&mut self, cell: u32, v: F192) -> Result { - self.ensure(cell as usize); let c = self.at(cell); let s = c.state; if s == State::Written && c.word != v { @@ -459,7 +485,6 @@ impl Program { } fn link(&mut self, a: u32, b: u32) { for (x, y) in [(a, b), (b, a)] { - self.ensure(x as usize); let c = self.at(x); if c.state == State::Unwritten { c.state = State::Linked; @@ -481,7 +506,6 @@ impl Program { // exponent. A run makes fewer than `2^32` accesses in all, so it cannot wrap. #[inline(always)] fn bump_access_count(&mut self, cell: u32) -> u32 { - self.ensure(cell as usize); let c = self.at(cell); let count = c.count; c.count = count + 1; @@ -830,13 +854,12 @@ impl Program { m.put(if ha { ab } else { aa }, m.get(ac) * vk.inv())?; } } - let va = m.read(aa); - let vb = m.read(ab); + // Each access advances its cell's count, in flush order, so a cell two + // operands alias still pairs each access with its own count. + let (va, ra) = m.read(aa); + let (vb, rb) = m.read(ab); let vc = if is_xor { va + vb } else { va * vb }; - m.put(ac, vc)?; - let ra = m.bump_access_count(aa); - let rb = m.bump_access_count(ab); - let rc = m.bump_access_count(ac); + let rc = m.put_counted(ac, vc)?; let row = Xrow { pc, fp, @@ -853,9 +876,7 @@ impl Program { pc += 1; } Op::Set { o, k } => { - let a = fp + o; - m.put(a, k)?; - let r = m.bump_access_count(a); + let r = m.put_counted(fp + o, k)?; set.push(Srow { pc, fp, @@ -866,7 +887,7 @@ impl Program { } Op::Deref { o1, o2, o3, mode } => { let a1 = fp + o1; - let p = m.read(a1); + let (p, r1) = m.read(a1); let p_addr = p.as_base().ok_or_else(|| fail(Fault::WildPointer { value: p }))?; let base = g.log(p_addr).ok_or_else(|| fail(Fault::WildPointer { value: p }))?; let a2 = base + o2; @@ -898,7 +919,6 @@ impl Program { m.put(a2, v)?; } } - let r1 = m.bump_access_count(a1); let r2 = m.bump_access_count(a2); let r3 = m.bump_access_count(a3); deref.push(Drow { @@ -919,16 +939,16 @@ impl Program { // otherwise not balance. A guest branches on g-powers; the one // idiom that once branched on a word, `assert a != b`, takes an // inverse hint instead (§sec:prog-div-ne). - let c = in_k("JUMP condition", m.read(ac)).map_err(fail)?; - let d = in_k("JUMP target", m.read(ad)).map_err(fail)?; - let f = in_k("JUMP fp", m.read(af)).map_err(fail)?; + let (c, rc) = m.read(ac); + let c = in_k("JUMP condition", c).map_err(fail)?; + let (d, rd) = m.read(ad); + let d = in_k("JUMP target", d).map_err(fail)?; + let (f, rf) = m.read(af); + let f = in_k("JUMP fp", f).map_err(fail)?; // The is-nonzero witness `w = c⁻¹` is never used for control // flow, only recorded as a witness column, so it is not // computed here at all: `JumpTable::fill` batch-inverts every // row's condition at once (§the trace rows in `cpu::trace`). - let rc = m.bump_access_count(ac); - let rd = m.bump_access_count(ad); - let rf = m.bump_access_count(af); let taken = !c.is_zero(); jump.push(Jrow { pc, @@ -958,8 +978,13 @@ impl Program { Op::Keccak { .. } => { use crate::keccak::{ROW_CELLS, STATE_CELLS}; let k = crate::tables::KeccakOp::of(&op); - let cells: [u32; ROW_CELLS] = std::array::from_fn(|c| k.cell(fp, c)); - let input: [F192; STATE_CELLS] = std::array::from_fn(|c| m.read(cells[c])); + // The counts in flush order, so a cell two operands alias still pairs + // each access with its own count. + let mut r = [0; ROW_CELLS]; + let mut input = [F192::ZERO; STATE_CELLS]; + for c in 0..STATE_CELLS { + (input[c], r[c]) = m.read(k.cell(fp, c)); + } // The table reads every cell with a literal zero top limb, so anything // else could not balance the bus. for (cell, &value) in input.iter().enumerate() { @@ -973,16 +998,10 @@ impl Program { let mut lanes = crate::keccak::state_of_cells(k.in_odd, |c| input[c]); keccak_step(&lanes); primitives::hash::keccak_f(&mut lanes); - let written = STATE_CELLS + k.out_cells.cells(); let output = crate::keccak::cells_of_state(&lanes, k.out_odd); - for (&cell, &v) in cells[STATE_CELLS..written].iter().zip(&output) { - m.put(cell, v)?; - } - // In flush order, so a cell two operands alias still pairs each - // read with its own count. A cell not written keeps count 0. - let mut r = [0; ROW_CELLS]; - for (count, &cell) in r.iter_mut().zip(&cells[..written]) { - *count = m.bump_access_count(cell); + // A cell not written keeps count 0. + for (c, &v) in output[..k.out_cells.cells()].iter().enumerate() { + r[STATE_CELLS + c] = m.put_counted(k.cell(fp, STATE_CELLS + c), v)?; } keccak.push(Krow { pc, @@ -1070,7 +1089,6 @@ impl Program { mem_used, unconstrained_reads: m.unwritten_reads, trace, - gpow: g.into_powers(), }) } } diff --git a/crates/leanvm_core/src/cpu/hints.rs b/crates/leanvm_core/src/cpu/hints.rs index ebd97026b8..e27a10249c 100644 --- a/crates/leanvm_core/src/cpu/hints.rs +++ b/crates/leanvm_core/src/cpu/hints.rs @@ -10,18 +10,22 @@ pub type Off = u32; /// Cells a program can address: [`GPow`] covers no more. pub(crate) const MAX_CELLS: u32 = 1 << 28; -/// The `g^k` table paired with a reverse index `g^k ↦ k`, both grown on demand +/// The powers `g^k` paired with a reverse index `g^k ↦ k`, both grown on demand /// (recursion depth, and so the address range, is unbounded). /// /// The interpreter needs both directions: a cell index becomes the address `g^k`, /// and a pointer word read back out of memory must be inverted to the index it -/// addresses. Three things keep this cheap. +/// addresses. Four things keep this cheap. /// -/// The reverse index holds nothing but the exponent, confirming a candidate slot -/// against the forward table, so a slot is four bytes where a `HashMap` bucket is -/// sixteen plus a control byte. Keys are field elements, hence effectively -/// uniform, which is all the placement asks for: one multiplicative mix, then -/// linear probing. The table stays at most half full to keep probing short. +/// The forward powers are every `STRIDE`th one and the first `STRIDE`, so `g^k` is +/// one product: a table of every power would be as large as the address space, +/// built by the interpreter's one thread and read at random. +/// +/// A reverse-index slot holds the key beside its exponent, so a probe reads the +/// slot alone rather than computing the power a candidate exponent names. Keys +/// are field elements, hence effectively uniform, which is all the placement asks +/// for: one multiplicative mix, then linear probing. The table stays at most half +/// full to keep probing short. /// /// It indexes only the exponents [`Self::note`] announces, or a lookup resolved: /// a run addresses millions of cells but stores only the few thousand bases of @@ -33,27 +37,120 @@ pub(crate) const MAX_CELLS: u32 = 1 << 28; /// interpreter could not label, a jump target or a pointer into a buffer, costs at /// most `STRIDE` probes once. pub struct GPow { - pow: Vec, - /// `k + 1` at the slot `g^k` mixes to, for the announced `k`; 0 marks an empty - /// slot. Power-of-two length. - index: Vec, - announced: usize, - /// The same for every multiple `k` of `STRIDE` that `pow` covers. - coarse: Vec, + /// `g^{STRIDE·j}` for every multiple of `STRIDE` covered. + strides: Vec, + /// `g^0` to `g^{STRIDE - 1}`. + first: [F64; STRIDE], + /// Exponents below this are covered. + covered: usize, + /// The announced exponents. + index: Index, + /// Every multiple of `STRIDE` covered. + coarse: Index, } /// Spacing of the coarse index, and so the most probes a missed lookup costs. const STRIDE: usize = 64; +/// Exponents by their g-power, open-addressed. A zero key marks an empty slot: no +/// g-power is zero. +/// +/// Bit `t` of `tags[s]` is set when a key mixing to home slot `s` and tag `t`, three +/// more bits of its mix, is in. A lookup of an absent key almost always misses its +/// tag and reads no slot, which is what a [`GPow::log`] that resolves does on all +/// but one of its probes: the tags are a sixteenth of the slots' size, so they stay +/// in cache where the slots do not. +struct Index { + slots: Vec<(u64, u32)>, + tags: Vec, + len: usize, +} + +impl Index { + fn new(slots: usize) -> Self { + Self { + slots: vec![(0, 0); slots], + tags: vec![0; slots], + len: 0, + } + } + + /// The home slot of `key` and its tag bit. + #[inline(always)] + fn home(&self, key: u64) -> (usize, u8) { + // Bits 32..64 of the mix are the well-distributed ones. + let mix = (key.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 32) as usize; + let slots = self.slots.len(); + (mix & (slots - 1), 1 << ((mix >> slots.trailing_zeros()) & 7)) + } + + #[inline] + fn find(&self, x: F64) -> Option { + let (mut i, tag) = self.home(x.0); + if self.tags[i] & tag == 0 { + return None; + } + let mask = self.slots.len() - 1; + loop { + let (key, k) = self.slots[i]; + if key == 0 { + return None; + } + if key == x.0 { + return Some(k); + } + i = (i + 1) & mask; + } + } + + /// Add `x = g^k`, which is not in the index yet. + fn insert(&mut self, x: F64, k: usize) { + self.reserve(self.len + 1); + self.place(x.0, k as u32); + self.len += 1; + } + + /// Make room for `n` entries, staying at most half full. + fn reserve(&mut self, n: usize) { + if 2 * n > self.slots.len() { + let slots = (2 * n).next_power_of_two(); + self.tags = vec![0; slots]; + for (key, k) in std::mem::replace(&mut self.slots, vec![(0, 0); slots]) { + if key != 0 { + self.place(key, k); + } + } + } + } + + fn place(&mut self, key: u64, k: u32) { + let (mut i, tag) = self.home(key); + self.tags[i] |= tag; + let mask = self.slots.len() - 1; + while self.slots[i].0 != 0 { + i = (i + 1) & mask; + } + self.slots[i] = (key, k); + } +} + impl GPow { /// Seeded with `g^0 = 1`, grown to cover index `upto`, nothing announced. pub fn new(upto: usize) -> Self { + let mut p = F64::ONE; + let first = std::array::from_fn(|_| { + let q = p; + p = mul_by_g(p); + q + }); let mut g = Self { - pow: vec![F64::ONE], - index: vec![0; 1 << 13], - announced: 0, - coarse: Vec::new(), + strides: vec![F64::ONE], + first, + covered: 1, + index: Index::new(1 << 13), + coarse: Index::new(1 << 4), }; + g.coarse.insert(F64::ONE, 0); g.extend(upto); g } @@ -72,18 +169,14 @@ impl GPow { /// index the interpreter reads has been covered by a [`Self::grow_to`]. #[inline(always)] pub fn pow(&self, k: usize) -> F64 { - self.pow[k] + assert!(k < self.covered, "g^{k} is past the covered range"); + self.strides[k / STRIDE] * self.first[k % STRIDE] } - /// The forward table, `g^0` to `g^{covered - 1}`. - pub(crate) fn into_powers(self) -> Vec { - self.pow - } - - /// Extend the forward table to cover index `upto`. + /// Cover index `upto`. #[inline] pub fn grow_to(&mut self, upto: usize) { - if upto >= self.pow.len() { + if upto >= self.covered { self.extend(upto); } } @@ -91,47 +184,25 @@ impl GPow { #[cold] fn extend(&mut self, upto: usize) { assert!(upto < MAX_CELLS as usize, "address space overflow (program too large)"); - // At least double, and a quarter more than a large jump asks: the small - // extends that follow fill spare capacity instead of copying the table. - let n = self.pow.len(); - if upto >= self.pow.capacity() { - let target = (2 * self.pow.capacity()) - .max(upto + 1 + (upto + 1) / 4) - .min(MAX_CELLS as usize); - self.pow.reserve_exact(target - n); - } - // ×g is ×x = `mul_by_g` (shift+fold), not a PMULL. - let mut p = *self.pow.last().unwrap(); - self.pow.extend((n..=upto).map(|_| { - p = mul_by_g(p); - p - })); - let (mut from, to) = (n.div_ceil(STRIDE), self.pow.len().div_ceil(STRIDE)); - if 2 * to > self.coarse.len() { - self.coarse = vec![0; (2 * to).next_power_of_two()]; - from = 0; - } - for j in from..to { - insert(&mut self.coarse, self.pow[j * STRIDE], j * STRIDE); + let step = mul_by_g(self.first[STRIDE - 1]); + let to = upto / STRIDE + 1; + self.coarse.reserve(to); + for j in self.strides.len()..to { + let p = self.strides[j - 1] * step; + self.strides.push(p); + self.coarse.insert(p, j * STRIDE); } + self.covered = upto + 1; } /// Announce that `g^k` may come back for inversion, because the interpreter /// is about to store it in memory as an address. Covers `k` first. Idempotent. pub fn note(&mut self, k: usize) { self.grow_to(k); - if find(&self.index, &self.pow, self.pow[k]).is_some() { - return; - } - if 2 * (self.announced + 1) > self.index.len() { - let wider = vec![0; 2 * self.index.len()]; - let old = std::mem::replace(&mut self.index, wider); - for e in old.into_iter().filter(|&e| e != 0) { - insert(&mut self.index, self.pow[e as usize - 1], e as usize - 1); - } + let x = self.pow(k); + if self.index.find(x).is_none() { + self.index.insert(x, k); } - insert(&mut self.index, self.pow[k], k); - self.announced += 1; } /// The discrete log of `x`, if it is a covered g-power. @@ -143,19 +214,20 @@ impl GPow { /// The exponent of `x` if announced: any covered one, for a table built by [`Self::dense`]. #[inline] pub(crate) fn lookup(&self, x: F64) -> Option { - find(&self.index, &self.pow, x) + self.index.find(x) } /// `log` for an address that is not announced: divide by `g` down to a - /// multiple of `STRIDE`, then announce the exponent. + /// multiple of `STRIDE`, then announce the exponent. Called on a lookup's miss, + /// so `x` is not in the index. #[cold] fn resolve(&mut self, x: F64) -> Option { let mut y = x; for j in 0..STRIDE { - if let Some(k) = find(&self.coarse, &self.pow, y) { + if let Some(k) = self.coarse.find(y) { let k = k as usize + j; - return (k < self.pow.len()).then(|| { - self.note(k); + return (k < self.covered).then(|| { + self.index.insert(x, k); k as u32 }); } @@ -165,31 +237,6 @@ impl GPow { } } -/// The `k` of a slot of `table` that holds `x = pow[k]`. -#[inline] -fn find(table: &[u32], pow: &[F64], x: F64) -> Option { - let mask = table.len() - 1; - let mut i = slot_of(x.0, mask); - while table[i] != 0 { - let k = table[i] - 1; - if pow[k as usize] == x { - return Some(k); - } - i = (i + 1) & mask; - } - None -} - -/// Put `k` in the first free slot of `table` from where `g^k = key` mixes to. -fn insert(table: &mut [u32], key: F64, k: usize) { - let mask = table.len() - 1; - let mut i = slot_of(key.0, mask); - while table[i] != 0 { - i = (i + 1) & mask; - } - table[i] = k as u32 + 1; -} - /// The inverse of [`mul_by_g`]: bit 0 of its output says whether it folded. #[inline(always)] fn div_by_g(a: F64) -> F64 { @@ -200,13 +247,6 @@ fn div_by_g(a: F64) -> F64 { }) } -/// Bits 32..64 of the mix are the well-distributed ones; the mask keeps as many -/// of them as the table is wide. -#[inline(always)] -fn slot_of(key: u64, mask: usize) -> usize { - (key.wrapping_mul(0x9E37_79B9_7F4A_7C15) >> 32) as usize & mask -} - /// Where a computed-advice bit buffer lives. Frame cells are ordinary memory, so /// a run of them serves as well as a heap region and costs no `DEREF` to index at /// a compile-time offset; the heap form stays for a buffer indexed at run time. diff --git a/crates/leanvm_core/src/cpu/layout.rs b/crates/leanvm_core/src/cpu/layout.rs index ab2e81f1d7..985ac9b79b 100644 --- a/crates/leanvm_core/src/cpu/layout.rs +++ b/crates/leanvm_core/src/cpu/layout.rs @@ -525,12 +525,9 @@ impl Program { let sch = schema(); // g^0..g^{span-1}, so every pc/frame/operand/count fill is an O(1) lookup instead // of an O(log) power: a frame base or an operand is below the cells the run used. - // The run's own table covers them unless a count outgrew it. + // Built here across the pool rather than by the run's one thread. let span = exec.mem_used.max(bytecode_size).max(tr.counts); - let gpow = match exec.gpow.get(..span) { - Some(gpow) => std::borrow::Cow::Borrowed(gpow), - None => std::borrow::Cow::Owned(primitives::field::g_powers(span)), - }; + let gpow = primitives::field::g_powers(span); // The public layout (flush/count blocks, placements, boundary, taus) is a pure // function of the program + announced sizes + public input, with no committed diff --git a/crates/leanvm_core/src/cpu/trace.rs b/crates/leanvm_core/src/cpu/trace.rs index 5508fe2c76..9c63a53020 100644 --- a/crates/leanvm_core/src/cpu/trace.rs +++ b/crates/leanvm_core/src/cpu/trace.rs @@ -91,10 +91,6 @@ impl Rows { pub(crate) fn get(&self, i: usize) -> &T { &self.chunks[i >> CHUNK_LOG][i & (CHUNK - 1)] } - - pub(crate) fn iter(&self) -> impl DoubleEndedIterator { - self.chunks.iter().flatten() - } } pub(crate) struct Trace { diff --git a/crates/leanvm_core/src/gkr.rs b/crates/leanvm_core/src/gkr.rs index c04afeff36..30ff50e61e 100644 --- a/crates/leanvm_core/src/gkr.rs +++ b/crates/leanvm_core/src/gkr.rs @@ -7,7 +7,8 @@ //! into `E` upstream, [`crate::leaf`]). //! //! On AVX-512 the large rounds run eight pairs of rows at a time in transposed form -//! (`vertical`), and skip the runs of rows known to be the identity. +//! (`vertical`), or with enough threads two rounds per pass, eight blocks of four rows at +//! a time (`paired`), and skip the runs of rows known to be the identity. use std::ops::Range; @@ -266,6 +267,12 @@ pub(crate) const SKIP_IDENTITY: bool = cfg!(target_arch = "x86_64"); #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] const VERTICAL_ROWS: usize = 1 << 10; +/// Threads from which a large layer runs its rounds two per pass ([`paired`]). Those +/// kernels multiply more to read the layer half as often, which pays only once the +/// threads together wait on memory; fewer run the rounds one at a time. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] +const PAIRED_THREADS: usize = 8; + /// Two binary product levels contracted into one degree-four layer. struct QuaternaryLayerState { /// Four child tables interleaved in their original order. This lets the @@ -279,8 +286,13 @@ struct QuaternaryLayerState { groups: Vec, #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] spare: Vec, + /// Once folded by the paired kernels, the rows live here instead. + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + quads: Vec, + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + spare_quads: Vec, /// Runs of rows known to be the identity, sorted: the kernels skip them, and - /// the vertical ones leave their groups unwritten. + /// the vertical ones leave their groups and quads unwritten. ones: Vec>, } @@ -298,6 +310,10 @@ impl QuaternaryLayerState { groups: Vec::new(), #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] spare: Vec::new(), + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + quads: Vec::new(), + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + spare_quads: Vec::new(), ones: Vec::new(), } } @@ -338,6 +354,27 @@ impl QuaternaryLayerState { } } + /// The paired kernels' view of the rows. + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + fn paired_input(&self) -> paired::Input<'_> { + let data = if self.quads.is_empty() { + paired::Data::Rows(&self.values) + } else { + paired::Data::Quads(&self.quads) + }; + paired::Input { data, ones: &self.ones } + } + + /// Move the rows the paired kernels left back into `values`. + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + fn leave_quads(&mut self) { + if !self.quads.is_empty() { + self.values = paired::to_rows(&self.quads, &self.ones, self.logical_rows); + self.quads = Vec::new(); + self.spare_quads = Vec::new(); + } + } + fn round_message_scalar(&self, eq: &SplitEq) -> [F192; 4] { let stored_rows = self.values.len() / 4; let full_pairs = stored_rows / 2; @@ -530,6 +567,26 @@ fn fold_and_message(trees: &mut [QuaternaryLayerState; 3], challenge: F192, eq: trees.each_mut().map(|tree| tree.fold_and_message_scalar(challenge, eq)) } +/// Bind every tree's next two variables to `r` in the paired kernels, returning the +/// polynomial of the two rounds after when given their weights. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] +fn fold_pair(trees: &mut [QuaternaryLayerState; 3], r: [F192; 2], eq: Option<&SplitEq>) -> Option<[paired::Grid; 3]> { + let mut spares = trees.each_mut().map(|tree| std::mem::take(&mut tree.spare_quads)); + let grids = paired::fold( + trees.each_ref().map(QuaternaryLayerState::paired_input), + r, + eq, + spares.each_mut(), + ); + for (tree, spare) in trees.iter_mut().zip(spares) { + tree.spare_quads = std::mem::replace(&mut tree.quads, spare); + tree.values = Vec::new(); + tree.logical_rows /= 4; + tree.ones = coarsen(&tree.ones, 2); + } + grids +} + /// The AVX-512 kernels: eight pairs of rows at a time, transposed so that every /// product is lane-wise ([`primitives::field::x8`]). After its first fold a layer /// keeps its rows transposed, in [`Group`]s, until it is small enough for the scalar @@ -543,7 +600,7 @@ mod vertical { use std::ops::Range; /// Whether `ones` covers `rows`. - fn covered(ones: &[Range], rows: Range) -> bool { + pub(super) fn covered(ones: &[Range], rows: Range) -> bool { let i = ones.partition_point(|r| r.start <= rows.start); i > 0 && ones[i - 1].end >= rows.end } @@ -779,6 +836,331 @@ mod vertical { } } +/// The AVX-512 kernels that run a layer's rounds two per pass. Four rows sharing all but +/// their two lowest index bits form a block, and each child of a block is bilinear in those +/// two bits: +/// +/// ```text +/// V_c(Y1, Y2) = v_00 + Y1 (v_00 + v_10) + Y2 (v_00 + v_01) + Y1 Y2 (v_00 + v_10 + v_01 + v_11) +/// ``` +/// +/// One pass sums `eq(q)·∏_c V_c(Y1, Y2)` over the blocks `q`, a [`Grid`] that gives the +/// first round's message, and the second's once `Y1` is bound. The next pass folds each +/// block into one row at both challenges and sums the next pair's grid from the folded +/// rows while they are in registers, so a layer is read once every two rounds. +/// +/// Eight blocks run at a time, transposed so that every product is lane-wise +/// ([`primitives::field::x8`]). After its first fold a layer keeps its rows so, in +/// [`Quad`]s, until it is small enough for the scalar kernels. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] +mod paired { + use super::vertical::covered; + use super::{SplitEq, coarsen}; + use primitives::field::F192; + use primitives::field::x8::{F192x8, Sum8}; + use primitives::stream::Stream; + use std::ops::Range; + + /// Eight blocks of four rows: `q[4b + c]` lane `p` is child `c` of row `4p + b`. + pub(super) type Quad = [F192x8; 16]; + + /// Quads per parallel task. + const TASK: usize = 32; + + /// Quads below which a dispatch costs more than it saves. + const PARALLEL: usize = 64; + + /// Quad `i` of a row table, the identity past its end. + #[inline(always)] + fn load_quad(values: &[F192], i: usize) -> Quad { + match values.get(128 * i..128 * i + 128) { + Some(run) => F192x8::load_blocks(run.try_into().unwrap()), + None => { + let mut padded = [F192::ONE; 128]; + let tail = &values[(128 * i).min(values.len())..]; + padded[..tail.len()].copy_from_slice(tail); + F192x8::load_blocks(&padded) + } + } + } + + /// Each block of `q` at `Y1 = r[0]`, then `Y2 = r[1]`: lane `p` of child `c` is block `p`'s row. + #[inline(always)] + fn fold_quad(q: &Quad, r: [F192x8; 2]) -> [F192x8; 4] { + std::array::from_fn(|c| { + let low = q[c] + r[0] * (q[c] + q[4 + c]); + let high = q[8 + c] + r[0] * (q[8 + c] + q[12 + c]); + low + r[1] * (low + high) + }) + } + + /// Four folded quads, rows `8k..8k + 8` in `rows[k]`, as the quad of the eight blocks they make. + #[inline(always)] + fn regroup(rows: [[F192x8; 4]; 4]) -> Quad { + let mut q = [F192x8::splat(F192::ZERO); 16]; + for c in 0..4 { + let (even01, odd01) = F192x8::deinterleave(rows[0][c], rows[1][c]); + let (even23, odd23) = F192x8::deinterleave(rows[2][c], rows[3][c]); + (q[c], q[8 + c]) = F192x8::deinterleave(even01, even23); + (q[4 + c], q[12 + c]) = F192x8::deinterleave(odd01, odd23); + } + q + } + + /// A value added by XOR: an element, a vector of them, or an array of either. + trait Xor: Copy { + fn xor(self, rhs: Self) -> Self; + } + + impl Xor for F192 { + #[inline(always)] + fn xor(self, rhs: Self) -> Self { + self + rhs + } + } + + impl Xor for F192x8 { + #[inline(always)] + fn xor(self, rhs: Self) -> Self { + self + rhs + } + } + + impl Xor for [T; N] { + #[inline(always)] + fn xor(self, rhs: Self) -> Self { + std::array::from_fn(|i| self[i].xor(rhs[i])) + } + } + + /// The product of two linear polynomials by Karatsuba, over the coefficient product `mul`. + #[inline(always)] + fn product2([u0, u1]: [T; 2], [w0, w1]: [T; 2], mul: impl Fn(T, T) -> P) -> [P; 3] { + let (low, high, sum) = (mul(u0, w0), mul(u1, w1), mul(u0.xor(u1), w0.xor(w1))); + [low, sum.xor(low).xor(high), high] + } + + /// Karatsuba's six products of two quadratics: the three of equal degrees, then the + /// product of each pair's sums. Being combined by sums alone ([`combine3`]), products + /// summed over many operands are combined once, after the sum. + #[inline(always)] + fn terms3(u: [T; 3], w: [T; 3], mul: impl Fn(T, T) -> P) -> [P; 6] { + let sum = |i: usize, j: usize| mul(u[i].xor(u[j]), w[i].xor(w[j])); + [ + mul(u[0], w[0]), + mul(u[1], w[1]), + mul(u[2], w[2]), + sum(0, 1), + sum(0, 2), + sum(1, 2), + ] + } + + /// The product's coefficients from [`terms3`]: each pair's sum holds its cross term + /// and the two products of equal degrees. + fn combine3([z0, z1, z2, s01, s02, s12]: [P; 6]) -> [P; 5] { + [ + z0, + s01.xor(z0).xor(z1), + s02.xor(z0).xor(z2).xor(z1), + s12.xor(z1).xor(z2), + z2, + ] + } + + /// The running sums of block products' Karatsuba terms, in `Y2` then `Y1`. + type Sums = [[Sum8; 6]; 6]; + + /// Add the terms of eight blocks' `weight·∏_c V_c`. The weight scales child 0, and with it the product. + #[inline(always)] + fn add_quad(sums: &mut Sums, q: &Quad, weight: F192x8) { + // Child `c` as `[[1, Y1], [Y2, Y1 Y2]]` coefficients. + let child = |c: usize| { + let y1 = q[c] + q[4 + c]; + [[q[c], y1], [q[c] + q[8 + c], y1 + q[8 + c] + q[12 + c]]] + }; + let [[a, b], [c, d]] = child(0); + let first = [[a * weight, b * weight], [c * weight, d * weight]]; + // Two pairs of bilinear factors make two biquadratics, nine products each. + let biquadratic = |a, b| product2(a, b, |u, w| product2(u, w, |x: F192x8, y| x * y)); + let (left, right) = (biquadratic(first, child(1)), biquadratic(child(2), child(3))); + for (sums, (u, w)) in sums.iter_mut().zip(terms3(left, right, |u, w| (u, w))) { + for (sum, (x, y)) in sums.iter_mut().zip(terms3(u, w, |x, y| (x, y))) { + *sum ^= x.mul_sum(y); + } + } + } + + /// `eq` at the eight blocks of quad `i`. + #[inline(always)] + fn weight(eq: &SplitEq, i: usize) -> F192x8 { + let shift = eq.lo_bits - 3; + eq.lo8[i & ((1 << shift) - 1)] * F192x8::splat(eq.hi[i >> shift]) + } + + /// `P(Y1, Y2) = Σ_q eq(q)·∏_c V_c(Y1, Y2)` of two consecutive rounds, its coefficients + /// indexed `[Y2 power][Y1 power]`. `Y1` is bound first. + pub(super) struct Grid([[F192; 5]; 5]); + + impl Grid { + fn new(sums: [[F192; 6]; 6]) -> Self { + Self(combine3(sums.map(combine3))) + } + + /// The first round's `[X¹..X⁴]`, its halves weighted by `eq(p, Y2)`. In characteristic + /// two that is `1 + p` at `Y2 = 0` and `p` at `Y2 = 1`, so `[X^k] = P_{0,k} + p·Σ_{m≥1} P_{m,k}`. + pub(super) fn first_round(&self, p: F192) -> [F192; 4] { + let [low, upper @ ..] = &self.0; + std::array::from_fn(|i| low[i + 1] + p * upper.iter().fold(F192::ZERO, |sum, row| sum + row[i + 1])) + } + + /// The second round's `[X¹..X⁴]` once `Y1` is bound to `r`: `[X^m] = Σ_k r^k P_{m,k}`. + pub(super) fn second_round(&self, r: F192) -> [F192; 4] { + std::array::from_fn(|i| self.0[i + 1].iter().rev().fold(F192::ZERO, |sum, &c| sum * r + c)) + } + } + + pub(super) enum Data<'a> { + Rows(&'a [F192]), + Quads(&'a [Quad]), + } + + /// A layer's rows as the kernels read them, with the runs known to be the identity. + pub(super) struct Input<'a> { + pub(super) data: Data<'a>, + pub(super) ones: &'a [Range], + } + + impl Input<'_> { + fn len(&self) -> usize { + match self.data { + Data::Rows(values) => values.len().div_ceil(128), + Data::Quads(quads) => quads.len(), + } + } + + /// Quad `i`, or `None` where it is the identity: past the end or where `ones` says so. + #[inline(always)] + fn get(&self, i: usize) -> Option { + if i >= self.len() || covered(self.ones, 32 * i..32 * i + 32) { + return None; + } + Some(match self.data { + Data::Rows(values) => load_quad(values, i), + Data::Quads(quads) => quads[i], + }) + } + } + + /// Each tree's grid from `task(t, quads)`'s reduced sums, over its `lens[t]` quads, + /// the three trees in one dispatch. + fn grids(lens: [usize; 3], task: impl Fn(usize, Range) -> [[F192; 6]; 6] + Sync) -> [Grid; 3] { + let tasks: Vec<(usize, usize)> = (0..3) + .flat_map(|t| (0..lens[t]).step_by(TASK).map(move |first| (t, first))) + .collect(); + let zero = [[[F192::ZERO; 6]; 6]; 3]; + let run = |i: usize| { + let (t, first) = tasks[i]; + let mut sums = zero; + sums[t] = task(t, first..(first + TASK).min(lens[t])); + sums + }; + let add = |a: [[[F192; 6]; 6]; 3], b| a.xor(b); + let sums = if lens.iter().sum::() > PARALLEL { + parallel::map_reduce(tasks.len(), || zero, run, add) + } else { + (0..tasks.len()).map(run).fold(zero, add) + }; + sums.map(Grid::new) + } + + /// Every tree's grid. An identity quad adds only to `P_{0,0}`, which neither message reads. + pub(super) fn grid(inputs: [Input; 3], eq: &SplitEq) -> [Grid; 3] { + grids(inputs.each_ref().map(Input::len), |t, quads| { + let mut sums = [[Sum8::ZERO; 6]; 6]; + for i in quads { + if let Some(q) = inputs[t].get(i) { + add_quad(&mut sums, &q, weight(eq, i)); + } + } + sums.map(|row| row.map(Sum8::reduce)) + }) + } + + /// Fold every tree's blocks at `r` into its `out`, in one dispatch for the three, + /// returning the next two rounds' grids when given their weights. + pub(super) fn fold( + inputs: [Input; 3], + r: [F192; 2], + eq: Option<&SplitEq>, + outs: [&mut Vec; 3], + ) -> Option<[Grid; 3]> { + let n_out = inputs.each_ref().map(|input| input.len().div_ceil(4)); + let mut outs = outs; + let dst: [parallel::SendPtr; 3] = std::array::from_fn(|t| { + let out = &mut *outs[t]; + if out.capacity() < n_out[t] { + *out = Vec::new(); + // SAFETY: the tasks below write every quad before anything reads it. + *out = unsafe { primitives::uninit_vec(n_out[t]) }; + } else { + // SAFETY: as above; `Quad` is plain data. + unsafe { out.set_len(n_out[t]) }; + } + parallel::SendPtr(out.as_mut_ptr()) + }); + let r = r.map(F192x8::splat); + let ones = inputs.each_ref().map(|input| coarsen(input.ones, 2)); + let identity = [F192x8::splat(F192::ONE); 4]; + let grids = grids(n_out, |t, quads| { + let stream = Stream::new(); + let mut sums = [[Sum8::ZERO; 6]; 6]; + for o in quads { + // Identity blocks fold to identity rows, which add nothing, and their + // quad is left unwritten. + if covered(&ones[t], 32 * o..32 * o + 32) { + continue; + } + let rows = std::array::from_fn(|k| inputs[t].get(4 * o + k).map_or(identity, |q| fold_quad(&q, r))); + let quad = regroup(rows); + if let Some(eq) = eq { + add_quad(&mut sums, &quad, weight(eq, o)); + } + // The next pass reads it, long after this one evicted it. + // SAFETY: tasks own disjoint quads of the outputs. + unsafe { stream.copy(dst[t].slice(o, 1), std::slice::from_ref(&quad)) }; + } + sums.map(|row| row.map(Sum8::reduce)) + }); + eq.map(|_| grids) + } + + /// The first `rows` rows of `quads`, identity quads filled in. + pub(super) fn to_rows(quads: &[Quad], ones: &[Range], rows: usize) -> Vec { + let mut values = vec![F192::ONE; 128 * quads.len()]; + for (i, (quad, out)) in quads.iter().zip(values.as_chunks_mut::<128>().0).enumerate() { + if covered(ones, 32 * i..32 * i + 32) { + continue; + } + for (j, lanes) in quad.iter().enumerate() { + for (p, e) in lanes.to_array().into_iter().enumerate() { + out[16 * p + j] = e; + } + } + } + values.truncate(4 * rows); + values + } +} + +/// Send a round's `[X¹..X⁴]` messages, batched by `lambda`, and draw its challenge. +fn send(messages: [[F192; 4]; 3], lambda: F192, ps: &mut ProverState) -> F192 { + let coeffs: [F192; 4] = + std::array::from_fn(|k| messages[0][k] + lambda * (messages[1][k] + lambda * messages[2][k])); + ps.add_scalars(&coeffs); + ps.sample() +} + /// The result of a batched grand-product proof: the three leaf evaluations, all reduced to one shared point. pub struct ProductTriple { pub point: Vec, @@ -841,23 +1223,42 @@ pub(crate) fn prove_product_triple(leaves: [Leaves; 3], ps: &mut ProverState) -> } } let mut round_point = Vec::with_capacity(round_count); - // Round `j` weights its pairs by `eq(point[1 + j..], ·)`. - let mut messages = if round_count > 0 { - let eq = SplitEq::new(&point[1..]); + // Round `j` weights its pairs by `eq(point[1 + j..], ·)`. Paired, the two rounds + // from `j` weight their blocks by `eq(point[2 + j..], ·)`. + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + if parallel::num_threads() >= PAIRED_THREADS { + let mut grids = (width >= VERTICAL_ROWS).then(|| { + let inputs = trees.each_ref().map(QuaternaryLayerState::paired_input); + paired::grid(inputs, &SplitEq::new(&point[2..])) + }); + while let Some(grid) = grids { + let j = round_point.len(); + let r0 = send(grid.each_ref().map(|g| g.first_round(point[j + 1])), lambda, ps); + let r1 = send(grid.each_ref().map(|g| g.second_round(r0)), lambda, ps); + round_point.extend([r0, r1]); + let next = (width >> (j + 2) >= VERTICAL_ROWS).then(|| SplitEq::new(&point[j + 4..])); + grids = fold_pair(&mut trees, [r0, r1], next.as_ref()); + } + for tree in &mut trees { + tree.leave_quads(); + } + } + let first = round_point.len(); + let mut messages = if first < round_count { + let eq = SplitEq::new(&point[first + 1..]); trees.each_ref().map(|tree| tree.round_message(&eq)) } else { [[F192::ZERO; 4]; 3] }; - for round in 0..round_count { - let mut coeffs = [0, 1, 2, 3].map(|coefficient| { - messages[0][coefficient] + lambda * (messages[1][coefficient] + lambda * messages[2][coefficient]) - }); - // The kernel accumulates `q(0) + q(1)`, the wire carries `c1`. This is + for round in first..round_count { + // The kernels accumulate `q(0) + q(1)`, the wire carries `c1`. This is // `ProverState::add_round_poly(_, true)` bar its dropped `c0`, which the // claim fixes and the prover therefore never forms. - coeffs[0] = coeffs[0] + coeffs[1] + coeffs[2] + coeffs[3]; - ps.add_scalars(&coeffs); - let challenge = ps.sample(); + let challenge = send( + messages.map(|[sum, c2, c3, c4]| [sum + c2 + c3 + c4, c2, c3, c4]), + lambda, + ps, + ); round_point.push(challenge); if round + 1 < round_count { let eq = SplitEq::new(&point[2 + round..]); @@ -978,6 +1379,68 @@ mod tests { } } + /// The paired kernels against the scalar ones round by round, over ragged prefixes + /// and runs of identity rows. + #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[test] + fn paired_rounds_match_single_rounds() { + let c1 = |[sum, c2, c3, c4]: [F192; 4]| [sum + c2 + c3 + c4, c2, c3, c4]; + for width in [1usize << 10, 1 << 13] { + let ones = [3..9, 64..200, 256..width - 24]; + let tree = |n: usize| { + let mut values: Vec = (0..n).map(|i| element(i, n)).collect(); + for row in ones.iter().flat_map(Clone::clone).filter(|&row| 4 * row < n) { + values[4 * row..(4 * row + 4).min(n)].fill(F192::ONE); + } + let mut tree = QuaternaryLayerState::new(values, width); + tree.ones = ones.to_vec(); + tree + }; + for lens in [[4 * width, 4 * width - 5, 4 * width / 3 + 7], [33, 4 * width - 1, 1000]] { + let mut reference = lens.map(tree); + let mut paired = lens.map(tree); + let point: Vec = (0..width.ilog2()).map(|i| F192::new(3 + u64::from(i), 1, 2)).collect(); + let eq = |from: usize| SplitEq::new(&point[from..]); + let mut messages = reference.each_ref().map(|tree| tree.round_message_scalar(&eq(1))); + let inputs = paired.each_ref().map(QuaternaryLayerState::paired_input); + let mut grids = Some(paired::grid(inputs, &eq(2))); + let mut j = 0; + while let Some(grid) = grids { + let r = [F192::new(j as u64 + 5, 13, 19), F192::new(j as u64 + 7, 3, 1)]; + for (g, &m) in grid.iter().zip(&messages) { + assert_eq!( + g.first_round(point[j + 1]), + c1(m), + "width={width}, lens={lens:?}, j={j}" + ); + } + messages = reference + .each_mut() + .map(|tree| tree.fold_and_message_scalar(r[0], &eq(j + 2))); + for (g, &m) in grid.iter().zip(&messages) { + assert_eq!(g.second_round(r[0]), c1(m), "width={width}, lens={lens:?}, j={j}"); + } + messages = reference + .each_mut() + .map(|tree| tree.fold_and_message_scalar(r[1], &eq(j + 3))); + let next = (width >> (j + 2) >= 32).then(|| eq(j + 4)); + grids = fold_pair(&mut paired, r, next.as_ref()); + j += 2; + } + for (p, r) in paired.iter_mut().zip(&mut reference) { + p.leave_quads(); + assert_eq!(p.round_message_scalar(&eq(j + 1)), r.round_message_scalar(&eq(j + 1))); + while r.logical_rows > 1 { + let challenge = F192::new(r.logical_rows as u64, 2, 9); + r.fold_scalar(challenge); + p.fold_scalar(challenge); + } + assert_eq!(p.children(), r.children(), "width={width}, lens={lens:?}"); + } + } + } + } + #[test] fn level_products_match_scalar() { for len in [4, 5, 63, 64, 65, 4 << 12, (4 << 12) + 3, (4 << 13) - 1] { @@ -1000,7 +1463,7 @@ mod tests { #[test] fn radix_four_claims_are_true_at_even_and_odd_depths() { - for mu in 0..=10 { + for mu in 0..=14 { let mut leaves: [Vec; 3] = [0, 1, 2].map(|lane| { (0..1usize << mu) .map(|row| F192::new((1 + row + lane * 100_003) as u64, row as u64, lane as u64)) @@ -1019,7 +1482,7 @@ mod tests { #[test] fn implicit_identity_suffix_matches_dense_padding() { - for mu in 3..=10 { + for mu in 3..=14 { let lengths = [(1usize << mu) - 3, (1usize << (mu - 1)) + 1, (1usize << (mu - 2)) + 3]; let mut leaves: [Vec; 3] = std::array::from_fn(|lane| { (0..lengths[lane]) diff --git a/crates/leanvm_core/src/keccak.rs b/crates/leanvm_core/src/keccak.rs index 085d3d12c6..4a293bbba2 100644 --- a/crates/leanvm_core/src/keccak.rs +++ b/crates/leanvm_core/src/keccak.rs @@ -102,11 +102,14 @@ pub(crate) fn state_of_cells(odd: bool, cell: impl Fn(usize) -> F192) -> [u64; S /// The thirteen canonical state cells holding 25 lanes in a layout, a free lane zero. pub(crate) fn cells_of_state(s: &[u64; STATE_LANES], odd: bool) -> [F192; STATE_CELLS] { - let lanes = cell_lanes(odd); - std::array::from_fn(|c| { - let (lo, hi) = lanes[c]; - F192::new(lo.map_or(0, |l| s[l]), hi.map_or(0, |h| s[h]), 0) - }) + // The layout is a constant, so the loop compiles to straight-line moves. + fn write(s: &[u64; STATE_LANES]) -> [F192; STATE_CELLS] { + std::array::from_fn(|c| { + let (lo, hi) = const { cell_lanes(ODD) }[c]; + F192::new(lo.map_or(0, |l| s[l]), hi.map_or(0, |h| s[h]), 0) + }) + } + if odd { write::(s) } else { write::(s) } } /// What a `KECCAK` instruction writes, given the cells it reads. diff --git a/crates/leanvm_core/src/leaf.rs b/crates/leanvm_core/src/leaf.rs index 5c617039d5..3e23a4697c 100644 --- a/crates/leanvm_core/src/leaf.rs +++ b/crates/leanvm_core/src/leaf.rs @@ -184,6 +184,7 @@ pub fn layout(blocks: &[Block]) -> Layout { /// A non-constant coordinate as `(source, coefficient)`: its leaf contribution is /// the mixed product `coeff · source(z)` with `source(z) ∈ K`, `coeff ∈ E`. /// `GCol` folds the `g^k` factor into the coefficient. +#[derive(Clone, Copy)] enum Term<'a> { Col(usize, F192), Prod(usize, usize, F192), @@ -191,6 +192,36 @@ enum Term<'a> { Public(&'a [F64], F192), } +impl Term<'_> { + /// Whether both are one source at one coefficient. + fn same(&self, other: &Self) -> bool { + match (self, other) { + (Term::Col(i, c), Term::Col(j, d)) => (i, c) == (j, d), + (Term::Prod(i, j, c), Term::Prod(k, l, d)) => (i, j, c) == (k, l, d), + (Term::Index(c), Term::Index(d)) => c == d, + (Term::Public(u, c), Term::Public(v, d)) => std::ptr::eq(*u, *v) && c == d, + _ => false, + } + } + + /// `coeff · source(z)`, unreduced, for the index column value `g^z = index`. + #[inline(always)] + fn at(&self, z: usize, cols: &[&[F64]], index: F64) -> F192Unreduced { + match self { + Term::Col(i, c) => c.mul_base_unreduced(cols[*i][z]), + Term::Prod(i, j, c) => c.mul_base_unreduced(cols[*i][z] * cols[*j][z]), + Term::Index(c) => c.mul_base_unreduced(index), + Term::Public(vals, c) => c.mul_base_unreduced(vals[z]), + } + } + + fn coeff(&self) -> F192 { + match self { + Term::Col(_, c) | Term::Prod(_, _, c) | Term::Index(c) | Term::Public(_, c) => *c, + } + } +} + /// Flatten one coordinate into leaf terms at coefficient `w`. A [`Coord::Sum`] /// spreads its children over the SAME `w`: they are one coordinate, so they share /// its `α`-power. @@ -228,65 +259,107 @@ struct LeafBlock<'a> { terms: Vec>, } -impl LeafBlock<'_> { - /// The leaf of real row `z`, whose index column value `g^z` is `index`. - #[inline] - fn leaf(&self, z: usize, cols: &[&[F64]], index: F64) -> F192 { - // The α-weighted coordinate sum defers its reductions: each mixed - // product contributes its three raw limb products (3 PMULL, no - // reduction tail), one combined reduction per row at the end, - // bit-identical to summing reduced `mul_base` terms. - let mut acc = F192Unreduced::ZERO; - for t in &self.terms { - acc ^= match t { - Term::Col(i, c) => c.mul_base_unreduced(cols[*i][z]), - Term::Prod(i, j, c) => c.mul_base_unreduced(cols[*i][z] * cols[*j][z]), - Term::Index(c) => c.mul_base_unreduced(index), - Term::Public(vals, c) => c.mul_base_unreduced(vals[z]), - }; +/// Blocks of one shape whose formulas share terms, as an interaction's push and pull +/// blocks over one table's columns do: member `m`'s leaf is its block's `const_part` +/// plus `shared` plus its own terms, so the shared products are formed once. +struct Family<'a, 'b> { + shared: Vec>, + members: Vec<(&'b LeafBlock<'a>, Vec>)>, +} + +/// The most blocks a [`Family`] holds. +const MEMBERS: usize = 2; + +impl<'a, 'b> Family<'a, 'b> { + fn single(blk: &'b LeafBlock<'a>) -> Self { + Self { + shared: blk.terms.clone(), + members: vec![(blk, Vec::new())], + } + } + + /// `blocks` (of one shape) as families, each block paired with the one sharing the + /// most terms with it, if any. + fn of(blocks: &[&'b LeafBlock<'a>]) -> Vec { + // The terms of `a` and `b` they share, then `a`'s and `b`'s others. + let split = |a: &[Term<'a>], b: &[Term<'a>]| { + let (mut shared, mut own_a, mut own_b) = (Vec::new(), Vec::new(), b.to_vec()); + for t in a { + match own_b.iter().position(|u| t.same(u)) { + Some(i) => shared.push(own_b.swap_remove(i)), + None => own_a.push(*t), + } + } + (shared, own_a, own_b) + }; + let mut left = blocks.to_vec(); + let mut families = Vec::new(); + while !left.is_empty() { + let a = left.remove(0); + let best = (0..left.len()).max_by_key(|&i| split(&a.terms, &left[i].terms).0.len()); + match best { + Some(i) if !split(&a.terms, &left[i].terms).0.is_empty() => { + let b = left.remove(i); + let (shared, own_a, own_b) = split(&a.terms, &b.terms); + families.push(Self { + shared, + members: vec![(a, own_a), (b, own_b)], + }); + } + _ => families.push(Self::single(a)), + } } - self.const_part + acc.reduce() + families } - /// The leaves of rows `z0..z0 + out.len()`. - fn fill(&self, z0: usize, out: &mut [F192], cols: &[&[F64]]) { - let real = self.rows.saturating_sub(z0).min(out.len()); - let (real_out, padding) = out.split_at_mut(real); - // A padding row's leaf is the identity, so it leaves the product alone. - padding.fill(F192::ONE); + /// Every member's leaves of rows `z0..z0 + len`, into `outs[m]`. + fn fill(&self, z0: usize, outs: &mut [&mut [F192]], cols: &[&[F64]]) { + debug_assert_eq!(outs.len(), self.members.len()); + let len = outs[0].len(); + let real = self.members[0].0.rows.saturating_sub(z0).min(len); + for out in outs.iter_mut() { + // A padding row's leaf is the identity, so it leaves the product alone. + out[real..].fill(F192::ONE); + } #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] - let done = self.fill_lanes(z0, real_out, cols); + let done = self.fill_lanes(z0, real, outs, cols); #[cfg(not(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq")))] let done = 0; + // Each mixed product contributes its raw limb products, and a leaf is reduced + // once, bit-identical to summing reduced `mul_base` terms. + let sum = |terms: &[Term], z: usize, index: F64| { + terms + .iter() + .fold(F192Unreduced::ZERO, |acc, t| acc ^ t.at(z, cols, index)) + }; let mut index = g_pow(z0 + done); - for (i, slot) in real_out.iter_mut().enumerate().skip(done) { - *slot = self.leaf(z0 + i, cols, index); + for i in done..real { + let shared = sum(&self.shared, z0 + i, index); + for ((blk, own), out) in self.members.iter().zip(outs.iter_mut()) { + out[i] = blk.const_part + (shared ^ sum(own, z0 + i, index)).reduce(); + } index = mul_by_g(index); } } - /// [`fill`](Self::fill) [`lanes::N`] rows at a time, every term one lane-wise mixed - /// product; returns how many rows it wrote. + /// [`fill`](Self::fill) over the first `real` rows, [`lanes::N`] at a time, every term one + /// lane-wise mixed product; returns how many rows it wrote. #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] - fn fill_lanes(&self, z0: usize, out: &mut [F192], cols: &[&[F64]]) -> usize { + fn fill_lanes(&self, z0: usize, real: usize, outs: &mut [&mut [F192]], cols: &[&[F64]]) -> usize { use lanes::{E, K, Mixed, N}; let load = |source: &[F64], z: usize| K::load(source[z..z + N].try_into().unwrap()); // `g^z` for the `N` rows, stepped by `g^N`. let mut index = K::new(std::array::from_fn(|i| g_pow(z0 + i))); let step = K::splat(g_pow(N)); - let coeffs: Vec = self - .terms + let coeffs = |terms: &[Term]| -> Vec { terms.iter().map(|t| E::splat(t.coeff())).collect() }; + let shared = coeffs(&self.shared); + let own: Vec<(E, Vec)> = self + .members .iter() - .map(|t| match t { - Term::Col(_, c) | Term::Prod(_, _, c) | Term::Index(c) | Term::Public(_, c) => E::splat(*c), - }) + .map(|(blk, terms)| (E::splat(blk.const_part), coeffs(terms))) .collect(); - let constant = E::splat(self.const_part); - let (chunks, _) = out.as_chunks_mut::(); - for (k, slot) in chunks.iter_mut().enumerate() { - let z = z0 + N * k; - let mut acc = Mixed::ZERO; - for (t, &c) in self.terms.iter().zip(&coeffs) { + let add = |acc: &mut Mixed, terms: &[Term], coeffs: &[E], z: usize, index: K| { + for (t, &c) in terms.iter().zip(coeffs) { match t { Term::Col(i, _) => acc.add(c, load(cols[*i], z)), Term::Prod(i, j, _) => acc.add(c, load(cols[*i], z) * load(cols[*j], z)), @@ -294,14 +367,23 @@ impl LeafBlock<'_> { Term::Public(vals, _) => acc.add(c, load(vals, z)), } } + }; + for k in 0..real / N { + let z = z0 + N * k; + let mut acc = Mixed::ZERO; + add(&mut acc, &self.shared, &shared, z, index); + for (((_, terms), (constant, c)), out) in self.members.iter().zip(&own).zip(outs.iter_mut()) { + let mut acc = acc; + add(&mut acc, terms, c, z, index); + (acc.reduce() + *constant).store((&mut out[z - z0..z - z0 + N]).try_into().unwrap()); + } index = index * step; - (acc.reduce() + constant).store(slot); } - N * chunks.len() + N * (real / N) } } -/// The lane-wise kernels [`LeafBlock::fill_lanes`] runs on: eight lanes with AVX-512 +/// The lane-wise kernels [`Family::fill_lanes`] runs on: eight lanes with AVX-512 /// ([`primitives::field::x8`]), four without ([`primitives::field::x4`]). #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] mod lanes { @@ -393,9 +475,12 @@ pub(crate) fn build_leaves(sides: [(&[Block], &Layout, &[F192], F192); 3], cols: .collect(); shapes.sort_unstable(); shapes.dedup(); - let groups: Vec> = shapes + let groups: Vec> = shapes .iter() - .map(|&shape| blocks.iter().filter(|b| (b.kappa, b.rows) == shape).collect()) + .map(|&shape| { + let group: Vec<&LeafBlock> = blocks.iter().filter(|b| (b.kappa, b.rows) == shape).collect(); + Family::of(&group) + }) .collect(); let tasks: Vec<(usize, Range)> = shapes .iter() @@ -404,7 +489,8 @@ pub(crate) fn build_leaves(sides: [(&[Block], &Layout, &[F192], F192); 3], cols: .flat_map(|(g, (&(kappa, _), group))| { let height = 1usize << kappa; // At least one sixteen per task, or its `sixteens` would go unwritten. - let rows = (TASK_LEAVES / group.len()).next_power_of_two().clamp(16, height); + let blocks: usize = group.iter().map(|family| family.members.len()).sum(); + let rows = (TASK_LEAVES / blocks).next_power_of_two().clamp(16, height); (0..height).step_by(rows).map(move |z| (g, z..z + rows)) }) .collect(); @@ -412,34 +498,39 @@ pub(crate) fn build_leaves(sides: [(&[Block], &Layout, &[F192], F192); 3], cols: .map(|level| level.each_mut().map(|v| parallel::SendPtr(v.as_mut_ptr()))); parallel::for_each(tasks.len(), |task| { let (group, ref rows) = tasks[task]; - let mut stage = [F192::ZERO; CHUNK]; + let mut stages = [[F192::ZERO; CHUNK]; MEMBERS]; let mut stage4 = [F192::ZERO; CHUNK / 4]; let mut stage16 = [F192::ZERO; CHUNK / 16]; // The GKR reads all three long after this pass has evicted them. let stream = primitives::stream::Stream::new(); for z0 in rows.clone().step_by(CHUNK) { let len = CHUNK.min(rows.len()); - let (leaves, fours, sixteens) = (&mut stage[..len], &mut stage4[..len / 4], &mut stage16[..len / 16]); - for blk in &groups[group] { - blk.fill(z0, leaves, cols); - // Past the block's real rows every leaf, and so every tuple's product, is the identity. - let live = if gkr::SKIP_IDENTITY { - blk.rows.saturating_sub(z0).next_multiple_of(16).min(len) - } else { - len - }; - gkr::products_into(&leaves[..live], &mut fours[..live / 4]); - gkr::products_into(&fours[..live / 4], &mut sixteens[..live / 16]); - fours[live / 4..].fill(F192::ONE); - sixteens[live / 16..].fill(F192::ONE); - let at = blk.offset + z0; - // SAFETY: a block's chunks are disjoint ranges of its side's leaves, blocks - // are disjoint, and a chunk owns the tuples it starts, `at` being a multiple - // of sixteen. - unsafe { - stream.copy(ptrs[0][blk.side].slice(at, len), leaves); - stream.copy(ptrs[1][blk.side].slice(at / 4, len / 4), fours); - stream.copy(ptrs[2][blk.side].slice(at / 16, len / 16), sixteens); + let (fours, sixteens) = (&mut stage4[..len / 4], &mut stage16[..len / 16]); + for family in &groups[group] { + let [first, second] = &mut stages; + let n = family.members.len(); + family.fill(z0, &mut [&mut first[..len], &mut second[..len]][..n], cols); + for ((blk, _), stage) in family.members.iter().zip(&stages) { + let leaves = &stage[..len]; + // Past the block's real rows every leaf, and so every tuple's product, is the identity. + let live = if gkr::SKIP_IDENTITY { + blk.rows.saturating_sub(z0).next_multiple_of(16).min(len) + } else { + len + }; + gkr::products_into(&leaves[..live], &mut fours[..live / 4]); + gkr::products_into(&fours[..live / 4], &mut sixteens[..live / 16]); + fours[live / 4..].fill(F192::ONE); + sixteens[live / 16..].fill(F192::ONE); + let at = blk.offset + z0; + // SAFETY: a block's chunks are disjoint ranges of its side's leaves, blocks + // are disjoint, and a chunk owns the tuples it starts, `at` being a multiple + // of sixteen. + unsafe { + stream.copy(ptrs[0][blk.side].slice(at, len), leaves); + stream.copy(ptrs[1][blk.side].slice(at / 4, len / 4), fours); + stream.copy(ptrs[2][blk.side].slice(at / 16, len / 16), sixteens); + } } } } @@ -448,7 +539,7 @@ pub(crate) fn build_leaves(sides: [(&[Block], &Layout, &[F192], F192); 3], cols: for (side, v) in values.iter_mut().enumerate() { let mut tail = lens[side]; for blk in blocks.iter().filter(|b| b.side == side && b.kappa < CHUNKED) { - blk.fill(0, &mut v[blk.offset..blk.offset + (1 << blk.kappa)], cols); + Family::single(blk).fill(0, &mut [&mut v[blk.offset..blk.offset + (1 << blk.kappa)]], cols); tail = tail.min(blk.offset); } if sides[side].0.is_empty() { diff --git a/crates/leanvm_core/src/tables.rs b/crates/leanvm_core/src/tables.rs index 00c648947b..68e51edde3 100644 --- a/crates/leanvm_core/src/tables.rs +++ b/crates/leanvm_core/src/tables.rs @@ -14,7 +14,7 @@ //! so a relation on machine words is written out lane by lane. A table hands its //! identities to the table sumcheck as a degree-2 form ([`Table::constraint`]). -use crate::cpu::{Drow, Jrow, Krow, Memory, Op, Rows, Srow, Trace}; +use crate::cpu::{Jrow, Memory, Op, Rows, Trace}; use crate::leaf::BusForm; use crate::leaf::Coord::{self, Col, Const, GCol, Prod}; use primitives::field::{F64, F192, mul_by_g}; @@ -183,6 +183,9 @@ pub struct FillCtx<'a> { /// buffer if the column is virtual. pub(crate) type ColumnOut<'a> = &'a mut [F64]; +/// Rows a fill gathers before streaming them out: a cache line of each column. +const LINE: usize = 8; + impl<'a> FillCtx<'a> { pub(crate) fn new( trace: &'a Trace, @@ -218,59 +221,58 @@ impl<'a> FillCtx<'a> { } } - /// Write local column `at`: `f` over the trace rows, then its padding value to the - /// end of the window. - fn col(&self, out: &mut [ColumnOut], rows: &Rows, at: usize, f: impl Fn(&R) -> F64 + Sync) { - self.cols(out, rows, at, |r| [f(r)]); - } - - /// Write the `N` local columns at `at..at + N` from one closure per row. - /// Columns fed by the same read fill together: the three lanes of a 192-bit - /// memory word are one random access, and splitting them across `N` passes - /// pays for it `N` times. + /// Write the local columns `cols` from one closure per row, then each column's + /// padding value to the end of its window. A table fills all its columns in one + /// pass: the row, its instruction and the memory words it names are read once, + /// and the three lanes of a 192-bit word are one random access. fn cols( &self, out: &mut [ColumnOut], rows: &Rows, - at: usize, + cols: [usize; N], f: impl Fn(&R) -> [F64; N] + Sync, ) { - self.cols_at(out, rows.len(), at, |i| f(rows.get(i))); + self.lines(out, rows.len(), cols, |start, len, buf| { + for j in 0..len { + let v = f(rows.get(start + j)); + for k in 0..N { + buf[k][j] = v[k]; + } + } + }); } - /// [`Self::cols`] over row indices, for values held in a side buffer rather - /// than read off the row. - fn cols_at( + /// [`Self::cols`] a line of rows at a time: `f(start, len, buf)` writes trace rows + /// `start..start + len` of column `cols[k]` to `buf[k][..len]`. + fn lines( &self, out: &mut [ColumnOut], n_rows: usize, - at: usize, - f: impl Fn(usize) -> [F64; N] + Sync, + cols: [usize; N], + f: impl Fn(usize, usize, &mut [[F64; LINE]; N]) + Sync, ) { let n = self.height; - let dst: [parallel::SendPtr; N] = std::array::from_fn(|k| { - assert_eq!(out[at + k].len(), n, "column {} has the wrong window length", at + k); - let col = at + k; + let dst: [parallel::SendPtr; N] = cols.map(|col| { + assert_eq!(out[col].len(), n, "column {col} has the wrong window length"); self.written[col / 64].fetch_or(1 << (col % 64), std::sync::atomic::Ordering::Relaxed); - parallel::SendPtr(out[at + k].as_mut_ptr()) + parallel::SendPtr(out[col].as_mut_ptr()) }); assert!(n_rows <= n, "a table's rows must fit its cube"); - let padding: [F64; N] = std::array::from_fn(|k| self.padding[at + k]); + let padding: [F64; N] = cols.map(|col| self.padding[col]); // A cache line of rows is gathered column by column, then streamed out: the // stack is read by a later pass, so its lines need not be fetched. - const LINE: usize = 8; parallel::for_each_chunk(n.div_ceil(LINE), |first, end| { let stream = primitives::stream::Stream::new(); for b in first..end { let start = b * LINE; let len = LINE.min(n - start); let mut buf = [[F64::ZERO; LINE]; N]; - for j in 0..len { - let i = start + j; - let v = if i < n_rows { f(i) } else { padding }; - for k in 0..N { - buf[k][j] = v[k]; - } + let rows = len.min(n_rows.saturating_sub(start)); + if rows > 0 { + f(start, rows, &mut buf); + } + for (column, &pad) in buf.iter_mut().zip(&padding) { + column[rows..len].fill(pad); } for (p, column) in dst.iter().zip(&buf) { // SAFETY: block `b` writes rows `start..start + len` of each of the @@ -521,14 +523,15 @@ impl Table for Arith { fn fill(&self, ctx: &FillCtx, out: &mut [ColumnOut]) { use arith::*; let rows = if self.is_xor { &ctx.trace.xor } else { &ctx.trace.mul }; - ctx.col(out, rows, PC, |r| ctx.g_at(r.pc)); - ctx.col(out, rows, FP, |r| ctx.g_at(r.fp)); - // The offsets and both operand words come out of ONE bytecode decode: split - // across passes, the row's instruction is fetched once per pass. - ctx.cols(out, rows, OA, |r| { + let cols = [ + PC, FP, OA, OB, OC, VA_LO, VA_HI, VA_TOP, VB_LO, VB_HI, VB_TOP, RA, RB, RC, RBC, + ]; + ctx.cols(out, rows, cols, |r| { let (a, b, c) = ctx.ternary_operands(r.pc); let (va, vb) = (ctx.limbs(r.fp + a), ctx.limbs(r.fp + b)); [ + ctx.g_at(r.pc), + ctx.g_at(r.fp), ctx.g_at(a), ctx.g_at(b), ctx.g_at(c), @@ -538,10 +541,12 @@ impl Table for Arith { vb[0], vb[1], vb[2], + ctx.g_at(r.ra), + ctx.g_at(r.rb), + ctx.g_at(r.rc), + ctx.g_at(r.bytecode_read), ] }); - ctx.cols(out, rows, RA, |r| [r.ra, r.rb, r.rc].map(|k| ctx.g_at(k))); - ctx.col(out, rows, RBC, |r| ctx.g_at(r.bytecode_read)); } } @@ -588,19 +593,23 @@ impl Table for SetTable { use set::*; let rows = &ctx.trace.set; // The offset and the stored immediate are the instruction's. - let imm = |r: &Srow| match ctx.prog[r.pc as usize] { - Op::Set { o, k } => (o, k), - op => unreachable!("a SET row's pc {} holds {op:?}", r.pc), - }; - ctx.col(out, rows, PC, |r| ctx.g_at(r.pc)); - ctx.col(out, rows, FP, |r| ctx.g_at(r.fp)); - ctx.col(out, rows, O, |r| ctx.g_at(imm(r).0)); - ctx.cols(out, rows, K_LO, |r| { - let k = imm(r).1; - k.to_limbs().map(F64) + ctx.cols(out, rows, [PC, FP, O, K_LO, K_HI, K_TOP, R, RBC], |r| { + let (o, k) = match ctx.prog[r.pc as usize] { + Op::Set { o, k } => (o, k), + op => unreachable!("a SET row's pc {} holds {op:?}", r.pc), + }; + let k = k.to_limbs().map(F64); + [ + ctx.g_at(r.pc), + ctx.g_at(r.fp), + ctx.g_at(o), + k[0], + k[1], + k[2], + ctx.g_at(r.r), + ctx.g_at(r.bytecode_read), + ] }); - ctx.col(out, rows, R, |r| ctx.g_at(r.r)); - ctx.col(out, rows, RBC, |r| ctx.g_at(r.bytecode_read)); } } @@ -671,36 +680,33 @@ impl Table for DerefTable { let rows = &ctx.trace.deref; // Offsets and store mode are the instruction's. Neither the store target's // address nor its value is committed. - let ins = |r: &Drow| match ctx.prog[r.pc as usize] { - Op::Deref { o1, o2, o3, mode } => (o1, o2, o3, mode), - op => unreachable!("a DEREF row's pc {} holds {op:?}", r.pc), - }; - ctx.col(out, rows, PC, |r| ctx.g_at(r.pc)); - ctx.col(out, rows, FP, |r| ctx.g_at(r.fp)); - debug_assert!( - rows.iter() - .all(|r| ctx.mem.word((r.fp + ins(r).0) as usize).as_base().is_some()), - "deref pointer must be K-valued" - ); - // The three offsets, the two mode flags, the pointer lane and the local - // word all follow from ONE bytecode decode. - ctx.cols(out, rows, O1, |r| { - let (o1, o2, o3, mode) = ins(r); + let cols = [PC, FP, O1, O2, O3, FPC, FFP, P, V3_LO, V3_HI, V3_TOP, R1, R2, R3, RBC]; + ctx.cols(out, rows, cols, |r| { + let (o1, o2, o3, mode) = match ctx.prog[r.pc as usize] { + Op::Deref { o1, o2, o3, mode } => (o1, o2, o3, mode), + op => unreachable!("a DEREF row's pc {} holds {op:?}", r.pc), + }; + let p = ctx.mem.word((r.fp + o1) as usize); + debug_assert!(p.as_base().is_some(), "deref pointer must be K-valued"); let v3 = ctx.limbs(r.fp + o3); [ + ctx.g_at(r.pc), + ctx.g_at(r.fp), ctx.g_at(o1), ctx.g_at(o2), ctx.g_at(o3), mode.f_pc(), mode.f_fp(), - F64(ctx.mem.word((r.fp + o1) as usize).c0), + F64(p.c0), v3[0], v3[1], v3[2], + ctx.g_at(r.r1), + ctx.g_at(r.r2), + ctx.g_at(r.r3), + ctx.g_at(r.bytecode_read), ] }); - ctx.cols(out, rows, R1, |r| [r.r1, r.r2, r.r3].map(|k| ctx.g_at(k))); - ctx.col(out, rows, RBC, |r| ctx.g_at(r.bytecode_read)); } } @@ -792,60 +798,62 @@ impl Table for JumpTable { op => unreachable!("a JUMP row's pc {} holds {op:?}", r.pc), }; let cell = |r: &Jrow, o: u32| ctx.mem.word((r.fp + o) as usize); - let cond = |r: &Jrow| { - let c = cell(r, ins(r).0); - debug_assert!(c.as_base().is_some(), "a JUMP condition outside K at pc {}", r.pc); - F64(c.c0) - }; - ctx.col(out, rows, PC, |r| ctx.g_at(r.pc)); - ctx.col(out, rows, FP, |r| ctx.g_at(r.fp)); - // The three offsets and the three cells they name come out of ONE decode. - // Those cells are K-valued on every row, taken or not (`cpu::execute` - // rejects anything else), so each is one lane and the memory flush carries - // literal zeros above it. - ctx.cols(out, rows, OC, |r| { - let (oc, od, of) = ins(r); - [ - ctx.g_at(oc), - ctx.g_at(od), - ctx.g_at(of), - F64(cell(r, oc).c0), - F64(cell(r, od).c0), - F64(cell(r, of).c0), - ] - }); - // The is-nonzero witness `w = c⁻¹` (0 where c = 0) for every row, in one - // batched Montgomery inversion. `prefix[i]` is the - // running product of the nonzero conditions before row `i`, so `acc` ends - // as their full product (nonzero, hence invertible). The taken indicator - // `b = [c ≠ 0]` falls out of the same pass, so it costs no extra decode. - // The conditions lie in K, a subfield, so the inversion runs in K. - let (w, b) = { + // Every column, in local order, so `buf[c]` is column `c`. + let cols: [usize; N] = std::array::from_fn(|c| c); + ctx.lines(out, rows.len(), cols, |start, len, buf| { + // The is-nonzero witness `w = c⁻¹` (0 where c = 0) by one batched Montgomery + // inversion per line: `prefix[j]` is the running product of the nonzero + // conditions before row `j`, so `acc` ends as their full product (nonzero, + // hence invertible). The conditions lie in K, a subfield, so the inversion + // runs in K. + let mut prefix = [F64::ONE; LINE]; let mut acc = F64::ONE; - let mut prefix: Vec = Vec::with_capacity(rows.len()); - let mut b = vec![F64::ZERO; rows.len()]; - for (i, r) in rows.iter().enumerate() { - prefix.push(acc); - let c = cond(r); - if !c.is_zero() { + for j in 0..len { + let r = rows.get(start + j); + let (oc, od, of) = ins(r); + // The three cells are K-valued on every row, taken or not + // (`cpu::execute` rejects anything else), so each is one lane and the + // memory flush carries literal zeros above it. + let c = cell(r, oc); + debug_assert!(c.as_base().is_some(), "a JUMP condition outside K at pc {}", r.pc); + let c = F64(c.c0); + let row = [ + (PC, ctx.g_at(r.pc)), + (FP, ctx.g_at(r.fp)), + (OC, ctx.g_at(oc)), + (OD, ctx.g_at(od)), + (OF, ctx.g_at(of)), + (V_COND, c), + (V_PC, F64(cell(r, od).c0)), + (V_FP, F64(cell(r, of).c0)), + (RC, ctx.g_at(r.rc)), + (RD, ctx.g_at(r.rd)), + (RF, ctx.g_at(r.rf)), + (RBC, ctx.g_at(r.bytecode_read)), + ]; + for (col, v) in row { + buf[col][j] = v; + } + prefix[j] = acc; + // The taken indicator `b = [c ≠ 0]`. + let taken = !c.is_zero(); + buf[B][j] = F64(u64::from(taken)); + if taken { acc *= c; - b[i] = F64::ONE; } } let mut inv = acc.inv(); - let mut w = vec![F64::ZERO; rows.len()]; - for i in (0..rows.len()).rev() { - let c = cond(rows.get(i)); - if !c.is_zero() { - w[i] = inv * prefix[i]; + for j in (0..len).rev() { + let c = buf[V_COND][j]; + buf[W][j] = if c.is_zero() { + F64::ZERO + } else { + let w = inv * prefix[j]; inv *= c; - } + w + }; } - (w, b) - }; - ctx.cols_at(out, rows.len(), W, |i| [w[i], b[i]]); - ctx.cols(out, rows, RC, |r| [r.rc, r.rd, r.rf].map(|k| ctx.g_at(k))); - ctx.col(out, rows, RBC, |r| ctx.g_at(r.bytecode_read)); + }); } } @@ -962,44 +970,43 @@ impl Table for KeccakTable { /// Every column but the input lanes, which are `q_keccak`'s and have no window here. The output lanes are the /// input permuted again, since a row writes only the output cells its bytecode requests. fn fill(&self, ctx: &FillCtx, out: &mut [ColumnOut]) { - use crate::keccak::state_of_cells; + use crate::keccak::{ROW_CELLS, state_of_cells}; use keccakt::*; + use primitives::hash::STATE_LANES; let rows = &ctx.trace.keccak; - let op = |r: &Krow| KeccakOp::of(&ctx.prog[r.pc as usize]); let flag = |b: bool| F64(u64::from(b)); - ctx.col(out, rows, PC, |r| ctx.g_at(r.pc)); - ctx.col(out, rows, FP, |r| ctx.g_at(r.fp)); - ctx.cols(out, rows, O_M0, |r| { - let k = op(r); - let [m0, m1, m2, m3, tail, b8, cap, o] = k.offsets.map(|o| ctx.g_at(o)); - let (cell, high) = FREE_LIMB[usize::from(k.in_odd)]; - let free = ctx.mem.word(k.cell(r.fp, cell) as usize); - let free = F64(if high { free.c1 } else { free.c0 }); - let (two, all) = k.out_cells.flags(); - [ - m0, - m1, - m2, - m3, - tail, - b8, - cap, - o, - flag(k.in_odd), - flag(k.out_odd), - flag(two), - flag(all), - free, - ] - }); - ctx.cols(out, rows, V_OUT, |r| { - let k = op(r); - let mut lanes = state_of_cells(k.in_odd, |c| ctx.mem.word(k.cell(r.fp, c) as usize)); - primitives::hash::keccak_f(&mut lanes); - lanes.map(F64) + // Every column but the input lanes, in local order: `buf[c]` is column `c` below + // `V_IN`, and column `c + STATE_LANES` from there. + const COLS: usize = N - STATE_LANES; + let cols: [usize; COLS] = std::array::from_fn(|c| if c < V_IN { c } else { c + STATE_LANES }); + ctx.lines(out, rows.len(), cols, |start, len, buf| { + for j in 0..len { + let r = rows.get(start + j); + let k = KeccakOp::of(&ctx.prog[r.pc as usize]); + buf[PC][j] = ctx.g_at(r.pc); + buf[FP][j] = ctx.g_at(r.fp); + for (i, &o) in k.offsets.iter().enumerate() { + buf[O_M0 + i][j] = ctx.g_at(o); + } + let (two, all) = k.out_cells.flags(); + buf[IN_ODD][j] = flag(k.in_odd); + buf[OUT_ODD][j] = flag(k.out_odd); + buf[OUT_TWO][j] = flag(two); + buf[OUT_ALL][j] = flag(all); + let (cell, high) = FREE_LIMB[usize::from(k.in_odd)]; + let free = ctx.mem.word(k.cell(r.fp, cell) as usize); + buf[FREE][j] = F64(if high { free.c1 } else { free.c0 }); + let mut lanes = state_of_cells(k.in_odd, |c| ctx.mem.word(k.cell(r.fp, c) as usize)); + primitives::hash::keccak_f(&mut lanes); + for (l, &lane) in lanes.iter().enumerate() { + buf[V_OUT - STATE_LANES + l][j] = F64(lane); + } + for c in 0..ROW_CELLS { + buf[R0 - STATE_LANES + c][j] = ctx.g_at(r.r[c]); + } + buf[RBC - STATE_LANES][j] = ctx.g_at(r.bytecode_read); + } }); - ctx.cols(out, rows, R0, |r| r.r.map(|k| ctx.g_at(k))); - ctx.col(out, rows, RBC, |r| ctx.g_at(r.bytecode_read)); } } diff --git a/crates/pcs/src/ntt/deep.rs b/crates/pcs/src/ntt/deep.rs index df056b3dfd..7558c39fb0 100644 --- a/crates/pcs/src/ntt/deep.rs +++ b/crates/pcs/src/ntt/deep.rs @@ -43,29 +43,33 @@ impl<'s> DeepPass<'s> { pub(super) fn run(&self, ntt: &AdditiveNttF64, win: &mut Window, src: Source<'_>) { match src { - Source::Rows(msg) => self.with_message(ntt, win, msg), + Source::Rows(msg) => self.with_message(ntt, Some(win), msg), _ => self.in_place(ntt, win), } } - /// A separate message: build each task in scratch, hash it, then write it out once. + /// A separate message and no buffer: each task is built in scratch and goes to the sink only. + pub(super) fn unkept(&self, ntt: &AdditiveNttF64, msg: &[F64]) { + self.with_message(ntt, None, msg); + } + + /// A separate message: build each task in scratch, hash it, then write it out once into `out`, if given. /// /// Streamed out whole, the codeword is written without ever being read. - fn with_message(&self, ntt: &AdditiveNttF64, win: &mut Window, msg: &[F64]) { + fn with_message(&self, ntt: &AdditiveNttF64, out: Option<&mut Window>, msg: &[F64]) { let Plan { log_d, num_ntts, - start, deep_start, .. } = self.plan; let (task_len, outer, log_task, sink) = (self.task_len, self.outer, self.log_task, self.sink); - let (base, total) = (win.base(), win.len()); - let block_len = total >> start; + let out = out.map(|win| (win.base(), win.stream())); + let block_len = msg.len(); parallel::for_each_chunk(1 << outer, |lo, hi| { with_scratch(task_len, |scratch| { // One fence at the end of the task covers all its streaming stores. - let stream = win.stream().then(Stream::new); + let stream = out.is_some_and(|(_, stream)| stream).then(Stream::new); for task in lo..hi { // A task sits at the same offset in every replica, or covers whole replicas. // @@ -81,6 +85,7 @@ impl<'s> DeepPass<'s> { if let Some(f) = sink { f(task << log_task, scratch); } + let Some((base, _)) = out else { continue }; // SAFETY: the task's rows lie inside the codeword, and this task alone owns them. let dst = unsafe { std::slice::from_raw_parts_mut(base.add(task * task_len), task_len) }; match &stream { diff --git a/crates/pcs/src/ntt/encode.rs b/crates/pcs/src/ntt/encode.rs index 6479ae85e6..5308707577 100644 --- a/crates/pcs/src/ntt/encode.rs +++ b/crates/pcs/src/ntt/encode.rs @@ -37,6 +37,7 @@ pub(crate) enum Codeword { /// All of it. Whole(Vec, Plan), /// The buffer as the gathered passes left it: the deep layers run again on each row read. + /// With no gathered pass, one replica of the message: every replica is the same. BeforeDeep(Vec, Plan), /// No buffer: each row is recomputed from the message of this many lane-major lanes. Nothing(Plan, usize), @@ -124,6 +125,8 @@ impl AdditiveNttF64 { /// - The codeword equals the in-place encode's, and every word is written before it is read. /// - With `defer_deep`, the deep pass is not written back: each deep sub-block is finished in scratch /// and handed to `on_rows` from there, and the codeword is kept [`Codeword::BeforeDeep`]. + /// - When every layer is deep, that is the message's replicas: only the message is kept, and no buffer + /// is written at all. pub(crate) fn encode_interleaved_with( &self, msg: &[F64], @@ -132,6 +135,12 @@ impl AdditiveNttF64 { defer_deep: bool, on_rows: &RowSink<'_>, ) -> Codeword { + let log_d = log2_strict_usize(msg.len() / num_ntts) + log_inv_rate; + let plan = Plan::new(log_d, num_ntts, log_inv_rate); + if defer_deep && plan.deep_start == plan.start && plan.fuses_rows() { + DeepPass::new(&plan, Some(on_rows), true).unkept(self, msg); + return Codeword::BeforeDeep(msg.to_vec(), plan); + } let mut done = None; // SAFETY: the fill writes every element before it reads any. let data = unsafe { diff --git a/crates/pcs/src/ntt/kernels.rs b/crates/pcs/src/ntt/kernels.rs index 301eb21582..5f27b10461 100644 --- a/crates/pcs/src/ntt/kernels.rs +++ b/crates/pcs/src/ntt/kernels.rs @@ -116,10 +116,10 @@ pub(super) fn butterfly_interleaved_fused_3layer( num_ntts, t, #[inline(always)] - |v, t| { + |v, t, vectors| { radix8_schedule( #[inline(always)] - |top, bot, k| butterfly_regs(v, top, bot, t[k]), + |top, bot, k| butterfly_regs(v, top, bot, t[k], vectors), ); }, ); @@ -180,7 +180,7 @@ fn fused_sweep_avx512( stride_rows: usize, num_ntts: usize, twiddles: &[F64; T], - kernel: impl Fn(&mut RowVectors, &[Twiddle; T]), + kernel: impl Fn(&mut RowVectors, &[Twiddle; T], usize), ) { fused_rows::(block, src, stride_rows, num_ntts, |src, rows| { fused_lanes_avx512( @@ -188,7 +188,7 @@ fn fused_sweep_avx512( rows, twiddles, #[inline(always)] - |v, t| kernel(v, t), + |v, t, vectors| kernel(v, t, vectors), ) }); } @@ -265,10 +265,10 @@ pub(super) fn butterfly_interleaved_fused_2layer( num_ntts, t, #[inline(always)] - |v, t| { + |v, t, vectors| { radix4_schedule( #[inline(always)] - |top, bot, k| butterfly_regs(v, top, bot, t[k]), + |top, bot, k| butterfly_regs(v, top, bot, t[k], vectors), ); }, ); @@ -641,12 +641,12 @@ type Twiddle = (core::arch::x86_64::__m512i, bool); #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] type RowVectors = [[core::arch::x86_64::__m512i; LANE_VECTORS]; N]; -/// Butterfly rows `top` and `bot` of every vector in flight. +/// Butterfly rows `top` and `bot` of the first `vectors` vectors in flight. #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] #[inline] #[target_feature(enable = "vpclmulqdq", enable = "avx512f", enable = "avx512bw")] -fn butterfly_regs(v: &mut RowVectors, top: usize, bot: usize, (tw, zero): Twiddle) { - for k in 0..LANE_VECTORS { +fn butterfly_regs(v: &mut RowVectors, top: usize, bot: usize, (tw, zero): Twiddle, vectors: usize) { + for k in 0..vectors { let (u, w) = (v[top][k], v[bot][k]); let new_u = if zero { u } else { mul_xor_avx512(u, w, tw) }; v[top][k] = new_u; @@ -659,7 +659,8 @@ fn butterfly_regs(v: &mut RowVectors, top: usize, bot: usize, /// - The vectors load from `src`, or from `rows` when there is none, and store to `rows`. /// - A vector is loaded and stored once, however many butterflies `kernel` applies to it. /// - The kernel orders its butterflies breadth-first, so the vectors in flight hide each other's latency. -/// - The last chunk may be short: its vectors past the row's end load as zeros and are not stored. +/// - The kernel runs on the first `vectors` vectors it is handed: all of them, but in the last chunk only those +/// that hold lanes. Their lanes past the row's end load as zeros and are not stored. #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] #[inline] #[target_feature(enable = "vpclmulqdq", enable = "avx512f", enable = "avx512bw")] @@ -667,7 +668,7 @@ fn fused_lanes_avx512( src: Option<[&[F64]; N]>, rows: &mut [&mut [F64]; N], twiddles: &[F64; T], - kernel: impl Fn(&mut RowVectors, &[Twiddle; T]), + kernel: impl Fn(&mut RowVectors, &[Twiddle; T], usize), ) { use core::arch::x86_64::*; use primitives::simd::{load, load_prefix, store, store_prefix}; @@ -690,7 +691,7 @@ fn fused_lanes_avx512( }), None => std::array::from_fn(|i| std::array::from_fn(|k| load(&*chunks[i], k))), }; - kernel(&mut v, &tw); + kernel(&mut v, &tw, LANE_VECTORS); for i in 0..N { for k in 0..LANE_VECTORS { store(chunks[i], k, v[i][k]); @@ -700,17 +701,23 @@ fn fused_lanes_avx512( } if lane < len { let reg = lane / 8; - // The lanes of vector `k` of the short chunk: all eight, the row's last few, or none. - let n = |k: usize| len.saturating_sub(lane + 8 * k).min(8); + // The lanes of vector `k` of the short chunk: all eight, or the row's last few. + let n = |k: usize| (len - lane - 8 * k).min(8); + let vectors = (len - lane).div_ceil(8); let mut v: RowVectors = std::array::from_fn(|i| { std::array::from_fn(|k| match &src { + _ if k >= vectors => _mm512_setzero_si512(), Some(src) => load_prefix(src[i], reg + k, n(k)), None => load_prefix(&*rows[i], reg + k, n(k)), }) }); - kernel(&mut v, &tw); + // Each count is its own call, so the kernel's loops over the vectors unroll. + match vectors { + 1 => kernel(&mut v, &tw, 1), + _ => kernel(&mut v, &tw, LANE_VECTORS), + } for i in 0..N { - for k in 0..LANE_VECTORS { + for k in 0..vectors { store_prefix(rows[i], reg + k, n(k), v[i][k]); } } diff --git a/crates/pcs/src/ntt/layers.rs b/crates/pcs/src/ntt/layers.rs index e517e708ca..9ba2adbdad 100644 --- a/crates/pcs/src/ntt/layers.rs +++ b/crates/pcs/src/ntt/layers.rs @@ -13,6 +13,8 @@ impl AdditiveNttF64 { /// - The domain has `2^log_d` rows and splits into `2^outer_log` equal sub-blocks. /// - The buffer is one of them; its index fixes the global block index, and so the twiddle, of each block. /// - Layers fuse into radix-8 sweeps, with radix-4 sweeps where a lone radix-2 would be left over. + /// - A buffer larger than L1 runs depth first: after a sweep, each of its blocks finishes alone, while in cache. + /// With `live`, it stays breadth first. /// /// With `live` (sorted), blocks that hold none of those buffer rows are left alone. /// @@ -101,10 +103,22 @@ impl AdditiveNttF64 { layer += 1; } src = None; + // A buffer larger than L1 goes depth first: each block of the sweep finishes alone, while it is in cache. + if live.is_none() && layer < log_d && buf.len() > DEPTH_FIRST_WORDS { + let block_elems = num_ntts << (log_d - layer); + let first = sub_idx << (layer - outer_log); + for (b, block) in buf.chunks_exact_mut(block_elems).enumerate() { + self.run_layers(block, None, log_d, num_ntts, layer, layer, first + b, None); + } + return; + } } } } +/// Words of a buffer whose layers run breadth first: one L1 data cache. +const DEPTH_FIRST_WORDS: usize = 1 << 12; + /// Call `f` with each block of `1 << log_rows` rows that holds a row of `live` (sorted). /// With no `live`, call it with every block of `num_blocks`. fn for_each_block(num_blocks: usize, log_rows: usize, live: Option<&[usize]>, mut f: impl FnMut(usize)) { diff --git a/crates/pcs/src/ntt/openings.rs b/crates/pcs/src/ntt/openings.rs index dd38b64d1b..3f142d00b0 100644 --- a/crates/pcs/src/ntt/openings.rs +++ b/crates/pcs/src/ntt/openings.rs @@ -68,7 +68,10 @@ impl AdditiveNttF64 { Codeword::Whole(ref words, plan) => gather_rows(words, plan.num_ntts, positions), Codeword::BeforeDeep(ref words, plan) => { let sub_len = plan.sub_len(); - self.finished_rows(&plan, positions, |_, sub| &words[sub * sub_len..][..sub_len]) + // A kept message is one replica, which every sub-block of a coset is a piece of. + self.finished_rows(&plan, positions, |_, sub| { + &words[(sub * sub_len) % words.len()..][..sub_len] + }) } Codeword::Nothing(plan, n_lanes) => { let log_sub = plan.log_sub(); diff --git a/crates/pcs/src/ntt/tests.rs b/crates/pcs/src/ntt/tests.rs index 4a84088230..78a6652151 100644 --- a/crates/pcs/src/ntt/tests.rs +++ b/crates/pcs/src/ntt/tests.rs @@ -292,13 +292,20 @@ fn lane_major_encode_matches_transpose_then_in_place() { /// must equal the full encode's, wherever the plan splits off a deep pass. Whether a shape /// splits depends on the cache budget and the pool, so only some shape must. /// -/// A deep-only plan with a separate message builds the whole codeword and returns `None`. +/// A deep-only plan with a separate message keeps only the message, and finishes each row from one replica of it: +/// `(9, 8, 1)` and, with tasks of several replicas, `(10, 48, 7)`. /// The in-place, deep-only, deferred branch runs in `lane_major_encode_matches_transpose_then_in_place`, /// at shape `(5, 2, 7)`. #[test] fn deferred_deep_rows_match_the_full_encode() { let mut rng = Rng::new(0xDEE9); - let shapes = [(14usize, 51usize, 1usize), (15, 64, 2), (9, 8, 1), (4, 8, 4)]; + let shapes = [ + (14usize, 51usize, 1usize), + (15, 64, 2), + (9, 8, 1), + (10, 48, 7), + (4, 8, 4), + ]; let mut splits = 0; for (log_d, lanes, log_inv_rate) in shapes { let ntt = AdditiveNttF64::standard(log_d); diff --git a/crates/primitives/src/field/x8.rs b/crates/primitives/src/field/x8.rs index 62fad9d8cf..ca800a8b90 100644 --- a/crates/primitives/src/field/x8.rs +++ b/crates/primitives/src/field/x8.rs @@ -164,6 +164,17 @@ impl F192x8 { std::array::from_fn(|j| Self(std::array::from_fn(|k| blocks[(3 * j + k) / 8][(3 * j + k) % 8]))) } + /// Eight runs of sixteen consecutive elements, transposed: `out[j]` lane `p` is `src[16p + j]`. + #[inline(always)] + pub fn load_blocks(src: &[F192; 128]) -> [Self; 16] { + let q = src.as_ptr().cast::(); + // As in `load_runs`, over six blocks of each run's 48 qwords. + // SAFETY: the 128 elements are 384 readable qwords. + let blocks: [[__m512i; 8]; 6] = + std::array::from_fn(|b| transpose8(std::array::from_fn(|p| unsafe { load_u64x8(q.add(48 * p + 8 * b)) }))); + std::array::from_fn(|j| Self(std::array::from_fn(|k| blocks[(3 * j + k) / 8][(3 * j + k) % 8]))) + } + /// The even lanes of `a` then of `b`, and the odd lanes of `a` then of `b`. #[inline(always)] pub fn deinterleave(a: Self, b: Self) -> (Self, Self) {