diff --git a/crates/experimental_prover/keccak_gkr/src/kernels.rs b/crates/experimental_prover/keccak_gkr/src/kernels.rs index c8f9b9db61..fbb4954bb6 100644 --- a/crates/experimental_prover/keccak_gkr/src/kernels.rs +++ b/crates/experimental_prover/keccak_gkr/src/kernels.rs @@ -1,10 +1,11 @@ //! The bit-level kernels of the layer prover, over one block position at a time: the 32 packs of //! one bit position of one block. //! -//! Each has a portable form, an AVX-512 one (AVX-512BW, VPCLMULQDQ, GFNI), an AVX2 one -//! (PCLMULQDQ, and VPCLMULQDQ where it is) and a NEON one (PMULL), which agree bit for bit. The -//! vector forms are compiled only where their features are. With AVX2 and GFNI a block's -//! [`skip_row`]s are summed on residues instead (`skip_gfni`). +//! Each has a portable form and, where it pays, an AVX-512 one (AVX-512BW, VPCLMULQDQ, GFNI), an +//! AVX2 one (PCLMULQDQ, and VPCLMULQDQ where it is) and a NEON one (PMULL), which agree bit for bit. +//! The vector forms are compiled only where their features are. With AVX2 and GFNI a block's +//! [`skip_row`]s are summed on residues instead (`skip_gfni`), and on x86 with AVX2 the table rows +//! come from byte maps (`table_maps`), so [`fold_groups`] keeps only its portable and NEON forms. use primitives::field::gf2_64::mul_wide; @@ -333,7 +334,10 @@ mod x86 { use core::arch::x86_64::*; use super::*; - pub use super::{embed_extend_portable as embed_extend, products_clmul as products, skip_row_portable as skip_row}; + pub use super::{ + embed_extend_portable as embed_extend, fold_groups_portable as fold_groups, products_clmul as products, + skip_row_portable as skip_row, + }; /// The matrices sending a pack's low and high byte to each byte of its embedding. An affine /// byte map reads output bit `t` off matrix byte `7 - t`. @@ -396,12 +400,6 @@ mod x86 { } } - /// Eight packs, a word each. - #[inline(always)] - unsafe fn load(p: *const u32) -> __m512i { - unsafe { _mm512_cvtepu32_epi64(_mm256_loadu_si256(p.cast())) } - } - /// Byte `k` of the embedding of each pack of `v`, in the low byte of the pack's 16 bits. #[inline(always)] unsafe fn embedding_byte(v: __m512i, k: usize) -> __m512i { @@ -414,41 +412,6 @@ mod x86 { ) } } - - /// The four sums are summed across their lanes together: two registers' lanes interleave into - /// one, which takes two shuffles where each register alone would leave the vector unit. - #[inline(always)] - pub fn fold_groups(e: &[u32; PACKS], weights: &[[u64; GROUP]; 2]) -> [[u128; 2]; 2] { - unsafe { - // `acc[2 * half + k]`, four lanes to sum. - let mut acc = [_mm512_setzero_si512(); 4]; - for g in 0..PACKS / 8 { - let ze = load(e.as_ptr().add(8 * g)); - for k in 0..2 { - let zw = _mm512_loadu_si512(weights[k].as_ptr().add(8 * (g % 2)).cast()); - let sum = &mut acc[2 * (g / 2) + k]; - *sum = _mm512_xor_si512(*sum, _mm512_clmulepi64_epi128(ze, zw, 0x00)); - *sum = _mm512_xor_si512(*sum, _mm512_clmulepi64_epi128(ze, zw, 0x11)); - } - } - // Lanes `a0 + a2, a1 + a3, b0 + b2, b1 + b3`. - let pair = |a: __m512i, b: __m512i| { - _mm512_xor_si512( - _mm512_shuffle_i64x2::<0b01_00_01_00>(a, b), - _mm512_shuffle_i64x2::<0b11_10_11_10>(a, b), - ) - }; - // Of two pairs, the four sums, in the order of `acc`. - let (t, u) = (pair(acc[0], acc[1]), pair(acc[2], acc[3])); - let four = _mm512_xor_si512( - _mm512_shuffle_i64x2::<0b10_00_10_00>(t, u), - _mm512_shuffle_i64x2::<0b11_01_11_01>(t, u), - ); - let mut a = [0u128; 4]; - _mm512_storeu_si512(a.as_mut_ptr().cast(), four); - [[a[0], a[1]], [a[2], a[3]]] - } - } } /// Compiled only where its features are, so its kernels are safe to call. The embedding is @@ -468,8 +431,8 @@ mod x86 { mod avx2 { use core::arch::x86_64::*; - pub use super::products_clmul as products; use super::*; + pub use super::{fold_groups_portable as fold_groups, products_clmul as products}; use crate::clmul::{V, Vector}; /// The 32 packs of a block position, embedded, four to a register. @@ -614,24 +577,6 @@ mod avx2 { } } } - - #[inline(always)] - pub fn fold_groups(e: &[u32; PACKS], weights: &[[u64; GROUP]; 2]) -> [[u128; 2]; 2] { - std::array::from_fn(|half| { - let mut acc = [V::zero(); 2]; - for q in (0..GROUP).step_by(2 * V::N) { - // SAFETY: `2 * N` packs and weights at `q`. - unsafe { - let ze = V::load_packs(e.as_ptr().add(GROUP * half + q)); - for (acc, weights) in acc.iter_mut().zip(weights) { - let zw = V::load(weights.as_ptr().add(q)); - *acc = acc.xor(ze.clmul::<0x00>(zw)).xor(ze.clmul::<0x11>(zw)); - } - } - } - acc.map(V::sum) - }) - } } /// Compiled only where its feature is, so its kernels are safe to call. diff --git a/crates/experimental_prover/keccak_gkr/src/lib.rs b/crates/experimental_prover/keccak_gkr/src/lib.rs index bc74084840..06b432c2af 100644 --- a/crates/experimental_prover/keccak_gkr/src/lib.rs +++ b/crates/experimental_prover/keccak_gkr/src/lib.rs @@ -82,14 +82,18 @@ //! arithmetic in the power basis of `s`, where a pack's value is its embedding's bits and a //! weighted sum is a carry-less product (`basis`, `power`). //! -//! `U`'s weighted bit sums are outer products: with GFNI (x86-64), one affine byte map per byte -//! of eight positions' weights applied to bit-transposed products (`products_gfni`); elsewhere, -//! the positions grouped by each byte of their weight (`prover`'s digit buckets). +//! `U`'s weighted bit sums are outer products: with AVX2 and GFNI, one affine byte map per byte of eight +//! positions' weights applied to the bit-transposed values of the products at six points of GF(2^8), which +//! the inverse evaluation map turns into coefficients once a layer (`products_gfni`); elsewhere, the +//! positions grouped by each byte of their weight (`prover`'s digit buckets). //! //! The four rounds' sums are binary polynomials of degree at most 173. With AVX2 and GFNI they //! are taken on residues modulo 24 irreducibles of degree 8, where a product is a byte product //! (`skip_gfni`). //! +//! With AVX2, the third walk weights each byte of a block's bits by a byte map, a GFNI affine map or two nibble +//! lookups, into its field element (`table_maps`). +//! //! The rounds that remain fold a table of one field element per half block and position. A pass over it //! binds two rounds' variables: over each quad of the rows it writes it sums the next round and, as a //! quadratic in that round's challenge, the round after, so the table is read once per two rounds (`prover`). @@ -116,13 +120,8 @@ mod prover; pub mod rmfe; #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] mod skip_gfni; -#[cfg(all( - target_arch = "x86_64", - target_feature = "avx512f", - target_feature = "avx512bw", - target_feature = "gfni" -))] -mod table_gfni; +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +mod table_maps; pub mod witness; #[cfg(test)] diff --git a/crates/experimental_prover/keccak_gkr/src/products_gfni.rs b/crates/experimental_prover/keccak_gkr/src/products_gfni.rs index 0461f90f1e..ab7c031975 100644 --- a/crates/experimental_prover/keccak_gkr/src/products_gfni.rs +++ b/crates/experimental_prover/keccak_gkr/src/products_gfni.rs @@ -1,37 +1,46 @@ -//! The product polynomial's bit sums by GFNI outer products (x86-64 with AVX2 and GFNI): the -//! same values as [`crate::prover`]'s digit buckets, `sum over positions j of weights[j] (x) -//! product(q, j)` for every pack `q` of a block and every coefficient of its product. +//! The product polynomial's bit sums by GFNI outer products (x86-64 with AVX2 and GFNI): once +//! [`coefficients`] has turned them back, the same values as [`crate::prover`]'s digit buckets, +//! `sum over positions j of weights[j] (x) product(q, j)` for every pack `q` of a block and every +//! coefficient of its product. +//! +//! A product of two embeddings has degree below `8 * POINTS`, so it is fixed by its values at +//! [`POINTS`] points of GF(2^8), the field of GF2P8MULB, with distinct minimal polynomials. A value +//! is a byte: the embeddings' values are affine byte maps of the witness's bytes ([`EVAL`]), and +//! a product's are byte products. The sums are taken over the values' bits, which the inverse of +//! the evaluation map ([`CRT`]) sends back to coefficients once a layer ([`coefficients`]). //! //! Eight consecutive `z` of one lane are a *lane byte*. For a lane byte's positions `j_0 .. j_7` //! and one byte `b` of their weights, GF2P8AFFINEQB with the matrix `M_b` maps a *column byte* -//! (bit `k` = one coefficient of `product(q, j_k)`) to byte `b` of the sum of the weights of the -//! positions whose coefficient is set: the lane byte's share of that coefficient's sum. One -//! instruction does this for 32 (pack, coefficient) column bytes at once. +//! (bit `k` = one bit of a value of `product(q, j_k)`) to byte `b` of the sum of the weights of +//! the positions whose bit is set: the lane byte's share of that bit's sum. One instruction does +//! this for 32 (pack, bit) column bytes at once. //! //! Per block, two phases: //! 1. per *row byte* with a nonzero weight (the same byte of the five lanes of a row, which `chi` -//! mixes): the five lane bytes' embeddings, the products of each active lane byte, and their -//! bit transposes into column bytes, staged register by register in a per-thread buffer; +//! mixes): the five lane bytes' values, the products of each active lane byte, and their byte +//! and bit transposes into column bytes, staged register by register in a per-thread buffer; //! 2. per staged register, every active lane byte's 16 matrices applied into accumulators held in //! registers (eight at a time on AVX2, all 16 with AVX-512), then one byte transpose back to //! field elements. //! -//! The 8x8 byte transpose without VPERMB ([`transpose_bytes8x8`]) leaves the positions in a fixed -//! order ([`TRANSPOSED_ORDER`]), which the matrices absorb ([`COLUMN_BIT_POSITION`]). -//! //! The two-phase outer-product scheme follows hashmancer-keccak's GFNI kernels; the code here is //! written for this crate's layout (16-instance packs, 128-bit weights). use core::arch::x86_64::*; use primitives::field::F128; -use verifier::keccak::BITS; +use verifier::keccak::{BITS, EMBED_BITS, PRODUCT_BITS}; -use crate::kernels::{PACKS, embed, products}; +use crate::kernels::PACKS; use crate::prover::PAYLOAD_BITS; -use crate::witness::words; +use crate::rmfe::RMFE; +use crate::table_maps::interleave_planes; +use crate::witness::{BLOCK_WORDS, words}; -/// Staged registers per active lane byte: 32 column bytes each, six coefficient bytes per pack. +/// Points a product is held at, a byte each. +const POINTS: usize = 6; +const _: () = assert!(PRODUCT_BITS <= 8 * POINTS && PAYLOAD_BITS == 8 * POINTS * PACKS); +/// Staged registers per active lane byte: 32 column bytes each, one per pack and bit of a value. const STAGED: usize = PAYLOAD_BITS / 32; /// Bytes of a weight. const WEIGHT_BYTES: usize = 16; @@ -41,68 +50,237 @@ type V = __m256i; /// The GF2P8AFFINEQB source that turns the instruction into an 8x8 bit transpose of its matrix. const BIT_TRANSPOSE: i64 = 0x8040_2010_0804_0201_u64 as i64; -/// The qword order [`transpose_bytes8x8`] leaves: output qword `d` byte `n` is byte `d` of input -/// `TRANSPOSED_ORDER[n]`. -const TRANSPOSED_ORDER: [usize; 8] = [0, 1, 4, 5, 2, 3, 6, 7]; /// The position that bit `i` of a column byte stands for, after the byte and bit transposes. -const COLUMN_BIT_POSITION: [usize; 8] = { - let mut p = [0; 8]; +const COLUMN_BIT_POSITION: [usize; 8] = [7, 6, 5, 4, 3, 2, 1, 0]; + +/// The byte product: binary polynomials modulo `x^8 + x^4 + x^3 + x + 1`. +const fn gf_mul(a: u8, b: u8) -> u8 { + let (mut a, mut product) = (a, 0); let mut i = 0; while i < 8 { - p[i] = TRANSPOSED_ORDER[7 - i]; + if b >> i & 1 == 1 { + product ^= a; + } + a = a << 1 ^ if a >> 7 == 1 { 0x1b } else { 0 }; + i += 1; + } + product +} + +/// `POWERS[i][n]` is point `i` to the power `n`. A point is the least of its eight conjugates, the +/// roots of its irreducible, so no two share one; an element of a proper subfield has fewer. +const POWERS: [[u8; 8 * POINTS]; POINTS] = { + let mut powers = [[1u8; 8 * POINTS]; POINTS]; + let (mut i, mut t) = (0, 0u8); + while i < POINTS { + t += 1; + let (mut conjugate, mut least) = (t, true); + let mut k = 1; + while k < 8 { + conjugate = gf_mul(conjugate, conjugate); + least &= conjugate > t; + k += 1; + } + if least { + let mut n = 1; + while n < 8 * POINTS { + powers[i][n] = gf_mul(powers[i][n - 1], t); + n += 1; + } + i += 1; + } + } + powers +}; + +/// The value at point `i` of the polynomial with coefficient bits `p`. +const fn value(p: u32, i: usize) -> u8 { + let (mut v, mut n) = (0, 0); + while n < EMBED_BITS { + if p >> n & 1 == 1 { + v ^= POWERS[i][n]; + } + n += 1; + } + v +} + +/// The matrix of the affine byte map sending bit `i` of a byte to `images[i]`: the map reads +/// output bit `t` off matrix byte `7 - t`. +const fn byte_map(images: [u8; 8]) -> u64 { + let mut matrix = 0u64; + let mut t = 0; + while t < 8 { + let mut i = 0; + while i < 8 { + matrix |= ((images[i] >> t & 1) as u64) << (8 * (7 - t) + i); + i += 1; + } + t += 1; + } + matrix +} + +/// `EVAL[h][i]` sends byte `h` of a pack to its share of the value of the pack's embedding at +/// point `i`. +const EVAL: [[u64; POINTS]; 2] = { + let mut maps = [[0u64; POINTS]; 2]; + let mut h = 0; + while h < 2 { + let mut i = 0; + while i < POINTS { + let mut images = [0u8; 8]; + let mut b = 0; + while b < 8 { + images[b] = value(RMFE.emb[8 * h + b], i); + b += 1; + } + maps[h][i] = byte_map(images); + i += 1; + } + h += 1; + } + maps +}; + +/// The value of `ones` at each point. +const ONES: [u8; POINTS] = { + let mut ones = [0u8; POINTS]; + let mut i = 0; + while i < POINTS { + ones[i] = value(RMFE.ones, i); i += 1; } - p + ones }; -/// In-lane byte interleave of the lane's two qwords (16-bit element `d` = byte `d` of both). -const INTERLEAVE: [u8; 32] = { +/// `CRT[n]` is the mask of the value bits (bit `t` of the value at point `i` at bit `8 i + t`) +/// whose sum is coefficient `n`: row `n` of the inverse of the evaluation map on polynomials of +/// fewer than `8 * POINTS` coefficients, by Gauss-Jordan elimination. +const CRT: [u64; 8 * POINTS] = { + const N: usize = 8 * POINTS; + // Row `8 i + t` sends a polynomial to bit `t` of its value at point `i`, beside the row of the + // identity that the row operations turn into the inverse's. + let mut rows = [[0u64; 2]; N]; + let mut r = 0; + while r < N { + let mut n = 0; + while n < N { + rows[r][0] |= ((POWERS[r / 8][n] >> (r % 8) & 1) as u64) << n; + n += 1; + } + rows[r][1] = 1 << r; + r += 1; + } + let mut c = 0; + while c < N { + let mut p = c; + while rows[p][0] >> c & 1 == 0 { + p += 1; + } + let pivot = rows[p]; + rows.swap(p, c); + let mut r = 0; + while r < N { + if r != c && rows[r][0] >> c & 1 == 1 { + rows[r][0] ^= pivot[0]; + rows[r][1] ^= pivot[1]; + } + r += 1; + } + c += 1; + } + let mut crt = [0u64; N]; + let mut n = 0; + while n < N { + crt[n] = rows[n][1]; + n += 1; + } + crt +}; + +/// `SLOT[q][8 i + t]` is the entry of [`Scratch::out`] holding bit `t` of pack `q`'s values at +/// point `i`. Value registers ([`evaluate`]) hold pack `16 s + 8 l + u` at byte `16 l + 8 s + u`, +/// and [`ProductSums::stage_lane_byte`] puts byte `16 l + 2 j + d` at qword `2 l + d` of register +/// `8 i + j`, whose byte `n` is entry `32 r + n`. +const SLOT: [[u16; 8 * POINTS]; PACKS] = { + let mut slot = [[0u16; 8 * POINTS]; PACKS]; + let mut i = 0; + while i < POINTS { + let mut j = 0; + while j < 8 { + let mut l = 0; + while l < 2 { + let mut d = 0; + while d < 2 { + let byte = 2 * j + d; + let q = 16 * (byte / 8) + 8 * l + byte % 8; + let mut t = 0; + while t < 8 { + slot[q][8 * i + t] = (32 * (8 * i + j) + 8 * (2 * l + d) + t) as u16; + t += 1; + } + d += 1; + } + l += 1; + } + j += 1; + } + i += 1; + } + slot +}; + +/// Per-lane byte shuffle of a pack's two bytes apart: the lane's low bytes, then its high ones. +const SPLIT: [u8; 32] = { let mut t = [0u8; 32]; let mut i = 0; while i < 32 { let j = i % 16; - t[i] = if j % 2 == 0 { (j / 2) as u8 } else { (8 + j / 2) as u8 }; + t[i] = if j < 8 { 2 * j as u8 } else { 2 * (j - 8) as u8 + 1 }; i += 1; } t }; -/// 8x8 byte transpose of the qwords `a_0 .. a_7` (`p` = `a_0 .. a_3`, `q` = `a_4 .. a_7`): output -/// qword `d` (`d < 4` in the first register) byte `n` = byte `d` of `a_{TRANSPOSED_ORDER[n]}`. Two -/// in-lane byte shuffles, two 16-bit unpacks and two dword permutes. -/// -/// # Safety -/// AVX2 (the module's build). +/// The values of a block position's 32 packs' embeddings at each point: register `i` holds point +/// `i`, pack `16 s + 8 l + u` at byte `16 l + 8 s + u`. #[inline(always)] -unsafe fn transpose_bytes8x8(p: V, q: V) -> [V; 2] { +fn evaluate(words: &[u64; BLOCK_WORDS]) -> [V; POINTS] { + // SAFETY: AVX2 and GFNI are enabled at compile time (module cfg); the loads are of the eight + // words and of `SPLIT`. unsafe { - let il = _mm256_loadu_si256(INTERLEAVE.as_ptr().cast()); - let (p, q) = (_mm256_shuffle_epi8(p, il), _mm256_shuffle_epi8(q, il)); - // Dword d of lane 0: bytes d of (a0, a1, a4, a5); of lane 1: of (a2, a3, a6, a7). - let (e, f) = (_mm256_unpacklo_epi16(p, q), _mm256_unpackhi_epi16(p, q)); - let idx = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); - [_mm256_permutevar8x32_epi32(e, idx), _mm256_permutevar8x32_epi32(f, idx)] + let split = _mm256_loadu_si256(SPLIT.as_ptr().cast()); + let w0 = _mm256_shuffle_epi8(_mm256_loadu_si256(words.as_ptr().cast()), split); + let w1 = _mm256_shuffle_epi8(_mm256_loadu_si256(words.as_ptr().add(4).cast()), split); + let (low, high) = (_mm256_unpacklo_epi64(w0, w1), _mm256_unpackhi_epi64(w0, w1)); + std::array::from_fn(|i| { + _mm256_xor_si256( + _mm256_gf2p8affine_epi64_epi8(low, _mm256_set1_epi64x(EVAL[0][i] as i64), 0), + _mm256_gf2p8affine_epi64_epi8(high, _mm256_set1_epi64x(EVAL[1][i] as i64), 0), + ) + }) } } -/// 4x4 transpose of qwords: `out[i]` qword `k` = `r[k]` qword `i`. -/// -/// # Safety -/// AVX2. -#[inline(always)] -unsafe fn transpose_qwords4x4(r: [V; 4]) -> [V; 4] { - unsafe { - let t0 = _mm256_unpacklo_epi64(r[0], r[1]); - let t1 = _mm256_unpackhi_epi64(r[0], r[1]); - let t2 = _mm256_unpacklo_epi64(r[2], r[3]); - let t3 = _mm256_unpackhi_epi64(r[2], r[3]); - [ - _mm256_permute2x128_si256(t0, t2, 0x20), - _mm256_permute2x128_si256(t1, t3, 0x20), - _mm256_permute2x128_si256(t0, t2, 0x31), - _mm256_permute2x128_si256(t1, t3, 0x31), - ] - } +/// Per pack, the coefficients of the weighted sums of its products, from the weighted sums of +/// their value bits (entry [`SLOT`]`[q][8 i + t]` of `sums`). +pub fn coefficients(sums: &[F128]) -> Vec<[F128; PRODUCT_BITS]> { + assert_eq!(sums.len(), PAYLOAD_BITS); + SLOT.iter() + .map(|slot| { + let bits: [F128; 8 * POINTS] = std::array::from_fn(|r| sums[slot[r] as usize]); + std::array::from_fn(|n| { + let mut mask = CRT[n]; + let mut c = F128::ZERO; + while mask != 0 { + c += bits[mask.trailing_zeros() as usize]; + mask &= mask - 1; + } + c + }) + }) + .collect() } /// Where staged register `r` of active lane byte `a` lies in the buffer (`n_active` active lane @@ -189,7 +367,7 @@ impl ProductSums { } /// The weighted bit sums of a block (`BITS` positions of eight words) into `s.out`: entry - /// `48 q + c` is `sum over j of weights[j] * coefficient c of product(q, j)`. + /// [`SLOT`]`[q][8 i + t]` is `sum over j of weights[j] * bit t of product(q, j) at point i`. pub fn block_sums(&self, s: &mut Scratch, block: &[u64]) { let n_active = self.matrices.len(); self.stage(block, &mut s.staged, n_active); @@ -200,26 +378,29 @@ impl ProductSums { /// Phase 1: every active lane byte's column bytes into `staged[staged_index(r, a, n_active)]`. fn stage(&self, block: &[u64], staged: &mut [V], n_active: usize) { assert!(staged.len() >= STAGED * n_active); - let mut embedded = [[[0u32; PACKS]; 8]; 5]; - let mut prod = [[0u64; PACKS]; 8]; + // SAFETY: AVX2 is enabled at compile time. + let mut values = [[[unsafe { _mm256_setzero_si256() }; POINTS]; 8]; 5]; let mut a = 0; for &(y, zb, mask) in &self.row_bytes { let (y, zb) = (y as usize, zb as usize); - for (x, lane) in embedded.iter_mut().enumerate() { - for (k, e) in lane.iter_mut().enumerate() { - *e = embed(words(block, 64 * (x + 5 * y) + 8 * zb + k)); + for (x, lane) in values.iter_mut().enumerate() { + for (k, v) in lane.iter_mut().enumerate() { + *v = evaluate(words(block, 64 * (x + 5 * y) + 8 * zb + k)); } } for x in (0..5).filter(|x| mask >> x & 1 == 1) { - let (e0, e1, e2) = (&embedded[x], &embedded[(x + 1) % 5], &embedded[(x + 2) % 5]); - for (k, p) in prod.iter_mut().enumerate() { - *p = products(&e0[k], &e1[k], &e2[k]); - } assert!(a < n_active, "one staged column per active lane byte"); // SAFETY: AVX2 and GFNI are enabled at compile time; `a < n_active` (asserted) and - // `stage_lane_byte` writes registers `r < STAGED` only (`3 * 15 + 2`), so every store + // `stage_lane_byte` writes registers `r < STAGED` only (`8 * 5 + 7`), so every store // is inside `staged` (asserted size). - unsafe { Self::stage_lane_byte(&prod, staged.as_mut_ptr(), a, n_active) }; + unsafe { + Self::stage_lane_byte( + [&values[x], &values[(x + 1) % 5], &values[(x + 2) % 5]], + staged.as_mut_ptr(), + a, + n_active, + ) + }; a += 1; } } @@ -227,38 +408,40 @@ impl ProductSums { assert_eq!(a, n_active, "every active lane byte staged"); } - /// The column bytes of one lane byte's products (`prod[k][q]`: position `k`, pack `q`): register - /// `r` holds column qwords `4 r .. 4 r + 4`, qword `6 q + m` being coefficient byte `m` of pack - /// `q` (bit `i` of its byte `b`: coefficient `8 m + b` of position `COLUMN_BIT_POSITION[i]`). + /// The column bytes of one lane byte's products, from the values of its three factors' + /// embeddings (`e[f][k][i]`: factor `f`, position `k`, point `i`): register `8 i + j` holds, in + /// qword `2 l + d`, the column byte of each bit of the products' values at point `i` for the + /// pack at byte `16 l + 2 j + d` of a value register ([`SLOT`]). /// /// # Safety /// AVX2 and GFNI; `staged + staged_index(r, a, n_active)` writable for `r < STAGED`. #[inline(always)] - unsafe fn stage_lane_byte(prod: &[[u64; PACKS]; 8], staged: *mut V, a: usize, n_active: usize) { + unsafe fn stage_lane_byte(e: [&[[V; POINTS]; 8]; 3], staged: *mut V, a: usize, n_active: usize) { unsafe { let bit_transpose = _mm256_set1_epi64x(BIT_TRANSPOSE); - let ld = |k: usize, q: usize| _mm256_loadu_si256(prod[k].as_ptr().add(q).cast()); - for qq in (0..PACKS).step_by(4) { - // t[h][i] qword k = the product of pack qq + i at position 4 h + k. - let t = [ - transpose_qwords4x4([ld(0, qq), ld(1, qq), ld(2, qq), ld(3, qq)]), - transpose_qwords4x4([ld(4, qq), ld(5, qq), ld(6, qq), ld(7, qq)]), - ]; - let columns: [[V; 2]; 4] = std::array::from_fn(|i| { - let [lo, hi] = transpose_bytes8x8(t[0][i], t[1][i]); - [ - _mm256_gf2p8affine_epi64_epi8(bit_transpose, lo, 0), - _mm256_gf2p8affine_epi64_epi8(bit_transpose, hi, 0), - ] + for i in 0..POINTS { + let ones = _mm256_set1_epi8(ONES[i] as i8); + // `ones * (a + c) + b * c` at point `i`, per position. + let p: [V; 8] = std::array::from_fn(|k| { + let (a, b, c) = (e[0][k][i], e[1][k][i], e[2][k][i]); + _mm256_xor_si256( + _mm256_gf2p8mul_epi8(_mm256_xor_si256(a, c), ones), + _mm256_gf2p8mul_epi8(b, c), + ) }); - for pair in 0..2 { - // Packs q, q + 1 take three registers: bytes 0..4 of q; 4, 5 of q and 0, 1 - // of q + 1; 2..4 and 4, 5 of q + 1. - let (c0, c1) = (columns[2 * pair], columns[2 * pair + 1]); - let r = 3 * ((qq + 2 * pair) / 2); - *staged.add(staged_index(r, a, n_active)) = c0[0]; - *staged.add(staged_index(r + 1, a, n_active)) = _mm256_permute2x128_si256(c0[1], c1[0], 0x20); - *staged.add(staged_index(r + 2, a, n_active)) = _mm256_permute2x128_si256(c1[0], c1[1], 0x21); + // 8x32 byte transpose: qword `d` of lane `l` of `w[j]` holds byte `16 l + 2 j + d` of each position. + let w = interleave_planes!( + p, + _mm256_unpacklo_epi8, + _mm256_unpackhi_epi8, + _mm256_unpacklo_epi16, + _mm256_unpackhi_epi16, + _mm256_unpacklo_epi32, + _mm256_unpackhi_epi32 + ); + for (j, w) in w.into_iter().enumerate() { + *staged.add(staged_index(8 * i + j, a, n_active)) = + _mm256_gf2p8affine_epi64_epi8(bit_transpose, w, 0); } } } diff --git a/crates/experimental_prover/keccak_gkr/src/prover.rs b/crates/experimental_prover/keccak_gkr/src/prover.rs index db472c174b..49832d3947 100644 --- a/crates/experimental_prover/keccak_gkr/src/prover.rs +++ b/crates/experimental_prover/keccak_gkr/src/prover.rs @@ -30,15 +30,15 @@ fn ternary(b: usize) -> usize { /// Digits a weight splits into, and the values of one. const DIGIT_BITS: usize = 8; const DIGITS: usize = 128 / DIGIT_BITS; +// `add_rows` sums a digit's buckets into its two nibbles. +const _: () = assert!(DIGIT_BITS == 8); /// The bits of a block position's products, 48 to each pack. pub(crate) const PAYLOAD_BITS: usize = 48 * PACKS; -/// The packs of a part of a block position, and the words of their products: with AVX-512 the -/// whole position, three registers; with AVX2 half, three registers; elsewhere a quarter, a -/// block's worth of which stays in cache. -#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] +/// The packs of a part of a block position, and the words of their products: with AVX2 the whole +/// position, three registers with AVX-512 and six without; elsewhere a quarter, a block's worth of +/// which stays in cache. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] const PART: usize = PACKS; -#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] -const PART: usize = PACKS / 2; #[cfg(not(all(target_arch = "x86_64", target_feature = "avx2")))] const PART: usize = 8; const PART_WORDS: usize = 48 * PART / 64; @@ -92,7 +92,7 @@ impl Part { } } #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] - // SAFETY: both parts are three registers, and AVX2 is enabled at compile time. + // SAFETY: both parts are six registers, and AVX2 is enabled at compile time. unsafe { use core::arch::x86_64::*; let (dst, src) = (self.0.as_mut_ptr(), other.0.as_ptr()); @@ -155,31 +155,42 @@ impl Groups { /// `sum over positions of weight[j] (x) payloads[j]` into `rows`: row `DIGIT_BITS * d + i` /// sums the payloads whose weight has that bit set. Each group is summed in registers, so - /// the payloads are only read. - fn add_rows(&self, payloads: &[Part; BITS], rows: &mut [Part; 128], sums: &mut [Part; 1 << DIGIT_BITS]) { + /// the payloads are only read, then added into the sums of its value's low nibble and of its + /// high one, the latter kept in registers; those make the rows of the digit's low bits and of + /// its high bits. + fn add_rows(&self, payloads: &[Part; BITS], rows: &mut [Part; 128]) { for (rows, (order, starts)) in rows .as_chunks_mut::() .0 .iter_mut() .zip(self.order.iter().zip(&self.starts)) { - for (v, sum) in sums.iter_mut().enumerate() { - let mut acc = Part::ZERO; - for &p in &order[starts[v] as usize..starts[v + 1] as usize] { - // SAFETY: `order` holds positions. - acc.add(unsafe { payloads.get_unchecked(p as usize) }); + let mut low = [Part::ZERO; 16]; + let mut high: [Part; 16] = std::array::from_fn(|h| { + let mut nibble = Part::ZERO; + for (l, low) in low.iter_mut().enumerate() { + let v = 16 * h + l; + let mut acc = Part::ZERO; + for &p in &order[starts[v] as usize..starts[v + 1] as usize] { + // SAFETY: `order` holds positions. + acc.add(unsafe { payloads.get_unchecked(p as usize) }); + } + nibble.add(&acc); + low.add(&acc); } - *sum = acc; - } - // Bit `i` of the digit: the sums whose value has it, then fold it away. - for (i, row) in rows.iter_mut().enumerate().rev() { - let (low, high) = sums[..2 << i].split_at_mut(1 << i); - let mut acc = Part::ZERO; - for (low, high) in low.iter_mut().zip(&*high) { - acc.add(high); - low.add(high); + nibble + }); + for (sums, rows) in [&mut low, &mut high].into_iter().zip(rows.as_chunks_mut::<4>().0) { + // Bit `i` of the nibble: the sums whose value has it, then fold it away. + for (i, row) in rows.iter_mut().enumerate().rev() { + let (low, high) = sums[..2 << i].split_at_mut(1 << i); + let mut acc = Part::ZERO; + for (low, high) in low.iter_mut().zip(&*high) { + acc.add(high); + low.add(high); + } + *row = acc; } - *row = acc; } } } @@ -272,9 +283,8 @@ impl Layer<'_> { a }, ); - (0..PACKS) - .map(|q| std::array::from_fn(|c| sums[48 * q + c].reduce())) - .collect() + let sums: Vec = sums.iter().map(|s| s.reduce()).collect(); + crate::products_gfni::coefficients(&sums) } /// [`Self::pack_products`] by digit buckets: the block's positions grouped by each digit of @@ -292,11 +302,10 @@ impl Layer<'_> { ( vec![[Part::ZERO; BITS]; PACKS / PART], vec![[Part::ZERO; 128]; PACKS / PART], - Box::new([Part::ZERO; 1 << DIGIT_BITS]), ) }, || vec![F128Unreduced::ZERO; PAYLOAD_BITS], - |(payloads, rows, digit_sums), sums, task| { + |(payloads, rows), sums, task| { let (_, _, block) = self.block(task); for r in 0..5 { for z in 0..64 { @@ -311,7 +320,7 @@ impl Layer<'_> { } } for (payloads, rows) in payloads.iter().zip(rows.iter_mut()) { - groups.add_rows(payloads, rows, digit_sums); + groups.add_rows(payloads, rows); } // Row `r` holds bit `r` of each payload bit's weighted sum: transposed, they are // field elements, which the block's weight multiplies. @@ -411,30 +420,20 @@ impl Layer<'_> { let field = round.field; let fold: Vec = eq_table(challenges).iter().map(|&e| field.to_poly(e)).collect(); let fold: [[u64; GROUP]; 2] = std::array::from_fn(|k| std::array::from_fn(|q| fold[q][k])); - #[cfg(all( - target_arch = "x86_64", - target_feature = "avx512f", - target_feature = "avx512bw", - target_feature = "gfni" - ))] - let row_maps = crate::table_gfni::RowMaps::new(field, &fold); + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] + let row_maps = crate::table_maps::RowMaps::new(field, &fold); // Rows of segments that do not exist are zero, and so is their product. table[self.rounds.len() * layout.live * ROW..layout.len() * ROW].fill(0); pads[self.rounds.len() * ROW..].fill(0); - // The GFNI rows need no wide scratch. - let gfni = cfg!(all( - target_arch = "x86_64", - target_feature = "avx512f", - target_feature = "avx512bw", - target_feature = "gfni" - )); - let scratch = || vec![0u64; if gfni { 0 } else { 6 * BITS }]; + // The byte-map rows need no wide scratch. + let maps = cfg!(all(target_arch = "x86_64", target_feature = "avx2")); + let scratch = || vec![0u64; if maps { 0 } else { 6 * BITS }]; // A block's rows: both groups', or the padding block's one. let build = |wide: &mut Vec, block: &[u64], rows: &mut [u64]| { let halves = rows.len() / ROW; cfg_select! { - all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx512bw", target_feature = "gfni") => { - // Each group's value by affine byte maps (`table_gfni`). + all(target_arch = "x86_64", target_feature = "avx2") => { + // Each group's value by byte maps (`table_maps`). let _ = wide; row_maps.rows(block, rows, halves); } @@ -871,6 +870,36 @@ fn round_coefficients(zero: F128, infinity: F128, value: F128, r: F128) -> [F128 [c0, c1, c2] } +#[cfg(test)] +mod tests { + use primitives::test_rng::Rng; + + use super::*; + + /// The digit buckets' rows against summing the payloads bit by bit, on random weights and the + /// nibble edges. + #[test] + fn rows_are_the_weighted_sums() { + let mut rng = Rng::new(0xB0C); + let mut weights: Vec = (0..BITS).map(|_| rng.ext()).collect(); + weights[0] = F128::new(u64::MAX, u64::MAX); + weights[1] = F128::ZERO; + weights[2] = F128::new(0x0F0F_0F0F_0F0F_0F0F, 0xF0F0_F0F0_F0F0_F0F0); + let payloads: [Part; BITS] = std::array::from_fn(|_| Part(std::array::from_fn(|_| rng.next_u64()))); + let mut rows = [Part(std::array::from_fn(|_| rng.next_u64())); 128]; + Groups::new(&weights).add_rows(&payloads, &mut rows); + for (r, row) in rows.iter().enumerate() { + let mut want = Part::ZERO; + for (weight, payload) in weights.iter().zip(&payloads) { + if weight.to_limbs()[r / 64] >> (r % 64) & 1 == 1 { + want.add(payload); + } + } + assert_eq!(row.0, want.0, "row {r}"); + } + } +} + #[cfg(all(test, target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] mod gfni_tests { use primitives::test_rng::Rng; diff --git a/crates/experimental_prover/keccak_gkr/src/table_gfni.rs b/crates/experimental_prover/keccak_gkr/src/table_gfni.rs deleted file mode 100644 index f07553ba2d..0000000000 --- a/crates/experimental_prover/keccak_gkr/src/table_gfni.rs +++ /dev/null @@ -1,243 +0,0 @@ -//! The table pass's rows by GFNI (x86-64 with AVX-512BW and GFNI): the same rows as embedding each -//! position's packs, weighting them by the folded challenges with carry-less products and reducing. -//! -//! A position's value in a group, `reduce(sum over packs q of embed(pack q) * fold[q])`, is -//! GF(2)-linear in the group's 256 instance bits, so it is a sum of one 8x8 bit matrix per (input -//! byte, output byte): byte `t` of the group's 32 bytes holds bits `8 (t % 2) ..` of pack `t / 2`, -//! and each of its bits maps to the reduced product of one unit embedding by that pack's weight. -//! A block is byte-transposed 64 positions at a time (register `b` = byte `b` of the 64 positions), -//! each group then costs `32 * 16` affine products per 64 positions, and the 16 output bytes are -//! transposed back to the row's words. -//! -//! The scheme is the table fill of hashmancer-keccak's GFNI kernels. - -use core::arch::x86_64::*; - -use primitives::field::gf2_64::mul_wide; -use verifier::keccak::BITS; - -use crate::kernels::GROUP; -use crate::power::PowerField; -use crate::rmfe::RMFE; -use crate::witness::BLOCK_WORDS; - -/// Bytes of a group of a position (16 packs of 16 bits), and of a reduced value. -const GROUP_BYTES: usize = 32; -const VALUE_BYTES: usize = 16; -/// Positions per register. -const REGISTER_POSITIONS: usize = 64; - -/// The affine maps of one layer's fold weights: `matrices[t][o]` sends byte `t` of a group to byte `o` -/// of its value. -pub struct RowMaps { - matrices: [[u64; VALUE_BYTES]; GROUP_BYTES], -} - -impl RowMaps { - /// From the fold weights `fold[k][q]` (word `k` of pack `q`'s weight, in the power basis). - pub fn new(field: &PowerField, fold: &[[u64; GROUP]; 2]) -> Self { - // The value of bit `i` of pack `q` alone, image `16 q + i`: its unit embedding times the pack's - // weight, reduced, a row of three-word products laid out by word as the carry-less path's. - const IMAGES: usize = 16 * GROUP; - let mut wide = vec![0u64; 3 * IMAGES]; - for q in 0..GROUP { - for (i, &e) in RMFE.emb.iter().enumerate() { - for (k, w) in fold.iter().enumerate() { - let x = mul_wide(u64::from(e), w[q]); - wide[k * IMAGES + 16 * q + i] ^= x as u64; - wide[(k + 1) * IMAGES + 16 * q + i] ^= (x >> 64) as u64; - } - } - } - let mut images = vec![0u64; 2 * IMAGES]; - field.reduce_row(&mut images, &wide, IMAGES); - let bit = |image: usize, n: usize| (images[(n / 64) * IMAGES + image] >> (n % 64)) as u8 & 1; - // An affine byte map reads output bit `k` off matrix byte `7 - k`: its bit `c` is bit `8 o + k` - // of the image of input bit `c`. - let matrices = std::array::from_fn(|t| { - std::array::from_fn(|o| { - (0..8).fold(0u64, |m, k| { - let row = (0..8).fold(0u8, |row, c| row | bit(16 * (t / 2) + 8 * (t % 2) + c, 8 * o + k) << c); - m | u64::from(row) << (8 * (7 - k)) - }) - }) - }); - Self { matrices } - } - - /// The rows of a block's first `halves` groups: row `h`, word `k` of position `j` at - /// `rows[h * 2 BITS + k * BITS + j]`. - pub fn rows(&self, block: &[u64], rows: &mut [u64], halves: usize) { - assert!(block.len() >= BITS * BLOCK_WORDS && halves <= 2 && rows.len() >= halves * 2 * BITS); - // SAFETY: AVX-512F/BW and GFNI are enabled at compile time (module cfg); every load reads - // positions inside `block` and every store writes words inside `rows` (asserted). - unsafe { - for g in 0..BITS / REGISTER_POSITIONS { - let y = transpose_position_bytes(block.as_ptr().add(REGISTER_POSITIONS * g * BLOCK_WORDS)); - for h in 0..halves { - let acc = self.accumulate(&y[GROUP_BYTES * h..][..GROUP_BYTES]); - for k in 0..2 { - let words = byte_planes_to_words(acc[8 * k..][..8].try_into().unwrap()); - let dst = rows.as_mut_ptr().add(h * 2 * BITS + k * BITS + REGISTER_POSITIONS * g); - for (r, w) in words.iter().enumerate() { - _mm512_storeu_si512(dst.add(8 * r).cast(), *w); - } - } - } - } - } - } - - /// Output byte `o` of the 64 positions: the sum over input bytes `t` of `matrices[t][o]` applied to `y[t]`. - #[inline] - #[target_feature(enable = "avx512f", enable = "avx512bw", enable = "gfni")] - fn accumulate(&self, y: &[__m512i]) -> [__m512i; VALUE_BYTES] { - let mut acc = [_mm512_setzero_si512(); VALUE_BYTES]; - for t in (0..GROUP_BYTES).step_by(2) { - let (x0, x1, m0, m1) = (y[t], y[t + 1], &self.matrices[t], &self.matrices[t + 1]); - for (o, a) in acc.iter_mut().enumerate() { - let g0 = _mm512_gf2p8affine_epi64_epi8::<0>(x0, _mm512_set1_epi64(m0[o] as i64)); - let g1 = _mm512_gf2p8affine_epi64_epi8::<0>(x1, _mm512_set1_epi64(m1[o] as i64)); - *a = _mm512_ternarylogic_epi64::<0x96>(*a, g0, g1); - } - } - acc - } -} - -/// 4x4 transpose of 128-bit lanes: `out[l]` lane `g` = `x[g]` lane `l`. -#[inline] -#[target_feature(enable = "avx512f")] -fn transpose_lanes(x: [__m512i; 4]) -> [__m512i; 4] { - let t0 = _mm512_shuffle_i64x2::<0b01_00_01_00>(x[0], x[1]); - let t1 = _mm512_shuffle_i64x2::<0b11_10_11_10>(x[0], x[1]); - let t2 = _mm512_shuffle_i64x2::<0b01_00_01_00>(x[2], x[3]); - let t3 = _mm512_shuffle_i64x2::<0b11_10_11_10>(x[2], x[3]); - [ - _mm512_shuffle_i64x2::<0b10_00_10_00>(t0, t2), - _mm512_shuffle_i64x2::<0b11_01_11_01>(t0, t2), - _mm512_shuffle_i64x2::<0b10_00_10_00>(t1, t3), - _mm512_shuffle_i64x2::<0b11_01_11_01>(t1, t3), - ] -} - -/// The 64 bytes of 64 consecutive positions (eight words each, at `p`), transposed: register `b` -/// byte `n` is byte `b` of position `n`. -/// -/// # Safety -/// `p` is valid for reading 64 positions; AVX-512F/BW. -#[inline] -#[target_feature(enable = "avx512f", enable = "avx512bw")] -unsafe fn transpose_position_bytes(p: *const u64) -> [__m512i; 64] { - let mut y = [[_mm512_setzero_si512(); 16]; 4]; - for (q, yq) in y.iter_mut().enumerate() { - for (n, v) in yq.iter_mut().enumerate() { - // SAFETY: position `16 q + n` lies inside the caller's 64. - *v = unsafe { _mm512_loadu_si512(p.add(BLOCK_WORDS * (16 * q + n)).cast()) }; - } - // yq[k] lane l byte n = byte 16 l + k of position 16 q + n. - primitives::bits::transpose_bytes16x16_per_lane512(yq); - } - let mut out = [_mm512_setzero_si512(); 64]; - for k in 0..16 { - // out[16 l + k] lane q = y[q][k] lane l. - for (l, v) in transpose_lanes([y[0][k], y[1][k], y[2][k], y[3][k]]) - .into_iter() - .enumerate() - { - out[16 * l + k] = v; - } - } - out -} - -/// Eight byte planes of 64 positions (plane `b` byte `n` = byte `b` of position `n`) as the -/// positions' words: register `r` holds positions `8 r .. 8 r + 8`. -#[inline] -#[target_feature(enable = "avx512f", enable = "avx512bw")] -fn byte_planes_to_words(a: &[__m512i; 8]) -> [__m512i; 8] { - // Per 128-bit lane `L` (positions 16 L ..): bytes, then 16-bit pairs, then 32-bit quads interleave. - let u: [__m512i; 8] = std::array::from_fn(|i| { - let (x, y) = (a[2 * (i / 2)], a[2 * (i / 2) + 1]); - if i % 2 == 0 { - _mm512_unpacklo_epi8(x, y) - } else { - _mm512_unpackhi_epi8(x, y) - } - }); - // u[2 m + s]: lane L, 16-bit element e = bytes of planes 2m, 2m + 1 for position 16 L + 8 s + e. - let v: [__m512i; 8] = std::array::from_fn(|i| { - let (s, half) = (i / 4, i % 4 / 2); - let (x, y) = (u[s], u[2 + s]); - let (x, y) = if i % 2 == 0 { (x, y) } else { (u[4 + s], u[6 + s]) }; - if half == 0 { - _mm512_unpacklo_epi16(x, y) - } else { - _mm512_unpackhi_epi16(x, y) - } - }); - // v[4 s + 2 h + c]: lane L, 32-bit element e = planes 4c .. 4c + 4 of position 16 L + 8 s + 4 h + e. - let w: [__m512i; 8] = std::array::from_fn(|i| { - let (j, hi) = (i / 2, i % 2); - let (s, h) = (j / 2, j % 2); - let (x, y) = (v[4 * s + 2 * h], v[4 * s + 2 * h + 1]); - if hi == 0 { - _mm512_unpacklo_epi32(x, y) - } else { - _mm512_unpackhi_epi32(x, y) - } - }); - // w[2 j + e2]: lane L, qword d = position 16 L + 4 j + 2 e2 + d, all eight planes: lane L of - // w[m] holds positions 16 L + 2 m, 16 L + 2 m + 1. - let lo = transpose_lanes([w[0], w[1], w[2], w[3]]); - let hi = transpose_lanes([w[4], w[5], w[6], w[7]]); - // lo[L] = positions 16 L .. 16 L + 8, hi[L] = 16 L + 8 .. 16 L + 16. - [lo[0], hi[0], lo[1], hi[1], lo[2], hi[2], lo[3], hi[3]] -} - -#[cfg(test)] -mod tests { - use primitives::test_rng::Rng; - - use super::*; - use crate::basis::carry; - use crate::kernels::{embed, fold_groups}; - use crate::power::Element; - use crate::witness::words; - - /// The rows the per-position path builds: embeddings weighted by carry-less products, reduced. - #[test] - fn rows_match_the_carry_less_path() { - let mut rng = Rng::new(0x7AB1E); - { - let field = PowerField::new(rng.ext()).expect("a generator"); - // The fold weights: field elements in the power basis, by word. - let elems: Vec = (0..GROUP).map(|_| field.to_poly(rng.ext())).collect(); - let fold: [[u64; GROUP]; 2] = std::array::from_fn(|k| std::array::from_fn(|q| elems[q][k])); - let block: Vec = (0..BITS * BLOCK_WORDS).map(|_| rng.next_u64()).collect(); - let mut want = vec![0u64; 2 * 2 * BITS]; - let mut wide = vec![0u64; 2 * 3 * BITS]; - for j in 0..BITS { - let sums = fold_groups(&embed(words(&block, j)), &fold); - for (h, sum) in sums.iter().enumerate() { - let sum: [u64; 3] = carry(sum); - for (k, &w) in sum.iter().enumerate() { - wide[h * 3 * BITS + k * BITS + j] = w; - } - } - } - for h in 0..2 { - field.reduce_row( - &mut want[h * 2 * BITS..][..2 * BITS], - &wide[h * 3 * BITS..][..3 * BITS], - BITS, - ); - } - for halves in [2, 1] { - let mut got = vec![0u64; 2 * 2 * BITS]; - RowMaps::new(&field, &fold).rows(&block, &mut got, halves); - assert_eq!(got[..halves * 2 * BITS], want[..halves * 2 * BITS], "{halves} halves"); - } - } - } -} diff --git a/crates/experimental_prover/keccak_gkr/src/table_maps.rs b/crates/experimental_prover/keccak_gkr/src/table_maps.rs new file mode 100644 index 0000000000..8b7909b126 --- /dev/null +++ b/crates/experimental_prover/keccak_gkr/src/table_maps.rs @@ -0,0 +1,422 @@ +//! The table pass's rows by byte maps (x86-64 with AVX2): the same rows as embedding each +//! position's packs, weighting them by the folded challenges with carry-less products and reducing. +//! +//! A position's value in a group, `reduce(sum over packs q of embed(pack q) * fold[q])`, is +//! GF(2)-linear in the group's 256 instance bits, so it is a sum of one 8x8 bit matrix per (input +//! byte, output byte): byte `t` of the group's 32 bytes holds bits `8 (t % 2) ..` of pack `t / 2`, +//! and each of its bits maps to the reduced product of one unit embedding by that pack's weight. +//! A block is byte-transposed a register of positions at a time (register `b` = byte `b` of the +//! register's positions), each group then costs `32 * 16` byte maps per register, and the 16 +//! output bytes are transposed back to the row's words. A byte map is one GFNI affine product, 512 +//! bits wide with AVX-512BW, or without GFNI two `vpshufb` nibble lookups. +//! +//! The scheme is the table fill of hashmancer-keccak's GFNI kernels. + +use core::arch::x86_64::*; + +use primitives::field::gf2_64::mul_wide; +use verifier::keccak::BITS; + +use crate::kernels::GROUP; +use crate::power::PowerField; +use crate::rmfe::RMFE; +use crate::witness::BLOCK_WORDS; + +/// Bytes of a group of a position (16 packs of 16 bits), and of a reduced value. +const GROUP_BYTES: usize = 32; +const VALUE_BYTES: usize = 16; + +/// The byte maps of one layer's fold weights: `matrices[t][o]` sends byte `t` of a group to byte `o` +/// of its value. +pub struct RowMaps { + matrices: [[Matrix; VALUE_BYTES]; GROUP_BYTES], +} + +impl RowMaps { + /// From the fold weights `fold[k][q]` (word `k` of pack `q`'s weight, in the power basis). + pub fn new(field: &PowerField, fold: &[[u64; GROUP]; 2]) -> Self { + // The value of bit `i` of pack `q` alone, image `16 q + i`: its unit embedding times the pack's + // weight, reduced, a row of three-word products laid out by word as the carry-less path's. + const IMAGES: usize = 16 * GROUP; + let mut wide = vec![0u64; 3 * IMAGES]; + for q in 0..GROUP { + for (i, &e) in RMFE.emb.iter().enumerate() { + for (k, w) in fold.iter().enumerate() { + let x = mul_wide(u64::from(e), w[q]); + wide[k * IMAGES + 16 * q + i] ^= x as u64; + wide[(k + 1) * IMAGES + 16 * q + i] ^= (x >> 64) as u64; + } + } + } + let mut images = vec![0u64; 2 * IMAGES]; + field.reduce_row(&mut images, &wide, IMAGES); + let bit = |image: usize, n: usize| (images[(n / 64) * IMAGES + image] >> (n % 64)) as u8 & 1; + // An affine byte map reads output bit `k` off matrix byte `7 - k`: its bit `c` is bit `8 o + k` + // of the image of input bit `c`. + let matrices = std::array::from_fn(|t| { + std::array::from_fn(|o| { + (0..8).fold(0u64, |m, k| { + let row = (0..8).fold(0u8, |row, c| row | bit(16 * (t / 2) + 8 * (t % 2) + c, 8 * o + k) << c); + m | u64::from(row) << (8 * (7 - k)) + }) + }) + }); + Self { + matrices: matrices.map(|m| m.map(matrix)), + } + } +} + +/// The three in-lane unpack rounds from eight byte planes `$a` (plane `b` byte `n` of a 128-bit lane +/// `L` = byte `b` of position `16 L + n`) to the positions' words: register `m` lane `L` holds positions +/// `16 L + 2 m` and `16 L + 2 m + 1`. +macro_rules! interleave_planes { + ($a:expr, $lo8:ident, $hi8:ident, $lo16:ident, $hi16:ident, $lo32:ident, $hi32:ident) => {{ + let a = $a; + // Per lane: bytes, then 16-bit pairs, then 32-bit quads interleave. + let u: [_; 8] = std::array::from_fn(|i| { + let (x, y) = (a[2 * (i / 2)], a[2 * (i / 2) + 1]); + if i % 2 == 0 { $lo8(x, y) } else { $hi8(x, y) } + }); + // u[2 m + s]: lane L, 16-bit element e = bytes of planes 2m, 2m + 1 for position 16 L + 8 s + e. + let v: [_; 8] = std::array::from_fn(|i| { + let (s, half) = (i / 4, i % 4 / 2); + let (x, y) = if i % 2 == 0 { + (u[s], u[2 + s]) + } else { + (u[4 + s], u[6 + s]) + }; + if half == 0 { $lo16(x, y) } else { $hi16(x, y) } + }); + // v[4 s + 2 h + c]: lane L, 32-bit element e = planes 4c .. 4c + 4 of position 16 L + 8 s + 4 h + e. + let w: [_; 8] = std::array::from_fn(|i| { + let (j, hi) = (i / 2, i % 2); + let (s, h) = (j / 2, j % 2); + let (x, y) = (v[4 * s + 2 * h], v[4 * s + 2 * h + 1]); + if hi == 0 { $lo32(x, y) } else { $hi32(x, y) } + }); + w + }}; +} +#[cfg(target_feature = "gfni")] +pub(crate) use interleave_planes; + +cfg_select! { + all(target_feature = "avx512f", target_feature = "avx512bw", target_feature = "gfni") => { + /// Positions per register. + const REGISTER_POSITIONS: usize = 64; + + /// A matrix, broadcast by the affine product's memory operand. + type Matrix = u64; + + #[inline] + fn matrix(m: u64) -> Matrix { + m + } + + impl RowMaps { + /// The rows of a block's first `halves` groups: row `h`, word `k` of position `j` at + /// `rows[h * 2 BITS + k * BITS + j]`. + pub fn rows(&self, block: &[u64], rows: &mut [u64], halves: usize) { + assert!(block.len() >= BITS * BLOCK_WORDS && halves <= 2 && rows.len() >= halves * 2 * BITS); + // SAFETY: AVX-512F/BW and GFNI are enabled at compile time (module cfg); every load reads + // positions inside `block` and every store writes words inside `rows` (asserted). + unsafe { + for g in 0..BITS / REGISTER_POSITIONS { + let y = transpose_position_bytes(block.as_ptr().add(REGISTER_POSITIONS * g * BLOCK_WORDS)); + for h in 0..halves { + let acc = self.accumulate(&y[GROUP_BYTES * h..][..GROUP_BYTES]); + for k in 0..2 { + let words = byte_planes_to_words(acc[8 * k..][..8].try_into().unwrap()); + let dst = rows.as_mut_ptr().add(h * 2 * BITS + k * BITS + REGISTER_POSITIONS * g); + for (r, w) in words.iter().enumerate() { + _mm512_storeu_si512(dst.add(8 * r).cast(), *w); + } + } + } + } + } + } + + /// Output byte `o` of the 64 positions: the sum over input bytes `t` of `matrices[t][o]` applied to `y[t]`. + #[inline] + #[target_feature(enable = "avx512f", enable = "avx512bw", enable = "gfni")] + fn accumulate(&self, y: &[__m512i]) -> [__m512i; VALUE_BYTES] { + let mut acc = [_mm512_setzero_si512(); VALUE_BYTES]; + for t in (0..GROUP_BYTES).step_by(2) { + let (x0, x1, m0, m1) = (y[t], y[t + 1], &self.matrices[t], &self.matrices[t + 1]); + for (o, a) in acc.iter_mut().enumerate() { + let g0 = _mm512_gf2p8affine_epi64_epi8::<0>(x0, _mm512_set1_epi64(m0[o] as i64)); + let g1 = _mm512_gf2p8affine_epi64_epi8::<0>(x1, _mm512_set1_epi64(m1[o] as i64)); + *a = _mm512_ternarylogic_epi64::<0x96>(*a, g0, g1); + } + } + acc + } + } + + /// 4x4 transpose of 128-bit lanes: `out[l]` lane `g` = `x[g]` lane `l`. + #[inline] + #[target_feature(enable = "avx512f")] + fn transpose_lanes(x: [__m512i; 4]) -> [__m512i; 4] { + let t0 = _mm512_shuffle_i64x2::<0b01_00_01_00>(x[0], x[1]); + let t1 = _mm512_shuffle_i64x2::<0b11_10_11_10>(x[0], x[1]); + let t2 = _mm512_shuffle_i64x2::<0b01_00_01_00>(x[2], x[3]); + let t3 = _mm512_shuffle_i64x2::<0b11_10_11_10>(x[2], x[3]); + [ + _mm512_shuffle_i64x2::<0b10_00_10_00>(t0, t2), + _mm512_shuffle_i64x2::<0b11_01_11_01>(t0, t2), + _mm512_shuffle_i64x2::<0b10_00_10_00>(t1, t3), + _mm512_shuffle_i64x2::<0b11_01_11_01>(t1, t3), + ] + } + + /// The 64 bytes of 64 consecutive positions (eight words each, at `p`), transposed: register `b` + /// byte `n` is byte `b` of position `n`. + /// + /// # Safety + /// `p` is valid for reading 64 positions; AVX-512F/BW. + #[inline] + #[target_feature(enable = "avx512f", enable = "avx512bw")] + unsafe fn transpose_position_bytes(p: *const u64) -> [__m512i; 64] { + let mut y = [[_mm512_setzero_si512(); 16]; 4]; + for (q, yq) in y.iter_mut().enumerate() { + for (n, v) in yq.iter_mut().enumerate() { + // SAFETY: position `16 q + n` lies inside the caller's 64. + *v = unsafe { _mm512_loadu_si512(p.add(BLOCK_WORDS * (16 * q + n)).cast()) }; + } + // yq[k] lane l byte n = byte 16 l + k of position 16 q + n. + primitives::bits::transpose_bytes16x16_per_lane512(yq); + } + let mut out = [_mm512_setzero_si512(); 64]; + for k in 0..16 { + // out[16 l + k] lane q = y[q][k] lane l. + for (l, v) in transpose_lanes([y[0][k], y[1][k], y[2][k], y[3][k]]) + .into_iter() + .enumerate() + { + out[16 * l + k] = v; + } + } + out + } + + /// Eight byte planes of 64 positions (plane `b` byte `n` = byte `b` of position `n`) as the + /// positions' words: register `r` holds positions `8 r .. 8 r + 8`. + #[inline] + #[target_feature(enable = "avx512f", enable = "avx512bw")] + fn byte_planes_to_words(a: &[__m512i; 8]) -> [__m512i; 8] { + let w = interleave_planes!( + a, + _mm512_unpacklo_epi8, + _mm512_unpackhi_epi8, + _mm512_unpacklo_epi16, + _mm512_unpackhi_epi16, + _mm512_unpacklo_epi32, + _mm512_unpackhi_epi32 + ); + let lo = transpose_lanes([w[0], w[1], w[2], w[3]]); + let hi = transpose_lanes([w[4], w[5], w[6], w[7]]); + // lo[L] = positions 16 L .. 16 L + 8, hi[L] = 16 L + 8 .. 16 L + 16. + [lo[0], hi[0], lo[1], hi[1], lo[2], hi[2], lo[3], hi[3]] + } + } + _ => { + /// Positions per register. + const REGISTER_POSITIONS: usize = 32; + + cfg_select! { + target_feature = "gfni" => { + /// A matrix in every lane, so that the affine product reads it from memory. + type Matrix = __m256i; + + fn matrix(m: u64) -> Matrix { + // SAFETY: AVX is enabled at compile time (module cfg). + unsafe { _mm256_set1_epi64x(m as i64) } + } + + /// An input register as [`product`] reads it. + type Input = __m256i; + + #[inline(always)] + fn input(x: __m256i) -> Input { + x + } + + #[inline(always)] + fn product(x: Input, m: &Matrix) -> __m256i { + // SAFETY: GFNI and AVX are enabled at compile time (module cfg); registers only. + unsafe { _mm256_gf2p8affine_epi64_epi8::<0>(x, *m) } + } + } + _ => { + /// A matrix as two `vpshufb` tables: its images of the low nibbles, then of the high ones. + type Matrix = [u8; 32]; + + fn matrix(m: u64) -> Matrix { + // Output bit `k` is the parity of the input against matrix byte `7 - k`. + let image = |x: u8| (0..8).fold(0u8, |r, k| r | (((m >> (8 * (7 - k))) as u8 & x).count_ones() as u8 & 1) << k); + std::array::from_fn(|i| image(((i % 16) as u8) << (4 * (i / 16)))) + } + + /// An input register as [`product`] reads it: its low nibbles, then its high ones. + type Input = [__m256i; 2]; + + #[inline(always)] + fn input(x: __m256i) -> Input { + // SAFETY: AVX2 is enabled at compile time (module cfg); registers only. + unsafe { + let nibble = _mm256_set1_epi8(0x0F); + [_mm256_and_si256(x, nibble), _mm256_and_si256(_mm256_srli_epi16::<4>(x), nibble)] + } + } + + #[inline(always)] + fn product(x: Input, m: &Matrix) -> __m256i { + // SAFETY: AVX2 is enabled at compile time (module cfg), and each load reads 16 of the + // matrix's bytes. + unsafe { + let lo = _mm256_broadcastsi128_si256(_mm_loadu_si128(m.as_ptr().cast())); + let hi = _mm256_broadcastsi128_si256(_mm_loadu_si128(m.as_ptr().add(16).cast())); + _mm256_xor_si256(_mm256_shuffle_epi8(lo, x[0]), _mm256_shuffle_epi8(hi, x[1])) + } + } + } + } + + impl RowMaps { + /// The rows of a block's first `halves` groups: row `h`, word `k` of position `j` at + /// `rows[h * 2 BITS + k * BITS + j]`. + pub fn rows(&self, block: &[u64], rows: &mut [u64], halves: usize) { + assert!(block.len() >= BITS * BLOCK_WORDS && halves <= 2 && rows.len() >= halves * 2 * BITS); + // SAFETY: AVX2 is enabled at compile time (module cfg); every load reads + // positions inside `block` and every store writes words inside `rows` (asserted). + unsafe { + let mut y = [_mm256_setzero_si256(); GROUP_BYTES]; + for g in 0..BITS / REGISTER_POSITIONS { + for h in 0..halves { + let at = BLOCK_WORDS * REGISTER_POSITIONS * g + GROUP_BYTES / 8 * h; + transpose_group_bytes(block.as_ptr().add(at), &mut y); + for k in 0..2 { + let words = byte_planes_to_words(&self.accumulate(&y, k)); + let dst = rows.as_mut_ptr().add(h * 2 * BITS + k * BITS + REGISTER_POSITIONS * g); + for (r, w) in words.iter().enumerate() { + _mm256_storeu_si256(dst.add(4 * r).cast(), *w); + } + } + } + } + } + } + + /// Output bytes `8 k .. 8 k + 8` of the 32 positions, word `k` of their values: the sums over + /// input bytes `t` of `matrices[t][o]` applied to `y[t]`. Eight outputs at a time keep the sums + /// in registers. + #[inline] + #[target_feature(enable = "avx2")] + fn accumulate(&self, y: &[__m256i; GROUP_BYTES], k: usize) -> [__m256i; 8] { + let mut acc = [_mm256_setzero_si256(); 8]; + for (&x, m) in y.iter().zip(&self.matrices) { + let x = input(x); + for (a, m) in acc.iter_mut().zip(&m[8 * k..][..8]) { + *a = _mm256_xor_si256(*a, product(x, m)); + } + } + acc + } + } + + /// The 32 bytes of a group of 32 consecutive positions (eight words apart, from `p`), transposed: + /// `y[t]` byte `n` is byte `t` of the group of position `n`. + /// + /// # Safety + /// `p` is valid for reading the 32 positions' groups; AVX2. + #[inline] + #[target_feature(enable = "avx2")] + unsafe fn transpose_group_bytes(p: *const u64, y: &mut [__m256i; GROUP_BYTES]) { + for (l, y) in y.as_chunks_mut::<16>().0.iter_mut().enumerate() { + // Register `n`: bytes `16 l ..` of positions `n` and `16 + n`. + *y = std::array::from_fn(|n| { + // SAFETY: positions `n` and `16 + n` lie inside the caller's 32. + unsafe { + _mm256_loadu2_m128i( + p.add(BLOCK_WORDS * (16 + n) + 2 * l).cast(), + p.add(BLOCK_WORDS * n + 2 * l).cast(), + ) + } + }); + // y[k] lane i byte n = byte 16 l + k of position 16 i + n. + primitives::bits::transpose_bytes16x16_per_lane256(y); + } + } + + /// Eight byte planes of 32 positions (plane `b` byte `n` = byte `b` of position `n`) as the + /// positions' words: register `r` holds positions `4 r .. 4 r + 4`. + #[inline] + #[target_feature(enable = "avx2")] + fn byte_planes_to_words(a: &[__m256i; 8]) -> [__m256i; 8] { + let w = interleave_planes!( + a, + _mm256_unpacklo_epi8, + _mm256_unpackhi_epi8, + _mm256_unpacklo_epi16, + _mm256_unpackhi_epi16, + _mm256_unpacklo_epi32, + _mm256_unpackhi_epi32 + ); + std::array::from_fn(|r| { + let (x, y) = (w[2 * (r % 4)], w[2 * (r % 4) + 1]); + if r < 4 { + _mm256_permute2x128_si256::<0x20>(x, y) + } else { + _mm256_permute2x128_si256::<0x31>(x, y) + } + }) + } + } +} + +#[cfg(test)] +mod tests { + use primitives::test_rng::Rng; + + use super::*; + use crate::basis::carry; + use crate::kernels::{embed, fold_groups}; + use crate::power::Element; + use crate::witness::words; + + /// The rows the per-position path builds: embeddings weighted by carry-less products, reduced. + #[test] + fn rows_match_the_carry_less_path() { + let mut rng = Rng::new(0x7AB1E); + let field = PowerField::new(rng.ext()).expect("a generator"); + // The fold weights: field elements in the power basis, by word. + let elems: Vec = (0..GROUP).map(|_| field.to_poly(rng.ext())).collect(); + let fold: [[u64; GROUP]; 2] = std::array::from_fn(|k| std::array::from_fn(|q| elems[q][k])); + let block: Vec = (0..BITS * BLOCK_WORDS).map(|_| rng.next_u64()).collect(); + let mut want = vec![0u64; 2 * 2 * BITS]; + let mut wide = vec![0u64; 2 * 3 * BITS]; + for j in 0..BITS { + let sums = fold_groups(&embed(words(&block, j)), &fold); + for (h, sum) in sums.iter().enumerate() { + let sum: [u64; 3] = carry(sum); + for (k, &w) in sum.iter().enumerate() { + wide[h * 3 * BITS + k * BITS + j] = w; + } + } + } + for h in 0..2 { + field.reduce_row( + &mut want[h * 2 * BITS..][..2 * BITS], + &wide[h * 3 * BITS..][..3 * BITS], + BITS, + ); + } + for halves in [2, 1] { + let mut got = vec![0u64; 2 * 2 * BITS]; + RowMaps::new(&field, &fold).rows(&block, &mut got, halves); + assert_eq!(got[..halves * 2 * BITS], want[..halves * 2 * BITS], "{halves} halves"); + } + } +} diff --git a/crates/experimental_prover/lean_vm_core/src/gkr.rs b/crates/experimental_prover/lean_vm_core/src/gkr.rs index a5922c160c..0aa93bc5c4 100644 --- a/crates/experimental_prover/lean_vm_core/src/gkr.rs +++ b/crates/experimental_prover/lean_vm_core/src/gkr.rs @@ -6,7 +6,7 @@ //! Leaves and every layer are `E`-valued (the bus fingerprints mix `K`-columns //! into `E` upstream, [`crate::leaf`]). //! -//! On AVX-512 the large rounds run eight pairs of rows at a time in transposed form +//! With VPCLMULQDQ the large rounds run eight pairs of rows at a time in transposed form //! (`vertical`), and skip the runs of rows known to be the identity. use std::ops::Range; @@ -41,9 +41,9 @@ fn sum_tasks(tasks: usize, parallel: bool, f: impl Fn(usize) -> [F128; 4] + Sync /// `out[i] = ∏ current[4i..4i+4]`, for `out.len() == current.len() / 4`. pub(crate) fn products_into(current: &[F128], out: &mut [F128]) { debug_assert_eq!(out.len(), current.len() / 4); - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] let done = vertical::products_into(current, out); - #[cfg(not(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq")))] + #[cfg(not(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq")))] let done = 0; for (row, slot) in out.iter_mut().enumerate().skip(done) { let [left, right] = mul2( @@ -161,8 +161,8 @@ struct SplitEq { hi: Vec, lo_bits: usize, /// `lo` eight entries at a time, for the vertical kernels. - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] - lo8: Vec, + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + lo8: Vec, } impl SplitEq { @@ -171,13 +171,8 @@ impl SplitEq { let (low, high) = vars.split_at(lo_bits); let lo = eq_table(low); Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] - lo8: lo - .as_chunks::<8>() - .0 - .iter() - .map(primitives::field::x8::F128x8::load) - .collect(), + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + lo8: lo.as_chunks::<8>().0.iter().map(vertical::F128x8::from_array).collect(), lo, hi: eq_table(high), lo_bits, @@ -264,7 +259,7 @@ fn runs<'a>(range: Range, ones: &'a [Range]) -> impl Iterator, - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] spare: Vec, /// Runs of rows known to be the identity, sorted: the kernels skip them, and /// the vertical ones leave their groups unwritten. @@ -295,9 +290,9 @@ impl QuaternaryLayerState { Self { values, logical_rows: width, - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] groups: Vec::new(), - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] spare: Vec::new(), ones: Vec::new(), } @@ -305,7 +300,7 @@ impl QuaternaryLayerState { /// `(q(0)+q(1), [X²]q, [X³]q, [X⁴]q)`. fn round_message(&self, eq: &SplitEq) -> [F128; 4] { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] if self.logical_rows >= VERTICAL_ROWS { return vertical::message(&self.values, &self.ones, eq); } @@ -313,13 +308,13 @@ impl QuaternaryLayerState { } fn fold(&mut self, challenge: F128) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] self.leave_groups(); self.fold_scalar(challenge); } /// The vertical kernels' view of the rows, and the buffer the fold writes. - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] fn split(&mut self) -> (vertical::Input<'_>, &mut Vec) { let data = if self.groups.is_empty() { vertical::Data::Rows(&self.values) @@ -330,7 +325,7 @@ impl QuaternaryLayerState { } /// Move the rows back into `values` for the scalar kernels. - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] fn leave_groups(&mut self) { if !self.groups.is_empty() { self.values = vertical::to_rows(&self.groups, &self.ones); @@ -515,7 +510,7 @@ fn fold_and_message( challenge: F128, eq: &SplitEq, ) -> [[F128; 4]; N_BUS_TREES] { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] { if trees[0].logical_rows >= VERTICAL_ROWS { let [a, b, c] = trees.each_mut().map(QuaternaryLayerState::split); @@ -535,15 +530,26 @@ fn fold_and_message( trees.each_mut().map(|tree| tree.fold_and_message_scalar(challenge, eq)) } -/// The AVX-512 kernels: eight pairs of rows at a time, transposed so that every -/// product is lane-wise ([`primitives::field::x8`]). After its first fold a layer -/// keeps its rows transposed, in [`Group`]s, until it is small enough for the scalar -/// kernels. -#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] +/// The VPCLMULQDQ kernels: eight pairs of rows at a time, transposed so that every +/// product is lane-wise (`primitives::field::x8` on AVX-512, `f128x8`'s two halves +/// of `x4` lanes on AVX2). After its first fold a layer keeps its rows transposed, +/// in [`Group`]s, until it is small enough for the scalar kernels. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] mod vertical { use super::{N_BUS_TREES, SplitEq, add4, coarsen, sum_tasks}; use primitives::field::F128; - use primitives::field::x8::{F128x8, Sum8}; + // A message runs on `Lane`s: all eight lanes on AVX-512, and on AVX2 a four-lane half at a time, which + // keeps its live set within the sixteen YMM registers. + cfg_select! { + target_feature = "avx512f" => { + use primitives::field::x8::{F128x8 as Lane, Sum8 as Sum}; + pub(super) use primitives::field::x8::F128x8; + } + _ => { + use primitives::field::x4::{F128x4 as Lane, Sum4 as Sum}; + pub(super) use primitives::field::f128x8::F128x8; + } + } use primitives::stream::Stream; use std::ops::Range; @@ -573,10 +579,10 @@ mod vertical { } } - /// Add eight pairs' `eq`-weighted summands. The weight scales child 0's line, and - /// with it every product, so it costs two products rather than one per coefficient. + /// Add the `eq`-weighted summands of a group's pairs, in `Lane`s. The weight scales child 0's line, + /// and with it every product, so it costs two products rather than one per coefficient. #[inline(always)] - fn add_message(g: &Group, eq: F128x8, acc: &mut [Sum8; 4]) { + fn add_message(g: [Lane; 8], eq: Lane, acc: &mut [Sum; 4]) { let a0 = eq * g[0]; let d0 = eq * (g[0] + g[4]); let left0 = a0 * g[1]; @@ -601,7 +607,7 @@ mod vertical { struct Message<'a> { eq: &'a SplitEq, block: usize, - sums: [Sum8; 4], + sums: [Sum; 4], total: [F128; 4], } @@ -610,7 +616,7 @@ mod vertical { Self { eq, block: first >> (eq.lo_bits - 3), - sums: [Sum8::ZERO; 4], + sums: [Sum::ZERO; 4], total: [F128::ZERO; 4], } } @@ -622,11 +628,20 @@ mod vertical { self.flush(); self.block = g >> shift; } - add_message(group, self.eq.lo8[g & ((1 << shift) - 1)], &mut self.sums); + let eq = self.eq.lo8[g & ((1 << shift) - 1)]; + cfg_select! { + target_feature = "avx512f" => add_message(*group, eq, &mut self.sums), + _ => { + // One half after the other, unrolled: a loop over them runs slower. + let (e, h) = (eq.halves(), group.map(F128x8::halves)); + add_message(h.map(|lanes| lanes[0]), e[0], &mut self.sums); + add_message(h.map(|lanes| lanes[1]), e[1], &mut self.sums); + } + } } fn flush(&mut self) { - let sums = std::mem::replace(&mut self.sums, [Sum8::ZERO; 4]).map(Sum8::reduce); + let sums = std::mem::replace(&mut self.sums, [Sum::ZERO; 4]).map(Sum::reduce); let scaled = primitives::field::mul4([self.eq.hi[self.block]; 4], sums); self.total = add4(self.total, scaled); } @@ -658,7 +673,7 @@ mod vertical { let fold = |g: &Group, c: usize| g[c] + r * (g[c] + g[4 + c]); let mut out = [F128x8::splat(F128::ZERO); 8]; for c in 0..4 { - (out[c], out[4 + c]) = F128x8::deinterleave(fold(a, c), fold(b, c)); + (out[c], out[4 + c]) = F128x8::even_odd(fold(a, c), fold(b, c)); } out } @@ -775,10 +790,7 @@ mod vertical { let (chunks, _) = out.as_chunks_mut::<16>(); for (slot, run) in chunks.iter_mut().zip(current.as_chunks::<64>().0) { let g = F128x8::load_runs(run); - let (lo, hi) = F128x8::interleave(product(g[0], g[1], g[2], g[3]), product(g[4], g[5], g[6], g[7])); - let (first, second) = slot.split_at_mut(8); - lo.store(first.try_into().unwrap()); - hi.store(second.try_into().unwrap()); + F128x8::store_interleaved(product(g[0], g[1], g[2], g[3]), product(g[4], g[5], g[6], g[7]), slot); } 16 * chunks.len() } diff --git a/crates/experimental_prover/primitives/src/bits.rs b/crates/experimental_prover/primitives/src/bits.rs index 1a3932bd03..08616e4c30 100644 --- a/crates/experimental_prover/primitives/src/bits.rs +++ b/crates/experimental_prover/primitives/src/bits.rs @@ -10,25 +10,31 @@ /// So byte column `b` of the eight input rows becomes output row `b`, bit-transposed. /// /// - AVX-512 VBMI with GFNI: one byte permute and one affine map. +/// - AVX2 with GFNI: a byte gather in unpacks and shuffles, then one affine map. +/// - AVX2: the same gather, then three masked-swap rounds. /// - aarch64: a table lookup, then three masked-swap rounds. /// - Elsewhere: the same rounds on a gathered word per column. #[inline] pub fn bit_transpose_64bytes(input: &[u8; 64], output: &mut [u8; 64]) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"))] - // SAFETY: the features are enabled at compile time. - unsafe { - bit_transpose_64bytes_gfni(input, output) - } - #[cfg(target_arch = "aarch64")] - // SAFETY: aarch64 always has NEON. - unsafe { - bit_transpose_64bytes_neon(input, output) + cfg_select! { + all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni") => { + // SAFETY: the features are enabled at compile time. + unsafe { bit_transpose_64bytes_gfni(input, output) } + } + all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni") => { + // SAFETY: the features are enabled at compile time. + unsafe { bit_transpose_64bytes_gfni_avx2(input, output) } + } + all(target_arch = "x86_64", target_feature = "avx2") => { + // SAFETY: the feature is enabled at compile time. + unsafe { bit_transpose_64bytes_avx2(input, output) } + } + target_arch = "aarch64" => { + // SAFETY: aarch64 always has NEON. + unsafe { bit_transpose_64bytes_neon(input, output) } + } + _ => bit_transpose_64bytes_portable(input, output), } - #[cfg(not(any( - all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"), - target_arch = "aarch64" - )))] - bit_transpose_64bytes_portable(input, output); } /// Transpose the 8x8 bit matrix whose byte `r` is row `r` (Hacker's Delight, section 7-3). @@ -54,6 +60,7 @@ fn transpose_8x8_bits(mut x: u64) -> u64 { #[cfg_attr( any( all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"), + all(target_arch = "x86_64", target_feature = "avx2"), target_arch = "aarch64" ), allow(dead_code) @@ -111,6 +118,80 @@ pub fn bit_transpose_64bytes_m512(input: core::arch::x86_64::__m512i) -> core::a } } +/// Gather each byte column of the eight 8-byte rows into a word, in the row order `order` picks. +/// +/// After two unpack rounds, lane `l` of register `w` holds columns `4w + 2l` and `4w + 2l + 1`, their bytes at: +/// +/// ```text +/// rows 0, 1, 4, 5 bytes 0..4 (first column), 4..8 (second) +/// rows 2, 3, 6, 7 bytes 8..12 (first column), 12..16 (second) +/// ``` +/// +/// So `order[8c + k]` is the byte that becomes byte `k` of the lane's column `c`. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +#[inline] +#[target_feature(enable = "avx2")] +fn gather_columns_avx2(input: &[u8; 64], order: &[u8; 16]) -> [core::arch::x86_64::__m256i; 2] { + use core::arch::x86_64::*; + + // SAFETY: the loads read the two halves of the 64-byte input and the 16-byte order. + let (v0, v1, order) = unsafe { + ( + _mm256_loadu_si256(input.as_ptr().cast()), + _mm256_loadu_si256(input.as_ptr().add(32).cast()), + _mm256_broadcastsi128_si256(_mm_loadu_si128(order.as_ptr().cast())), + ) + }; + // Lane 0: rows 0 and 4, then rows 1 and 5, byte-interleaved; lane 1: rows 2 and 6, then 3 and 7. + let (t0, t1) = (_mm256_unpacklo_epi8(v0, v1), _mm256_unpackhi_epi8(v0, v1)); + // Dword `c` of lane 0 is column `c` of rows 0, 1, 4, 5; of lane 1, of rows 2, 3, 6, 7. + let (u0, u1) = (_mm256_unpacklo_epi8(t0, t1), _mm256_unpackhi_epi8(t0, t1)); + // Bring both halves of two columns into one lane, then order their bytes. + [u0, u1].map(|u| _mm256_shuffle_epi8(_mm256_permute4x64_epi64::<0b11_01_10_00>(u), order)) +} + +/// [`gather_columns_avx2`] with the rows reversed, then the AVX-512 arm's affine map. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] +#[cfg_attr(target_feature = "avx512vbmi", allow(dead_code))] +#[inline] +#[target_feature(enable = "avx2,gfni")] +fn bit_transpose_64bytes_gfni_avx2(input: &[u8; 64], output: &mut [u8; 64]) { + use core::arch::x86_64::*; + + const REVERSED: [u8; 16] = [11, 10, 3, 2, 9, 8, 1, 0, 15, 14, 7, 6, 13, 12, 5, 4]; + let unit = _mm256_set1_epi64x(0x8040_2010_0804_0201_u64 as i64); + for (i, columns) in gather_columns_avx2(input, &REVERSED).into_iter().enumerate() { + let rows = _mm256_gf2p8affine_epi64_epi8::<0>(unit, columns); + // SAFETY: the two stores fill the 64-byte output. + unsafe { _mm256_storeu_si256(output.as_mut_ptr().add(32 * i).cast(), rows) }; + } +} + +/// [`gather_columns_avx2`] in row order, then the three masked-swap rounds of [`transpose_8x8_bits`] on four words. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +#[cfg_attr(target_feature = "gfni", allow(dead_code))] +#[inline] +#[target_feature(enable = "avx2")] +fn bit_transpose_64bytes_avx2(input: &[u8; 64], output: &mut [u8; 64]) { + use core::arch::x86_64::*; + + const NATURAL: [u8; 16] = [0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15]; + #[inline] + #[target_feature(enable = "avx2")] + fn swap(x: __m256i, mask: u64) -> __m256i { + let mask = _mm256_set1_epi64x(mask as i64); + let t = _mm256_and_si256(_mm256_xor_si256(x, _mm256_srli_epi64::(x)), mask); + _mm256_xor_si256(x, _mm256_xor_si256(t, _mm256_slli_epi64::(t))) + } + for (i, x) in gather_columns_avx2(input, &NATURAL).into_iter().enumerate() { + let x = swap::<7>(x, 0x00AA_00AA_00AA_00AA); + let x = swap::<14>(x, 0x0000_CCCC_0000_CCCC); + let x = swap::<28>(x, 0x0000_0000_F0F0_F0F0); + // SAFETY: the two stores fill the 64-byte output. + unsafe { _mm256_storeu_si256(output.as_mut_ptr().add(32 * i).cast(), x) }; + } +} + /// `vqtbl4q_u8` gathers each byte column into a word, then three masked-swap rounds transpose its bits. #[cfg(target_arch = "aarch64")] #[inline(always)] @@ -291,6 +372,18 @@ mod tests { unsafe { bit_transpose_64bytes_gfni(&input, &mut got) }; assert_eq!(got, want, "gfni"); } + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] + { + // SAFETY: compiled only with the features enabled. + unsafe { bit_transpose_64bytes_gfni_avx2(&input, &mut got) }; + assert_eq!(got, want, "gfni_avx2"); + } + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] + { + // SAFETY: compiled only with the feature enabled. + unsafe { bit_transpose_64bytes_avx2(&input, &mut got) }; + assert_eq!(got, want, "avx2"); + } #[cfg(target_arch = "aarch64")] { // SAFETY: aarch64 always has NEON. diff --git a/crates/experimental_prover/primitives/src/field/f128x8.rs b/crates/experimental_prover/primitives/src/field/f128x8.rs index f94bf02909..d85433c4da 100644 --- a/crates/experimental_prover/primitives/src/field/f128x8.rs +++ b/crates/experimental_prover/primitives/src/field/f128x8.rs @@ -409,6 +409,47 @@ mod avx2 { let ((ae, ao), (be, bo)) = (split(a), split(b)); (Self([ae, be]), Self([ao, bo])) } + + /// Eight runs of eight consecutive elements, transposed: `out[j]` lane `p` is `src[8p + j]`. + #[inline(always)] + pub fn load_runs(src: &[F128; 64]) -> [Self; 8] { + let at = |p: usize, j: usize| src[8 * p + j..].as_ptr().cast::<__m128i>(); + std::array::from_fn(|j| { + Self(std::array::from_fn(|h| { + // Runs `4h + i` and `4h + i + 2` in one register; the unpacks then put lane `i` in qword `i`. + // SAFETY: the module's cfg enables avx2; each address is one element, 16 readable bytes. + let pair = |i: usize| unsafe { _mm256_loadu2_m128i(at(4 * h + i + 2, j), at(4 * h + i, j)) }; + let (a, b) = (pair(0), pair(1)); + // SAFETY: the module's cfg enables avx2. + F128x4(unsafe { [_mm256_unpacklo_epi64(a, b), _mm256_unpackhi_epi64(a, b)] }) + })) + }) + } + + /// Stores the lanes `a0 b0 a1 b1 … a7 b7`. + #[inline(always)] + pub fn store_interleaved(a: Self, b: Self, dst: &mut [F128; 16]) { + let q = dst.as_mut_ptr().cast::<__m256i>(); + for h in 0..2 { + let ([a0, a1], [b0, b1]) = (a.0[h].0, b.0[h].0); + // SAFETY: the module's cfg enables avx2; `dst` is eight writable 32-byte pairs of elements. + unsafe { + // Elements `[x0, x2]` and `[x1, x3]` of each. + let (al, ah) = (_mm256_unpacklo_epi64(a0, a1), _mm256_unpackhi_epi64(a0, a1)); + let (bl, bh) = (_mm256_unpacklo_epi64(b0, b1), _mm256_unpackhi_epi64(b0, b1)); + _mm256_storeu_si256(q.add(4 * h), _mm256_permute2x128_si256::<0x20>(al, bl)); + _mm256_storeu_si256(q.add(4 * h + 1), _mm256_permute2x128_si256::<0x20>(ah, bh)); + _mm256_storeu_si256(q.add(4 * h + 2), _mm256_permute2x128_si256::<0x31>(al, bl)); + _mm256_storeu_si256(q.add(4 * h + 3), _mm256_permute2x128_si256::<0x31>(ah, bh)); + } + } + } + + /// Lanes `0..4` and `4..8`. + #[inline(always)] + pub fn halves(self) -> [F128x4; 2] { + self.0 + } } impl core::ops::Add for F128x8 { @@ -841,6 +882,23 @@ mod tests { odd.to_array(), core::array::from_fn(|j| if j < 4 { x0[2 * j + 1] } else { x1[2 * (j - 4) + 1] }) ); + #[cfg(all( + target_arch = "x86_64", + target_feature = "vpclmulqdq", + target_feature = "avx2", + not(target_feature = "avx512f") + ))] + { + let mut stored = [F128::ZERO; 16]; + F128x8::store_interleaved(even, odd, &mut stored); + assert_eq!(stored, core::array::from_fn(|i| if i < 8 { x0[i] } else { x1[i - 8] })); + let [lo, hi] = xa.halves(); + assert_eq!([lo.to_array(), hi.to_array()].concat(), x0); + let runs: [F128; 64] = core::array::from_fn(|_| rng.ext()); + for (j, run) in F128x8::load_runs(&runs).iter().enumerate() { + assert_eq!(run.to_array(), core::array::from_fn(|p| runs[8 * p + j])); + } + } let (ka, kb) = (F64x8::from_array(&k0), F64x8::from_array(&k1)); assert_eq!( diff --git a/crates/experimental_prover/primitives/src/field/x4.rs b/crates/experimental_prover/primitives/src/field/x4.rs index 7a63f4b7ff..bc615e7818 100644 --- a/crates/experimental_prover/primitives/src/field/x4.rs +++ b/crates/experimental_prover/primitives/src/field/x4.rs @@ -3,7 +3,7 @@ //! and the halves of the eight-lane `f128x8` types. use core::arch::x86_64::*; -use core::ops::{Add, BitXorAssign, Mul}; +use core::ops::{Add, BitXor, BitXorAssign, Mul}; use super::gf2_64::F64; use super::gf2_64x2::x86_64::{fold, reduce_halves256 as reduce_halves, times_x256}; @@ -26,6 +26,10 @@ pub struct Mixed4 { odd: [__m256i; 2], } +/// The sum over lanes of unreduced products, y-folded: per coefficient, two 128-bit partials. +#[derive(Clone, Copy, Debug)] +pub struct Sum4([__m256i; 2]); + #[inline(always)] fn xor(a: __m256i, b: __m256i) -> __m256i { // SAFETY: the module's cfg enables avx2. @@ -39,15 +43,28 @@ fn clmul(a: __m256i, b: __m256i) -> __m256i { unsafe { _mm256_clmulepi64_epi128::(a, b) } } -/// The y-folded Karatsuba product of the even or the odd lanes. +/// The y-folded Karatsuba product, its three coefficient products taken by `product`. #[inline(always)] -fn karatsuba(a: [__m256i; 2], b: [__m256i; 2]) -> [__m256i; 2] { - let (m0, m1) = (clmul::(a[0], b[0]), clmul::(a[1], b[1])); - let m01 = clmul::(xor(a[0], a[1]), xor(b[0], b[1])); +fn karatsuba(a: [__m256i; 2], b: [__m256i; 2], product: impl Fn(__m256i, __m256i) -> __m256i) -> [__m256i; 2] { + let (m0, m1) = (product(a[0], b[0]), product(a[1], b[1])); + let m01 = product(xor(a[0], a[1]), xor(b[0], b[1])); // SAFETY: the module's cfg enables avx2. fold(m0, m1, m01, xor, |p| unsafe { times_x256(p) }) } +/// The sum of the lanes of a coefficient pair, reduced once. +#[inline(always)] +fn reduce_lanes(c: [__m256i; 2]) -> F128 { + // SAFETY: the module's cfg enables avx2, and a vector and two qwords hold the same bits. + let coeffs = c.map(|v| unsafe { + std::mem::transmute::<__m128i, [u64; 2]>(_mm_xor_si128( + _mm256_castsi256_si128(v), + _mm256_extracti128_si256::<1>(v), + )) + }); + F128Unreduced::from_coeffs(coeffs).reduce() +} + impl F64x4 { #[inline(always)] pub fn new(k: [F64; 4]) -> Self { @@ -129,10 +146,54 @@ impl F128x4 { #[inline(always)] pub fn mul_unreduced(self, rhs: Self) -> Mixed4 { Mixed4 { - even: karatsuba::<0x00>(self.0, rhs.0), - odd: karatsuba::<0x11>(self.0, rhs.0), + even: karatsuba(self.0, rhs.0, clmul::<0x00>), + odd: karatsuba(self.0, rhs.0, clmul::<0x11>), } } + + /// The lane-wise products, summed over the lanes and left unreduced. Karatsuba's products are summed + /// before their one fold. + #[inline(always)] + pub fn mul_sum(self, rhs: Self) -> Sum4 { + Sum4(karatsuba(self.0, rhs.0, |x, y| { + xor(clmul::<0x00>(x, y), clmul::<0x11>(x, y)) + })) + } +} + +impl Mul for F128x4 { + type Output = Self; + #[inline(always)] + fn mul(self, rhs: Self) -> Self { + self.mul_unreduced(rhs).reduce() + } +} + +impl Sum4 { + // SAFETY: an all-zero bit pattern is a valid `__m256i`. + pub const ZERO: Self = Self(unsafe { std::mem::zeroed() }); + + /// The lane sum, reduced. + #[inline] + pub fn reduce(self) -> F128 { + reduce_lanes(self.0) + } +} + +impl BitXorAssign for Sum4 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + self.0 = std::array::from_fn(|k| xor(self.0[k], rhs.0[k])); + } +} + +impl BitXor for Sum4 { + type Output = Self; + #[inline(always)] + fn bitxor(mut self, rhs: Self) -> Self { + self ^= rhs; + self + } } /// The embedding of the base field. @@ -171,17 +232,7 @@ impl Mixed4 { /// The sum of the four lanes, reduced once. #[inline] pub fn sum(self) -> F128 { - let coeffs = std::array::from_fn(|i| { - let v = xor(self.even[i], self.odd[i]); - // SAFETY: the module's cfg enables avx2, and a vector and two qwords hold the same bits. - unsafe { - std::mem::transmute::<__m128i, [u64; 2]>(_mm_xor_si128( - _mm256_castsi256_si128(v), - _mm256_extracti128_si256::<1>(v), - )) - } - }); - F128Unreduced::from_coeffs(coeffs).reduce() + reduce_lanes(std::array::from_fn(|i| xor(self.even[i], self.odd[i]))) } #[inline(always)] @@ -244,6 +295,9 @@ mod tests { acc ^= vb.mul_unreduced(va); acc ^= va.mul_unreduced(va); assert_eq!(acc.sum(), (0..4).map(|i| a[i] * a[i]).sum()); + assert_eq!((va * vb).to_array(), std::array::from_fn(|i| a[i] * b[i])); + assert_eq!(va.mul_sum(vb).reduce(), (0..4).map(|i| a[i] * b[i]).sum()); + assert_eq!((va.mul_sum(vb) ^ vb.mul_sum(va)).reduce(), F128::ZERO); let k: [F64; 4] = std::array::from_fn(|_| F64(rng.next_u64())); assert_eq!(F128x4::from(F64x4::new(k)).to_array(), k.map(F128::from)); } diff --git a/crates/experimental_prover/primitives/src/field/x8.rs b/crates/experimental_prover/primitives/src/field/x8.rs index 92108525de..c4b6f0ed39 100644 --- a/crates/experimental_prover/primitives/src/field/x8.rs +++ b/crates/experimental_prover/primitives/src/field/x8.rs @@ -76,6 +76,12 @@ impl F128x8 { unsafe { Self([_mm512_set1_epi64(e.c0 as i64), _mm512_set1_epi64(e.c1 as i64)]) } } + /// Eight consecutive elements, under `f128x8`'s name. + #[inline(always)] + pub fn from_array(src: &[F128; 8]) -> Self { + Self::load(src) + } + /// Eight consecutive elements. #[inline(always)] pub fn load(src: &[F128; 8]) -> Self { @@ -117,20 +123,19 @@ impl F128x8 { /// The even lanes of `a` then of `b`, and the odd lanes of `a` then of `b`. #[inline(always)] - pub fn deinterleave(a: Self, b: Self) -> (Self, Self) { + pub fn even_odd(a: Self, b: Self) -> (Self, Self) { ( Self(std::array::from_fn(|k| select(a.0[k], EVEN, b.0[k]))), Self(std::array::from_fn(|k| select(a.0[k], ODD, b.0[k]))), ) } - /// The inverse of [`deinterleave`](Self::deinterleave): lanes `a0 b0 a1 b1 …` over two registers. + /// Stores the lanes `a0 b0 a1 b1 … a7 b7`, the inverse of [`even_odd`](Self::even_odd). #[inline(always)] - pub fn interleave(a: Self, b: Self) -> (Self, Self) { - ( - Self(std::array::from_fn(|k| select(a.0[k], LO, b.0[k]))), - Self(std::array::from_fn(|k| select(a.0[k], HI, b.0[k]))), - ) + pub fn store_interleaved(a: Self, b: Self, dst: &mut [F128; 16]) { + let (halves, _) = dst.as_chunks_mut::<8>(); + Self(std::array::from_fn(|k| select(a.0[k], LO, b.0[k]))).store(&mut halves[0]); + Self(std::array::from_fn(|k| select(a.0[k], HI, b.0[k]))).store(&mut halves[1]); } /// The even-lane and odd-lane unreduced products, y-folded. @@ -326,12 +331,13 @@ mod tests { assert_eq!((va.mul_sum(vb) ^ vb.mul_sum(va)).reduce(), F128::ZERO); assert_eq!(F128x8::splat(a[0]).to_array(), [a[0]; 8]); - let (even, odd) = F128x8::deinterleave(va, vb); + let (even, odd) = F128x8::even_odd(va, vb); let both: Vec = a.iter().chain(&b).copied().collect(); assert_eq!(even.to_array(), std::array::from_fn(|i| both[2 * i])); assert_eq!(odd.to_array(), std::array::from_fn(|i| both[2 * i + 1])); - let (lo, hi) = F128x8::interleave(even, odd); - assert_eq!((lo.to_array(), hi.to_array()), (a, b)); + let mut stored = [F128::ZERO; 16]; + F128x8::store_interleaved(even, odd, &mut stored); + assert_eq!(stored.to_vec(), both); let c = rng.ext(); let k: [F64; 8] = std::array::from_fn(|_| F64(rng.ext().c0)); diff --git a/crates/fiat_shamir_prover/src/lib.rs b/crates/fiat_shamir_prover/src/lib.rs index d1d67edcaf..9220f2b7e0 100644 --- a/crates/fiat_shamir_prover/src/lib.rs +++ b/crates/fiat_shamir_prover/src/lib.rs @@ -174,14 +174,13 @@ impl FiatShamirState { // The smallest passing nonce of batch `b`, the nonces `b * BATCH..(b + 1) * BATCH`. const BATCH: u64 = 64; let batch = |b: u64| { - let mut states = [base; BATCH as usize]; - for (nonce, state) in (b * BATCH..).zip(&mut states) { - state[lane] = nonce; - } - keccak256::keccak_f_many(&mut states); - let pass = - |&(nonce, state): &(u64, &[u64; STATE_LANES])| nonce >= from && state[2] & ((1 << bits) - 1) == 0; - (b * BATCH..).zip(&states).find(pass).map(|(nonce, _)| nonce) + keccak256::first_nonce( + &base, + lane, + (b * BATCH).max(from)..(b * BATCH).saturating_add(BATCH), + 2, + (1 << bits) - 1, + ) }; // A short search, a few batches, is not worth a dispatch to the pool. let nonce = if bits < 10 { @@ -276,11 +275,20 @@ mod tests { fn pow_predicate() { let mut sp = State::new(Digest(keccak256::hash(b"t")).words(), pi(1)); sp.observe(f(7)); - for bits in [2, 8, 13, 17] { + for bits in [2, 8, 10, 13, 17] { let mut clone = sp.clone(); let good = clone.grind::(bits).unwrap(); let expected = (0..=good).find(|&n| pow_bits_ok(&sp, n, bits)).unwrap(); assert_eq!(good, expected, "smallest nonce at {bits} bits"); + // From inside a batch, as `grind_sample` resumes: the smallest passing nonce at least `from`. + for from in [good + 1, 63] { + let next = (from..).find(|&n| pow_bits_ok(&sp, n, bits)).unwrap(); + assert_eq!( + sp.clone().grind_from::(bits, from), + Some(next), + "from {from} at {bits} bits" + ); + } // The grind's own permutation draws the next challenge, and the one after is fresh. let mut expected_state = sp.clone(); expected_state.absorb(E::new(good, 0), DS_POW_NONCE); diff --git a/crates/keccak256/src/lanes.rs b/crates/keccak256/src/lanes.rs index 50e5ccee2f..ef28c5f145 100644 --- a/crates/keccak256/src/lanes.rs +++ b/crates/keccak256/src/lanes.rs @@ -3,6 +3,7 @@ //! the target has them), or one at a time elsewhere. Every batched function is byte-identical to its one-input form. use crate::{END_BIT, OUT_LEN, PAD_FIRST, RATE, RATE_LANES, RC, STATE_LANES, hash_from_state, keccak_f}; +use core::ops::Range; /// One state lane across [`Lane::WIDTH`] states. The permutation is written once over this trait. pub(crate) trait Lane: Copy { @@ -136,6 +137,26 @@ pub fn keccak_f_many(states: &mut [[u64; STATE_LANES]]) { rest.iter_mut().for_each(keccak_f); } +/// The first of `nonces` that, written into lane `at` of `template`, leaves lane `out` of the permuted state zero under +/// `mask`, [`LANES`] nonces at a time. +pub fn first_nonce(template: &[u64; STATE_LANES], at: usize, nonces: Range, out: usize, mask: u64) -> Option { + let template: [Backend; STATE_LANES] = core::array::from_fn(|i| Backend::splat(template[i])); + let mut first = nonces.start; + while first < nonces.end { + let mut st = template; + st[at] = Backend::load(&core::array::from_fn::(|l| first.wrapping_add(l as u64))); + permute(&mut st); + let mut words = [0; LANES]; + st[out].store(&mut words); + let n = (nonces.end - first).min(LANES as u64) as usize; + if let Some(l) = words[..n].iter().position(|w| w & mask == 0) { + return Some(first + l as u64); + } + first = first.saturating_add(LANES as u64); + } + None +} + /// Batched Keccak-256 of `data` split into `len`-byte inputs, writing one 32-byte digest per input to `out`. pub fn hash_many(data: &[u8], len: usize, out: &mut [u8]) { hash_many_from_state(data, len, &[0; STATE_LANES], out); diff --git a/crates/keccak256/src/lib.rs b/crates/keccak256/src/lib.rs index a590cf5654..5bd9011f28 100644 --- a/crates/keccak256/src/lib.rs +++ b/crates/keccak256/src/lib.rs @@ -3,7 +3,7 @@ //! tree. //! //! - [`keccak_f`], the Keccak-f\[1600\] permutation, which the VM's `KECCAK` opcode computes and `keccak_gkr` -//! proves, and [`keccak_f_many`], its batched form. +//! proves, and [`keccak_f_many`], its batched form; [`first_nonce`], the provers' proof-of-work search over it. //! - [`hash`] / [`Hasher`], Keccak-256 over bytes, and [`hash_many`], its batched form over equal-length inputs. //! - [`positions`], the permutation one bit position at a time, as `keccak_gkr` proves it. //! @@ -91,7 +91,7 @@ pub const END_BIT: u64 = (PAD_LAST as u64) << 56; mod lanes; pub mod positions; -pub use lanes::{LANES, digest_halves_from, hash_many, hash_many_from_state, keccak_f_many}; +pub use lanes::{LANES, digest_halves_from, first_nonce, hash_many, hash_many_from_state, keccak_f_many}; #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] mod single_avx512; @@ -367,4 +367,38 @@ mod tests { assert_eq!(got, want); } } + + /// The nonce search against permuting each candidate, at every nonce lane a grind uses: a first hit at every + /// offset of a batch, ranges ending inside one, and a range up to `u64::MAX`, not of whole batches, with no hit. + #[test] + fn first_nonce_matches_keccak_f() { + let template: [u64; STATE_LANES] = std::array::from_fn(|i| (i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15)); + let lanes = LANES as u64; + for at in [0, 4, 8, 12] { + let pass = |nonce: u64, mask: u64| { + let mut state = template; + state[at] = nonce; + keccak_f(&mut state); + state[2] & mask == 0 + }; + let mut offsets = [false; LANES]; + for start in 0..256 { + for end in start..start + 3 * lanes { + let want = (start..end).find(|&nonce| pass(nonce, 0xf)); + assert_eq!( + first_nonce(&template, at, start..end, 2, 0xf), + want, + "{at} {start}..{end}" + ); + if let Some(hit) = want { + offsets[((hit - start) % lanes) as usize] = true; + } + } + } + assert!(offsets.iter().all(|&hit| hit), "a first hit at every offset of a batch"); + let top = u64::MAX - 3 * lanes - 1..u64::MAX; + assert_eq!(top.clone().find(|&nonce| pass(nonce, 0xffff)), None); + assert_eq!(first_nonce(&template, at, top, 2, 0xffff), None); + } + } }