diff --git a/crates/experimental_prover/keccak_gkr/src/kernels.rs b/crates/experimental_prover/keccak_gkr/src/kernels.rs index c8f9b9db61..7e78f8b0f1 100644 --- a/crates/experimental_prover/keccak_gkr/src/kernels.rs +++ b/crates/experimental_prover/keccak_gkr/src/kernels.rs @@ -3,8 +3,8 @@ //! //! Each has a portable form, an AVX-512 one (AVX-512BW, VPCLMULQDQ, GFNI), an AVX2 one //! (PCLMULQDQ, and VPCLMULQDQ where it is) and a NEON one (PMULL), which agree bit for bit. The -//! vector forms are compiled only where their features are. With AVX2 and GFNI a block's -//! [`skip_row`]s are summed on residues instead (`skip_gfni`). +//! vector forms are compiled only where their features are. On x86 a block's [`skip_row`]s are +//! summed on residues with GFNI (`skip_gfni`), and through cofactors without it (`skip_clmul`). use primitives::field::gf2_64::mul_wide; @@ -36,7 +36,7 @@ cfg_select! { mod simd { pub use super::{ embed_extend_portable as embed_extend, embed_portable as embed, fold_groups_portable as fold_groups, - products_portable as products, skip_row_portable as skip_row, + products_portable as products, row_products_portable as row_products, skip_row_portable as skip_row, }; } } @@ -176,6 +176,46 @@ pub fn products_portable(a: &[u32; PACKS], b: &[u32; PACKS], c: &[u32; PACKS]) - std::array::from_fn(|q| (mul_wide(ones, (a[q] ^ c[q]) as u64) ^ mul_wide(b[q] as u64, c[q] as u64)) as u64) } +/// The products of a row's five positions, which `chi` mixes cyclically, each handed to `f` as +/// it is made: `x` takes the embeddings of positions `x`, `x + 1` and `x + 2`, as [`products`] +/// does. +#[inline(always)] +pub fn row_products(words: [&[u64; BLOCK_WORDS]; 5], f: impl FnMut(usize, &[u64; PACKS])) { + simd::row_products(words, f) +} + +#[inline(always)] +pub fn row_products_portable(words: [&[u64; BLOCK_WORDS]; 5], mut f: impl FnMut(usize, &[u64; PACKS])) { + let e: [[u32; PACKS]; 5] = std::array::from_fn(|x| embed(words[x])); + for x in 0..5 { + f(x, &products(&e[x], &e[(x + 1) % 5], &e[(x + 2) % 5])); + } +} + +/// The words of a block position's products packed 48 bits each. +pub const PACKED_WORDS: usize = 3 * PACKS / 4; + +/// Products packed 48 bits each, product `q` at bits `48 q ..` of `out`: four to three words. +#[inline(always)] +pub fn pack_products(products: &[u64], out: &mut [u64]) { + for (out, p) in out.as_chunks_mut::<3>().0.iter_mut().zip(products.as_chunks::<4>().0) { + *out = [p[0] | p[1] << 48, p[1] >> 16 | p[2] << 32, p[2] >> 32 | p[3] << 16]; + } +} + +/// [`row_products`], each packed ([`pack_products`]) into `out[x]`. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq"))] +#[inline(always)] +pub fn row_packed(words: [&[u64; BLOCK_WORDS]; 5], out: [&mut [u64; PACKED_WORDS]; 5]) { + simd::row_packed(words, out) +} + +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq"))] +#[inline(always)] +pub fn row_packed_portable(words: [&[u64; BLOCK_WORDS]; 5], mut out: [&mut [u64; PACKED_WORDS]; 5]) { + row_products(words, |x, p| pack_products(p, &mut out[x][..])); +} + /// One row of five positions, which `chi` mixes cyclically: for every pattern, /// `acc[p] += sum over x of weights[x] * e[x + 1][p] * e[x + 2][p]`. #[inline(always)] @@ -256,6 +296,14 @@ cfg_select! { all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni") => { pub(crate) use crate::skip_gfni::SkipBlock; } + all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + not(all(target_feature = "avx512f", target_feature = "vpclmulqdq")) + ) => { + pub(crate) use crate::skip_clmul::SkipBlock; + } _ => { /// [`skip_row`] over every row and slice of a block, a position at a time. pub(crate) struct SkipBlock<'a> { @@ -265,8 +313,9 @@ cfg_select! { } impl<'a> SkipBlock<'a> { - /// For blocks summed at the position weights `weights`, as polynomials. - pub fn new(weights: &'a [[u64; 2]]) -> Box { + /// For blocks summed at the position weights `weights`, as polynomials. The sums are + /// exact, with no reduction modulo the minimal polynomial of the field's `s`. + pub fn new(weights: &'a [[u64; 2]], _field: &crate::power::PowerField) -> Box { Box::new(Self { acc: SkipAcc::new(), extended: [[0; SLOTS]; 5], @@ -333,7 +382,10 @@ mod x86 { use core::arch::x86_64::*; use super::*; - pub use super::{embed_extend_portable as embed_extend, products_clmul as products, skip_row_portable as skip_row}; + pub use super::{ + embed_extend_portable as embed_extend, products_clmul as products, row_packed_portable as row_packed, + row_products_portable as row_products, skip_row_portable as skip_row, + }; /// The matrices sending a pack's low and high byte to each byte of its embedding. An affine /// byte map reads output bit `t` off matrix byte `7 - t`. @@ -468,8 +520,8 @@ mod x86 { mod avx2 { use core::arch::x86_64::*; - pub use super::products_clmul as products; use super::*; + pub use super::{products_clmul as products, row_products_portable as row_products}; use crate::clmul::{V, Vector}; /// The 32 packs of a block position, embedded, four to a register. @@ -632,6 +684,229 @@ mod avx2 { acc.map(V::sum) }) } + + /// Table `6 * n + m`: byte `m` of `ones` times the embedding of each value of nibble `n` of a + /// pack, in both lanes. + #[cfg(not(target_feature = "vpclmulqdq"))] + static ONES_NIBBLES: [[u8; 32]; 24] = { + let mut tables = [[0u8; 32]; 24]; + let mut i = 0; + while i < 24 { + let (n, m) = (i / 6, i % 6); + let mut v = 0; + while v < 16 { + let mut e = 0u64; + let mut t = 0; + while t < 4 { + if v >> t & 1 == 1 { + e ^= RMFE.emb[4 * n + t] as u64; + } + t += 1; + } + let mut product = 0u64; + let mut k = 0; + while k < 32 { + if RMFE.ones >> k & 1 == 1 { + product ^= e << k; + } + k += 1; + } + tables[i][v] = (product >> (8 * m)) as u8; + tables[i][16 + v] = tables[i][v]; + v += 1; + } + i += 1; + } + tables + }; + + /// Per lane of eight packs, the low bytes of packs `0, 1, 4, 5`, then of `2, 3, 6, 7`, then + /// their high bytes. + #[cfg(not(target_feature = "vpclmulqdq"))] + const SPLIT: [u8; 32] = { + const LANE: [u8; 16] = [0, 2, 8, 10, 4, 6, 12, 14, 1, 3, 9, 11, 5, 7, 13, 15]; + let mut t = [0u8; 32]; + let mut i = 0; + while i < 32 { + t[i] = LANE[i % 16]; + i += 1; + } + t + }; + + /// `ones` times each pack's embedding, a linear map of the pack: per byte of the products, the + /// sum of four nibble lookups, then an 8x32 byte transpose to a word per pack and a shuffle to + /// the packed layout of [`Chunks`], two packs to a lane. + #[cfg(not(target_feature = "vpclmulqdq"))] + #[inline(always)] + fn ones_times(words: &[u64; BLOCK_WORDS]) -> [u64; PACKS] { + // SAFETY: the loads are of the eight words and the tables, the stores of `out`. + unsafe { + let split = _mm256_loadu_si256(SPLIT.as_ptr().cast()); + let idx = _mm256_setr_epi32(0, 4, 2, 6, 1, 5, 3, 7); + let half = |h: usize| { + let v = _mm256_shuffle_epi8(_mm256_loadu_si256(words.as_ptr().add(4 * h).cast()), split); + _mm256_permutevar8x32_epi32(v, idx) + }; + let (w0, w1) = (half(0), half(1)); + // Byte `16 l + 2 j + d` of each: pack `4 j + 2 l + d`. + let (low, high) = (_mm256_unpacklo_epi64(w0, w1), _mm256_unpackhi_epi64(w0, w1)); + let mask = _mm256_set1_epi8(0x0f); + let nibbles = [ + _mm256_and_si256(low, mask), + _mm256_and_si256(_mm256_srli_epi16(low, 4), mask), + _mm256_and_si256(high, mask), + _mm256_and_si256(_mm256_srli_epi16(high, 4), mask), + ]; + let look = |n: usize, m: usize| { + _mm256_shuffle_epi8(_mm256_loadu_si256(ONES_NIBBLES[6 * n + m].as_ptr().cast()), nibbles[n]) + }; + let zero = _mm256_setzero_si256(); + let p: [__m256i; 8] = std::array::from_fn(|m| { + if m < 6 { + _mm256_xor_si256( + _mm256_xor_si256(look(0, m), look(1, m)), + _mm256_xor_si256(look(2, m), look(3, m)), + ) + } else { + zero + } + }); + let y: [[__m256i; 2]; 4] = std::array::from_fn(|k| { + [ + _mm256_unpacklo_epi8(p[2 * k], p[2 * k + 1]), + _mm256_unpackhi_epi8(p[2 * k], p[2 * k + 1]), + ] + }); + let z: [[__m256i; 4]; 2] = std::array::from_fn(|s| { + std::array::from_fn(|h| { + let (u, v) = (y[2 * s][h / 2], y[2 * s + 1][h / 2]); + if h % 2 == 0 { + _mm256_unpacklo_epi16(u, v) + } else { + _mm256_unpackhi_epi16(u, v) + } + }) + }); + // A lane's second word's six bytes beside its first word's. + let pack = _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, + -1, -1, + ); + let mut out = [0u64; PACKS]; + for h in 0..4 { + // Register `2 h + d` holds packs `8 h + 4 d ..`, in order. + let w = [ + _mm256_unpacklo_epi32(z[0][h], z[1][h]), + _mm256_unpackhi_epi32(z[0][h], z[1][h]), + ]; + for (d, w) in w.into_iter().enumerate() { + _mm256_storeu_si256(out.as_mut_ptr().add(8 * h + 4 * d).cast(), _mm256_shuffle_epi8(w, pack)); + } + } + out + } + } + + /// Lane by lane, two packs' products packed into the lane's low twelve bytes, as + /// [`pack_products`] lays them out. + trait Chunks: Vector { + /// The products in each lane's low word of `low` and of `high`, the second 48 bits up. + fn chunks(low: Self, high: Self) -> Self; + /// Each lane's twelve bytes at `p + 12 l`. A lane's store runs four bytes past them, but + /// for the last lane of a `last` register. + /// + /// # Safety + /// `p` is valid for writing `12 * N` bytes, and four more unless `last`. + unsafe fn store_chunks(self, p: *mut u8, last: bool); + } + + /// Twelve bytes of a lane at `p`. + /// + /// # Safety + /// `p` is valid for writing twelve bytes. + #[inline(always)] + unsafe fn store_twelve(p: *mut u8, v: __m128i) { + unsafe { + _mm_storel_epi64(p.cast(), v); + _mm_storeu_si32(p.add(8).cast(), _mm_srli_si128::<8>(v)); + } + } + + impl Chunks for __m128i { + #[inline(always)] + fn chunks(low: Self, high: Self) -> Self { + // SAFETY: SSE2. + unsafe { _mm_xor_si128(low, _mm_slli_si128::<6>(high)) } + } + #[inline(always)] + unsafe fn store_chunks(self, p: *mut u8, last: bool) { + unsafe { + if last { + store_twelve(p, self); + } else { + _mm_storeu_si128(p.cast(), self); + } + } + } + } + + #[cfg(target_feature = "vpclmulqdq")] + impl Chunks for __m256i { + #[inline(always)] + fn chunks(low: Self, high: Self) -> Self { + // SAFETY: AVX2. + unsafe { _mm256_xor_si256(low, _mm256_bslli_epi128::<6>(high)) } + } + #[inline(always)] + unsafe fn store_chunks(self, p: *mut u8, last: bool) { + unsafe { + _mm_storeu_si128(p.cast(), _mm256_castsi256_si128(self)); + let high = _mm256_extracti128_si256::<1>(self); + if last { + store_twelve(p.add(12), high); + } else { + _mm_storeu_si128(p.add(12).cast(), high); + } + } + } + } + + /// [`row_products`] packed into `out` as [`pack_products`] lays them out, a lane's two + /// products packed in registers and stored twelve bytes apart. With PCLMULQDQ alone, a + /// product a lane, `ones * (a + c)` comes from [`ones_times`] of each position, which halves + /// the carry-less products; with VPCLMULQDQ the products are cheaper than the lookups. + #[inline(always)] + pub fn row_packed(words: [&[u64; BLOCK_WORDS]; 5], out: [&mut [u64; PACKED_WORDS]; 5]) { + let e: [[u32; PACKS]; 5] = std::array::from_fn(|x| embed(words[x])); + #[cfg(not(target_feature = "vpclmulqdq"))] + let t: [[u64; PACKS]; 5] = std::array::from_fn(|x| ones_times(words[x])); + #[cfg(target_feature = "vpclmulqdq")] + let ones = V::splat(RMFE.ones as u64); + for (x, out) in out.into_iter().enumerate() { + let (b, c) = (&e[(x + 1) % 5], &e[(x + 2) % 5]); + let dst = out.as_mut_ptr().cast::(); + for q in (0..PACKS).step_by(2 * V::N) { + // SAFETY: `2 * N` packs and words at `q`; the stores are of packs `q ..` (six + // bytes each), and four bytes past them but at the last packs. + unsafe { + let (zb, zc) = (V::load_packs(b.as_ptr().add(q)), V::load_packs(c.as_ptr().add(q))); + let (low, high) = (zb.clmul::<0x00>(zc), zb.clmul::<0x11>(zc)); + let chunks = cfg_select! { + target_feature = "vpclmulqdq" => {{ + let linear = V::load_packs(e[x].as_ptr().add(q)).xor(zc); + V::chunks(low.xor(linear.clmul::<0x00>(ones)), high.xor(linear.clmul::<0x11>(ones))) + }} + _ => {{ + let linear = V::load(t[x].as_ptr().add(q)).xor(V::load(t[(x + 2) % 5].as_ptr().add(q))); + V::chunks(low, high).xor(linear) + }} + }; + chunks.store_chunks(dst.add(6 * q), q + 2 * V::N == PACKS); + } + } + } + } } /// Compiled only where its feature is, so its kernels are safe to call. @@ -644,6 +919,7 @@ mod neon { pmull, pmull_hi as mul_hi_scheduled, pmull_hi_asm as mul_hi, pmull_lo_asm as mul_lo, }; + pub use super::row_products_portable as row_products; use super::*; /// A linear map is the sum of its nibbles' images, so each byte of sixteen embeddings is four @@ -868,6 +1144,28 @@ mod tests { embed(&std::array::from_fn(|_| rng.next_u64())), ); assert_eq!(products(&e, &b, &c), products_portable(&e, &b, &c)); + let row: [[u64; BLOCK_WORDS]; 5] = std::array::from_fn(|_| std::array::from_fn(|_| rng.next_u64())); + let er: [[u32; PACKS]; 5] = std::array::from_fn(|x| embed_portable(&row[x])); + let expected: [[u64; PACKS]; 5] = + std::array::from_fn(|x| products_portable(&er[x], &er[(x + 1) % 5], &er[(x + 2) % 5])); + let mut got = [[0u64; PACKS]; 5]; + row_products(row.each_ref(), |x, p| got[x] = *p); + assert_eq!(got, expected); + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq"))] + { + // A guard word after each row: the packed stores run past a product but not past the row. + let mut packed = [[u64::MAX; PACKED_WORDS + 1]; 5]; + row_packed( + row.each_ref(), + packed.each_mut().map(|p| (&mut p[..PACKED_WORDS]).try_into().unwrap()), + ); + for (packed, expected) in packed.iter().zip(&expected) { + let mut want = [0u64; PACKED_WORDS + 1]; + pack_products(expected, &mut want); + want[PACKED_WORDS] = u64::MAX; + assert_eq!(*packed, want); + } + } let (mut xe, mut xp) = ([0u32; SLOTS], [0u32; SLOTS]); embed_extend(&words, &mut xe); @@ -898,11 +1196,14 @@ mod tests { let weights: Vec<[u64; 2]> = (0..BITS) .map(|_| std::array::from_fn(|_| if dense { u64::MAX } else { rng.next_u64() })) .collect(); - let mut skip = SkipBlock::new(&weights); + // A kernel's sums may be reduced modulo the minimal polynomial of `s`. + let field = crate::power::PowerField::new(rng.ext()).expect("a generator"); + let reduce = |[a, b, c]: [u64; 3]| field.reduce(&[a, b, c, 0]); + let mut skip = SkipBlock::new(&weights, &field); for (b, block) in blocks.iter().enumerate() { skip.run(block); for (i, expected) in skip_block_portable(block, &weights).into_iter().enumerate() { - assert_eq!(skip.get(i), expected, "block {b} slot {i}"); + assert_eq!(reduce(skip.get(i)), reduce(expected), "block {b} slot {i}"); } } } diff --git a/crates/experimental_prover/keccak_gkr/src/lib.rs b/crates/experimental_prover/keccak_gkr/src/lib.rs index bc74084840..b1b2008378 100644 --- a/crates/experimental_prover/keccak_gkr/src/lib.rs +++ b/crates/experimental_prover/keccak_gkr/src/lib.rs @@ -86,9 +86,10 @@ //! of eight positions' weights applied to bit-transposed products (`products_gfni`); elsewhere, //! the positions grouped by each byte of their weight (`prover`'s digit buckets). //! -//! The four rounds' sums are binary polynomials of degree at most 173. With AVX2 and GFNI they -//! are taken on residues modulo 24 irreducibles of degree 8, where a product is a byte product -//! (`skip_gfni`). +//! The four rounds' sums are binary polynomials of degree at most 173. With GFNI (x86-64) they are +//! taken on residues modulo 24 irreducibles of degree 8, where a product is a byte product +//! (`skip_gfni`). Without it, x86-64 multiplies out the weights on the packs first, so that a +//! sum takes fewer carry-less products (`skip_clmul`). //! //! The rounds that remain fold a table of one field element per half block and position. A pass over it //! binds two rounds' variables: over each quad of the rows it writes it sums the next round and, as a @@ -114,15 +115,18 @@ mod products_gfni; mod protocol; mod prover; pub mod rmfe; -#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] -mod skip_gfni; #[cfg(all( target_arch = "x86_64", - target_feature = "avx512f", - target_feature = "avx512bw", - target_feature = "gfni" + target_feature = "avx2", + target_feature = "pclmulqdq", + not(target_feature = "gfni"), + not(all(target_feature = "avx512f", target_feature = "vpclmulqdq")) ))] -mod table_gfni; +mod skip_clmul; +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] +mod skip_gfni; +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +mod table_maps; pub mod witness; #[cfg(test)] diff --git a/crates/experimental_prover/keccak_gkr/src/power.rs b/crates/experimental_prover/keccak_gkr/src/power.rs index f3e4264ef1..2fc94cb317 100644 --- a/crates/experimental_prover/keccak_gkr/src/power.rs +++ b/crates/experimental_prover/keccak_gkr/src/power.rs @@ -44,9 +44,9 @@ macro_rules! row_kernel { pub struct PowerField { pub basis: PowerBasis, /// `s^128`, the modulus without its leading term. - modulus: Element, + pub(crate) modulus: Element, /// `floor(x^256 / modulus)` without its leading term, for Barrett's reduction. - barrett: Element, + pub(crate) barrett: Element, eval: PolyEval, } diff --git a/crates/experimental_prover/keccak_gkr/src/products_gfni.rs b/crates/experimental_prover/keccak_gkr/src/products_gfni.rs index 0461f90f1e..08373427b6 100644 --- a/crates/experimental_prover/keccak_gkr/src/products_gfni.rs +++ b/crates/experimental_prover/keccak_gkr/src/products_gfni.rs @@ -2,36 +2,43 @@ //! same values as [`crate::prover`]'s digit buckets, `sum over positions j of weights[j] (x) //! product(q, j)` for every pack `q` of a block and every coefficient of its product. //! +//! A product of two embeddings has degree below `8 * POINTS`, so it is fixed by its values at +//! [`POINTS`] points of GF(2^8), the field of GF2P8MULB, with distinct minimal polynomials. A value +//! is a byte: the embeddings' values are affine byte maps of the witness's bytes ([`EVAL`]), and +//! a product's are byte products. The sums are taken over the values' bits, which the inverse of +//! the evaluation map ([`CRT`]) sends back to coefficients once a layer ([`coefficients`]). +//! //! Eight consecutive `z` of one lane are a *lane byte*. For a lane byte's positions `j_0 .. j_7` //! and one byte `b` of their weights, GF2P8AFFINEQB with the matrix `M_b` maps a *column byte* -//! (bit `k` = one coefficient of `product(q, j_k)`) to byte `b` of the sum of the weights of the -//! positions whose coefficient is set: the lane byte's share of that coefficient's sum. One -//! instruction does this for 32 (pack, coefficient) column bytes at once. +//! (bit `k` = one bit of a value of `product(q, j_k)`) to byte `b` of the sum of the weights of +//! the positions whose bit is set: the lane byte's share of that bit's sum. One instruction does +//! this for 32 (pack, bit) column bytes at once. //! //! Per block, two phases: //! 1. per *row byte* with a nonzero weight (the same byte of the five lanes of a row, which `chi` -//! mixes): the five lane bytes' embeddings, the products of each active lane byte, and their -//! bit transposes into column bytes, staged register by register in a per-thread buffer; +//! mixes): the five lane bytes' values, the products of each active lane byte, and their byte +//! and bit transposes into column bytes, staged register by register in a per-thread buffer; //! 2. per staged register, every active lane byte's 16 matrices applied into accumulators held in //! registers (eight at a time on AVX2, all 16 with AVX-512), then one byte transpose back to //! field elements. //! -//! The 8x8 byte transpose without VPERMB ([`transpose_bytes8x8`]) leaves the positions in a fixed -//! order ([`TRANSPOSED_ORDER`]), which the matrices absorb ([`COLUMN_BIT_POSITION`]). -//! //! The two-phase outer-product scheme follows hashmancer-keccak's GFNI kernels; the code here is //! written for this crate's layout (16-instance packs, 128-bit weights). use core::arch::x86_64::*; use primitives::field::F128; -use verifier::keccak::BITS; +use verifier::keccak::{BITS, EMBED_BITS, PRODUCT_BITS}; -use crate::kernels::{PACKS, embed, products}; +use crate::kernels::PACKS; use crate::prover::PAYLOAD_BITS; -use crate::witness::words; +use crate::rmfe::RMFE; +use crate::witness::{BLOCK_WORDS, words}; -/// Staged registers per active lane byte: 32 column bytes each, six coefficient bytes per pack. +/// Points a product is held at, a byte each. +const POINTS: usize = 6; +const _: () = assert!(PRODUCT_BITS <= 8 * POINTS && PAYLOAD_BITS == 8 * POINTS * PACKS); +/// Staged registers per active lane byte: 32 column bytes each, one per pack and bit of a value. const STAGED: usize = PAYLOAD_BITS / 32; /// Bytes of a weight. const WEIGHT_BYTES: usize = 16; @@ -41,68 +48,237 @@ type V = __m256i; /// The GF2P8AFFINEQB source that turns the instruction into an 8x8 bit transpose of its matrix. const BIT_TRANSPOSE: i64 = 0x8040_2010_0804_0201_u64 as i64; -/// The qword order [`transpose_bytes8x8`] leaves: output qword `d` byte `n` is byte `d` of input -/// `TRANSPOSED_ORDER[n]`. -const TRANSPOSED_ORDER: [usize; 8] = [0, 1, 4, 5, 2, 3, 6, 7]; /// The position that bit `i` of a column byte stands for, after the byte and bit transposes. -const COLUMN_BIT_POSITION: [usize; 8] = { - let mut p = [0; 8]; +const COLUMN_BIT_POSITION: [usize; 8] = [7, 6, 5, 4, 3, 2, 1, 0]; + +/// The byte product: binary polynomials modulo `x^8 + x^4 + x^3 + x + 1`. +const fn gf_mul(a: u8, b: u8) -> u8 { + let (mut a, mut product) = (a, 0); let mut i = 0; while i < 8 { - p[i] = TRANSPOSED_ORDER[7 - i]; + if b >> i & 1 == 1 { + product ^= a; + } + a = a << 1 ^ if a >> 7 == 1 { 0x1b } else { 0 }; + i += 1; + } + product +} + +/// `POWERS[i][n]` is point `i` to the power `n`. A point is the least of its eight conjugates, the +/// roots of its irreducible, so no two share one; an element of a proper subfield has fewer. +const POWERS: [[u8; 8 * POINTS]; POINTS] = { + let mut powers = [[1u8; 8 * POINTS]; POINTS]; + let (mut i, mut t) = (0, 0u8); + while i < POINTS { + t += 1; + let (mut conjugate, mut least) = (t, true); + let mut k = 1; + while k < 8 { + conjugate = gf_mul(conjugate, conjugate); + least &= conjugate > t; + k += 1; + } + if least { + let mut n = 1; + while n < 8 * POINTS { + powers[i][n] = gf_mul(powers[i][n - 1], t); + n += 1; + } + i += 1; + } + } + powers +}; + +/// The value at point `i` of the polynomial with coefficient bits `p`. +const fn value(p: u32, i: usize) -> u8 { + let (mut v, mut n) = (0, 0); + while n < EMBED_BITS { + if p >> n & 1 == 1 { + v ^= POWERS[i][n]; + } + n += 1; + } + v +} + +/// The matrix of the affine byte map sending bit `i` of a byte to `images[i]`: the map reads +/// output bit `t` off matrix byte `7 - t`. +const fn byte_map(images: [u8; 8]) -> u64 { + let mut matrix = 0u64; + let mut t = 0; + while t < 8 { + let mut i = 0; + while i < 8 { + matrix |= ((images[i] >> t & 1) as u64) << (8 * (7 - t) + i); + i += 1; + } + t += 1; + } + matrix +} + +/// `EVAL[h][i]` sends byte `h` of a pack to its share of the value of the pack's embedding at +/// point `i`. +const EVAL: [[u64; POINTS]; 2] = { + let mut maps = [[0u64; POINTS]; 2]; + let mut h = 0; + while h < 2 { + let mut i = 0; + while i < POINTS { + let mut images = [0u8; 8]; + let mut b = 0; + while b < 8 { + images[b] = value(RMFE.emb[8 * h + b], i); + b += 1; + } + maps[h][i] = byte_map(images); + i += 1; + } + h += 1; + } + maps +}; + +/// The value of `ones` at each point. +const ONES: [u8; POINTS] = { + let mut ones = [0u8; POINTS]; + let mut i = 0; + while i < POINTS { + ones[i] = value(RMFE.ones, i); + i += 1; + } + ones +}; + +/// `CRT[n]` is the mask of the value bits (bit `t` of the value at point `i` at bit `8 i + t`) +/// whose sum is coefficient `n`: row `n` of the inverse of the evaluation map on polynomials of +/// fewer than `8 * POINTS` coefficients, by Gauss-Jordan elimination. +const CRT: [u64; 8 * POINTS] = { + const N: usize = 8 * POINTS; + // Row `8 i + t` sends a polynomial to bit `t` of its value at point `i`, beside the row of the + // identity that the row operations turn into the inverse's. + let mut rows = [[0u64; 2]; N]; + let mut r = 0; + while r < N { + let mut n = 0; + while n < N { + rows[r][0] |= ((POWERS[r / 8][n] >> (r % 8) & 1) as u64) << n; + n += 1; + } + rows[r][1] = 1 << r; + r += 1; + } + let mut c = 0; + while c < N { + let mut p = c; + while rows[p][0] >> c & 1 == 0 { + p += 1; + } + let pivot = rows[p]; + rows.swap(p, c); + let mut r = 0; + while r < N { + if r != c && rows[r][0] >> c & 1 == 1 { + rows[r][0] ^= pivot[0]; + rows[r][1] ^= pivot[1]; + } + r += 1; + } + c += 1; + } + let mut crt = [0u64; N]; + let mut n = 0; + while n < N { + crt[n] = rows[n][1]; + n += 1; + } + crt +}; + +/// `SLOT[q][8 i + t]` is the entry of [`Scratch::out`] holding bit `t` of pack `q`'s values at +/// point `i`. Value registers ([`evaluate`]) hold pack `16 s + 8 l + u` at byte `16 l + 8 s + u`, +/// and [`ProductSums::stage_lane_byte`] puts byte `16 l + 2 j + d` at qword `2 l + d` of register +/// `8 i + j`, whose byte `n` is entry `32 r + n`. +const SLOT: [[u16; 8 * POINTS]; PACKS] = { + let mut slot = [[0u16; 8 * POINTS]; PACKS]; + let mut i = 0; + while i < POINTS { + let mut j = 0; + while j < 8 { + let mut l = 0; + while l < 2 { + let mut d = 0; + while d < 2 { + let byte = 2 * j + d; + let q = 16 * (byte / 8) + 8 * l + byte % 8; + let mut t = 0; + while t < 8 { + slot[q][8 * i + t] = (32 * (8 * i + j) + 8 * (2 * l + d) + t) as u16; + t += 1; + } + d += 1; + } + l += 1; + } + j += 1; + } i += 1; } - p + slot }; -/// In-lane byte interleave of the lane's two qwords (16-bit element `d` = byte `d` of both). -const INTERLEAVE: [u8; 32] = { +/// Per-lane byte shuffle of a pack's two bytes apart: the lane's low bytes, then its high ones. +const SPLIT: [u8; 32] = { let mut t = [0u8; 32]; let mut i = 0; while i < 32 { let j = i % 16; - t[i] = if j % 2 == 0 { (j / 2) as u8 } else { (8 + j / 2) as u8 }; + t[i] = if j < 8 { 2 * j as u8 } else { 2 * (j - 8) as u8 + 1 }; i += 1; } t }; -/// 8x8 byte transpose of the qwords `a_0 .. a_7` (`p` = `a_0 .. a_3`, `q` = `a_4 .. a_7`): output -/// qword `d` (`d < 4` in the first register) byte `n` = byte `d` of `a_{TRANSPOSED_ORDER[n]}`. Two -/// in-lane byte shuffles, two 16-bit unpacks and two dword permutes. -/// -/// # Safety -/// AVX2 (the module's build). +/// The values of a block position's 32 packs' embeddings at each point: register `i` holds point +/// `i`, pack `16 s + 8 l + u` at byte `16 l + 8 s + u`. #[inline(always)] -unsafe fn transpose_bytes8x8(p: V, q: V) -> [V; 2] { +fn evaluate(words: &[u64; BLOCK_WORDS]) -> [V; POINTS] { + // SAFETY: AVX2 and GFNI are enabled at compile time (module cfg); the loads are of the eight + // words and of `SPLIT`. unsafe { - let il = _mm256_loadu_si256(INTERLEAVE.as_ptr().cast()); - let (p, q) = (_mm256_shuffle_epi8(p, il), _mm256_shuffle_epi8(q, il)); - // Dword d of lane 0: bytes d of (a0, a1, a4, a5); of lane 1: of (a2, a3, a6, a7). - let (e, f) = (_mm256_unpacklo_epi16(p, q), _mm256_unpackhi_epi16(p, q)); - let idx = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); - [_mm256_permutevar8x32_epi32(e, idx), _mm256_permutevar8x32_epi32(f, idx)] + let split = _mm256_loadu_si256(SPLIT.as_ptr().cast()); + let w0 = _mm256_shuffle_epi8(_mm256_loadu_si256(words.as_ptr().cast()), split); + let w1 = _mm256_shuffle_epi8(_mm256_loadu_si256(words.as_ptr().add(4).cast()), split); + let (low, high) = (_mm256_unpacklo_epi64(w0, w1), _mm256_unpackhi_epi64(w0, w1)); + std::array::from_fn(|i| { + _mm256_xor_si256( + _mm256_gf2p8affine_epi64_epi8(low, _mm256_set1_epi64x(EVAL[0][i] as i64), 0), + _mm256_gf2p8affine_epi64_epi8(high, _mm256_set1_epi64x(EVAL[1][i] as i64), 0), + ) + }) } } -/// 4x4 transpose of qwords: `out[i]` qword `k` = `r[k]` qword `i`. -/// -/// # Safety -/// AVX2. -#[inline(always)] -unsafe fn transpose_qwords4x4(r: [V; 4]) -> [V; 4] { - unsafe { - let t0 = _mm256_unpacklo_epi64(r[0], r[1]); - let t1 = _mm256_unpackhi_epi64(r[0], r[1]); - let t2 = _mm256_unpacklo_epi64(r[2], r[3]); - let t3 = _mm256_unpackhi_epi64(r[2], r[3]); - [ - _mm256_permute2x128_si256(t0, t2, 0x20), - _mm256_permute2x128_si256(t1, t3, 0x20), - _mm256_permute2x128_si256(t0, t2, 0x31), - _mm256_permute2x128_si256(t1, t3, 0x31), - ] - } +/// Per pack, the coefficients of the weighted sums of its products, from the weighted sums of +/// their value bits (entry [`SLOT`]`[q][8 i + t]` of `sums`). +pub fn coefficients(sums: &[F128]) -> Vec<[F128; PRODUCT_BITS]> { + assert_eq!(sums.len(), PAYLOAD_BITS); + SLOT.iter() + .map(|slot| { + let bits: [F128; 8 * POINTS] = std::array::from_fn(|r| sums[slot[r] as usize]); + std::array::from_fn(|n| { + let mut mask = CRT[n]; + let mut c = F128::ZERO; + while mask != 0 { + c += bits[mask.trailing_zeros() as usize]; + mask &= mask - 1; + } + c + }) + }) + .collect() } /// Where staged register `r` of active lane byte `a` lies in the buffer (`n_active` active lane @@ -189,7 +365,7 @@ impl ProductSums { } /// The weighted bit sums of a block (`BITS` positions of eight words) into `s.out`: entry - /// `48 q + c` is `sum over j of weights[j] * coefficient c of product(q, j)`. + /// [`SLOT`]`[q][8 i + t]` is `sum over j of weights[j] * bit t of product(q, j) at point i`. pub fn block_sums(&self, s: &mut Scratch, block: &[u64]) { let n_active = self.matrices.len(); self.stage(block, &mut s.staged, n_active); @@ -200,26 +376,29 @@ impl ProductSums { /// Phase 1: every active lane byte's column bytes into `staged[staged_index(r, a, n_active)]`. fn stage(&self, block: &[u64], staged: &mut [V], n_active: usize) { assert!(staged.len() >= STAGED * n_active); - let mut embedded = [[[0u32; PACKS]; 8]; 5]; - let mut prod = [[0u64; PACKS]; 8]; + // SAFETY: AVX2 is enabled at compile time. + let mut values = [[[unsafe { _mm256_setzero_si256() }; POINTS]; 8]; 5]; let mut a = 0; for &(y, zb, mask) in &self.row_bytes { let (y, zb) = (y as usize, zb as usize); - for (x, lane) in embedded.iter_mut().enumerate() { - for (k, e) in lane.iter_mut().enumerate() { - *e = embed(words(block, 64 * (x + 5 * y) + 8 * zb + k)); + for (x, lane) in values.iter_mut().enumerate() { + for (k, v) in lane.iter_mut().enumerate() { + *v = evaluate(words(block, 64 * (x + 5 * y) + 8 * zb + k)); } } for x in (0..5).filter(|x| mask >> x & 1 == 1) { - let (e0, e1, e2) = (&embedded[x], &embedded[(x + 1) % 5], &embedded[(x + 2) % 5]); - for (k, p) in prod.iter_mut().enumerate() { - *p = products(&e0[k], &e1[k], &e2[k]); - } assert!(a < n_active, "one staged column per active lane byte"); // SAFETY: AVX2 and GFNI are enabled at compile time; `a < n_active` (asserted) and - // `stage_lane_byte` writes registers `r < STAGED` only (`3 * 15 + 2`), so every store + // `stage_lane_byte` writes registers `r < STAGED` only (`8 * 5 + 7`), so every store // is inside `staged` (asserted size). - unsafe { Self::stage_lane_byte(&prod, staged.as_mut_ptr(), a, n_active) }; + unsafe { + Self::stage_lane_byte( + [&values[x], &values[(x + 1) % 5], &values[(x + 2) % 5]], + staged.as_mut_ptr(), + a, + n_active, + ) + }; a += 1; } } @@ -227,38 +406,54 @@ impl ProductSums { assert_eq!(a, n_active, "every active lane byte staged"); } - /// The column bytes of one lane byte's products (`prod[k][q]`: position `k`, pack `q`): register - /// `r` holds column qwords `4 r .. 4 r + 4`, qword `6 q + m` being coefficient byte `m` of pack - /// `q` (bit `i` of its byte `b`: coefficient `8 m + b` of position `COLUMN_BIT_POSITION[i]`). + /// The column bytes of one lane byte's products, from the values of its three factors' + /// embeddings (`e[f][k][i]`: factor `f`, position `k`, point `i`): register `8 i + j` holds, in + /// qword `2 l + d`, the column byte of each bit of the products' values at point `i` for the + /// pack at byte `16 l + 2 j + d` of a value register ([`SLOT`]). /// /// # Safety /// AVX2 and GFNI; `staged + staged_index(r, a, n_active)` writable for `r < STAGED`. #[inline(always)] - unsafe fn stage_lane_byte(prod: &[[u64; PACKS]; 8], staged: *mut V, a: usize, n_active: usize) { + unsafe fn stage_lane_byte(e: [&[[V; POINTS]; 8]; 3], staged: *mut V, a: usize, n_active: usize) { unsafe { let bit_transpose = _mm256_set1_epi64x(BIT_TRANSPOSE); - let ld = |k: usize, q: usize| _mm256_loadu_si256(prod[k].as_ptr().add(q).cast()); - for qq in (0..PACKS).step_by(4) { - // t[h][i] qword k = the product of pack qq + i at position 4 h + k. - let t = [ - transpose_qwords4x4([ld(0, qq), ld(1, qq), ld(2, qq), ld(3, qq)]), - transpose_qwords4x4([ld(4, qq), ld(5, qq), ld(6, qq), ld(7, qq)]), - ]; - let columns: [[V; 2]; 4] = std::array::from_fn(|i| { - let [lo, hi] = transpose_bytes8x8(t[0][i], t[1][i]); + for i in 0..POINTS { + let ones = _mm256_set1_epi8(ONES[i] as i8); + // `ones * (a + c) + b * c` at point `i`, per position. + let p: [V; 8] = std::array::from_fn(|k| { + let (a, b, c) = (e[0][k][i], e[1][k][i], e[2][k][i]); + _mm256_xor_si256( + _mm256_gf2p8mul_epi8(_mm256_xor_si256(a, c), ones), + _mm256_gf2p8mul_epi8(b, c), + ) + }); + // 8x32 byte transpose: qword `d` of lane `l` of `w[j]` holds byte `16 l + 2 j + d` + // of each position, in order. + let y: [[V; 2]; 4] = std::array::from_fn(|m| { [ - _mm256_gf2p8affine_epi64_epi8(bit_transpose, lo, 0), - _mm256_gf2p8affine_epi64_epi8(bit_transpose, hi, 0), + _mm256_unpacklo_epi8(p[2 * m], p[2 * m + 1]), + _mm256_unpackhi_epi8(p[2 * m], p[2 * m + 1]), ] }); - for pair in 0..2 { - // Packs q, q + 1 take three registers: bytes 0..4 of q; 4, 5 of q and 0, 1 - // of q + 1; 2..4 and 4, 5 of q + 1. - let (c0, c1) = (columns[2 * pair], columns[2 * pair + 1]); - let r = 3 * ((qq + 2 * pair) / 2); - *staged.add(staged_index(r, a, n_active)) = c0[0]; - *staged.add(staged_index(r + 1, a, n_active)) = _mm256_permute2x128_si256(c0[1], c1[0], 0x20); - *staged.add(staged_index(r + 2, a, n_active)) = _mm256_permute2x128_si256(c1[0], c1[1], 0x21); + let z: [[V; 4]; 2] = std::array::from_fn(|s| { + std::array::from_fn(|h| { + let (u, v) = (y[2 * s][h / 2], y[2 * s + 1][h / 2]); + if h % 2 == 0 { + _mm256_unpacklo_epi16(u, v) + } else { + _mm256_unpackhi_epi16(u, v) + } + }) + }); + for h in 0..4 { + let w = [ + _mm256_unpacklo_epi32(z[0][h], z[1][h]), + _mm256_unpackhi_epi32(z[0][h], z[1][h]), + ]; + for (d, w) in w.into_iter().enumerate() { + let r = 8 * i + 2 * h + d; + *staged.add(staged_index(r, a, n_active)) = _mm256_gf2p8affine_epi64_epi8(bit_transpose, w, 0); + } } } } diff --git a/crates/experimental_prover/keccak_gkr/src/prover.rs b/crates/experimental_prover/keccak_gkr/src/prover.rs index db472c174b..6e6a0137dc 100644 --- a/crates/experimental_prover/keccak_gkr/src/prover.rs +++ b/crates/experimental_prover/keccak_gkr/src/prover.rs @@ -16,7 +16,7 @@ use primitives::transcript::ProverState; use verifier::keccak::{BITS, LOCAL_BITS, along, linear_transpose}; use crate::basis::{PolyEval, add_mul}; -use crate::kernels::{GROUP, MIXED, PACKS, SKIP, SkipBlock, TERNARY, embed, mixed_points, products, slot}; +use crate::kernels::{GROUP, MIXED, PACKS, SKIP, SkipBlock, TERNARY, mixed_points, slot}; use crate::power::{Element, PowerField, get, set}; use crate::protocol::{Claim, chi_weights, mu_at}; use crate::witness::{Shape, Witness, transpose64, words}; @@ -33,12 +33,10 @@ const DIGITS: usize = 128 / DIGIT_BITS; /// The bits of a block position's products, 48 to each pack. pub(crate) const PAYLOAD_BITS: usize = 48 * PACKS; /// The packs of a part of a block position, and the words of their products: with AVX-512 the -/// whole position, three registers; with AVX2 half, three registers; elsewhere a quarter, a -/// block's worth of which stays in cache. -#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] +/// whole position, three registers; with AVX2 the whole position, six registers; elsewhere a +/// quarter, a block's worth of which stays in cache. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] const PART: usize = PACKS; -#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] -const PART: usize = PACKS / 2; #[cfg(not(all(target_arch = "x86_64", target_feature = "avx2")))] const PART: usize = 8; const PART_WORDS: usize = 48 * PART / 64; @@ -64,12 +62,11 @@ impl Part { const ZERO: Self = Self([0; PART_WORDS]); /// Product `q` at bits `48 * q ..`: four products to three words. + #[cfg(not(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq")))] #[inline(always)] fn pack(products: &[u64; PART]) -> Self { let mut out = [0u64; PART_WORDS]; - for (out, p) in out.as_chunks_mut::<3>().0.iter_mut().zip(products.as_chunks::<4>().0) { - *out = [p[0] | p[1] << 48, p[1] >> 16 | p[2] << 32, p[2] >> 32 | p[3] << 16]; - } + crate::kernels::pack_products(products, &mut out); Self(out) } @@ -92,7 +89,7 @@ impl Part { } } #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] - // SAFETY: both parts are three registers, and AVX2 is enabled at compile time. + // SAFETY: both parts are six registers, and AVX2 is enabled at compile time. unsafe { use core::arch::x86_64::*; let (dst, src) = (self.0.as_mut_ptr(), other.0.as_ptr()); @@ -155,7 +152,9 @@ impl Groups { /// `sum over positions of weight[j] (x) payloads[j]` into `rows`: row `DIGIT_BITS * d + i` /// sums the payloads whose weight has that bit set. Each group is summed in registers, so - /// the payloads are only read. + /// the payloads are only read, then added into the sums of its value's low nibble and of its + /// high one (`sums[..16]`, `sums[16..32]`), the latter kept in registers; those make the + /// rows of the digit's low bits and of its high bits. fn add_rows(&self, payloads: &[Part; BITS], rows: &mut [Part; 128], sums: &mut [Part; 1 << DIGIT_BITS]) { for (rows, (order, starts)) in rows .as_chunks_mut::() @@ -163,23 +162,33 @@ impl Groups { .iter_mut() .zip(self.order.iter().zip(&self.starts)) { - for (v, sum) in sums.iter_mut().enumerate() { - let mut acc = Part::ZERO; - for &p in &order[starts[v] as usize..starts[v + 1] as usize] { - // SAFETY: `order` holds positions. - acc.add(unsafe { payloads.get_unchecked(p as usize) }); + let (low, high) = sums[..32].split_at_mut(16); + low.fill(Part::ZERO); + for (h, high) in high.iter_mut().enumerate() { + let mut nibble = Part::ZERO; + for (l, low) in low.iter_mut().enumerate() { + let v = 16 * h + l; + let mut acc = Part::ZERO; + for &p in &order[starts[v] as usize..starts[v + 1] as usize] { + // SAFETY: `order` holds positions. + acc.add(unsafe { payloads.get_unchecked(p as usize) }); + } + nibble.add(&acc); + low.add(&acc); } - *sum = acc; + *high = nibble; } - // Bit `i` of the digit: the sums whose value has it, then fold it away. - for (i, row) in rows.iter_mut().enumerate().rev() { - let (low, high) = sums[..2 << i].split_at_mut(1 << i); - let mut acc = Part::ZERO; - for (low, high) in low.iter_mut().zip(&*high) { - acc.add(high); - low.add(high); + for (sums, rows) in [low, high].into_iter().zip(rows.as_chunks_mut::<4>().0) { + // Bit `i` of the nibble: the sums whose value has it, then fold it away. + for (i, row) in rows.iter_mut().enumerate().rev() { + let (low, high) = sums[..2 << i].split_at_mut(1 << i); + let mut acc = Part::ZERO; + for (low, high) in low.iter_mut().zip(&*high) { + acc.add(high); + low.add(high); + } + *row = acc; } - *row = acc; } } } @@ -272,9 +281,8 @@ impl Layer<'_> { a }, ); - (0..PACKS) - .map(|q| std::array::from_fn(|c| sums[48 * q + c].reduce())) - .collect() + let sums: Vec = sums.iter().map(|s| s.reduce()).collect(); + crate::products_gfni::coefficients(&sums) } /// [`Self::pack_products`] by digit buckets: the block's positions grouped by each digit of @@ -301,11 +309,20 @@ impl Layer<'_> { for r in 0..5 { for z in 0..64 { let js = row_positions(r, z); - let e: [[u32; PACKS]; 5] = std::array::from_fn(|x| embed(words(block, js[x]))); - for x in 0..5 { - let p = products(&e[x], &e[(x + 1) % 5], &e[(x + 2) % 5]); - for (q, part) in payloads.iter_mut().enumerate() { - part[js[x]] = Part::pack(p[PART * q..][..PART].try_into().unwrap()); + let row = js.map(|j| words(block, j)); + cfg_select! { + all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq") => { + // A part is a whole position, so the products pack straight into it. + // SAFETY: a row's five positions are distinct and below `BITS`. + let parts = unsafe { payloads[0].get_disjoint_unchecked_mut(js) }; + crate::kernels::row_packed(row, parts.map(|p| &mut p.0)); + } + _ => { + crate::kernels::row_products(row, |x, p| { + for (q, part) in payloads.iter_mut().enumerate() { + part[js[x]] = Part::pack(p[PART * q..][..PART].try_into().unwrap()); + } + }); } } } @@ -363,7 +380,7 @@ impl Layer<'_> { let (nb, written) = (self.witness.shape.n_blocks(), self.witness.written()); let mixed = parallel::map_reduce_with_state( self.rounds.len() * written, - || SkipBlock::new(weights), + || SkipBlock::new(weights, field), || vec![[0u64; 5]; MIXED], |skip, total, task| { let (segment, g) = (task / written, task % written); @@ -411,37 +428,27 @@ impl Layer<'_> { let field = round.field; let fold: Vec = eq_table(challenges).iter().map(|&e| field.to_poly(e)).collect(); let fold: [[u64; GROUP]; 2] = std::array::from_fn(|k| std::array::from_fn(|q| fold[q][k])); - #[cfg(all( - target_arch = "x86_64", - target_feature = "avx512f", - target_feature = "avx512bw", - target_feature = "gfni" - ))] - let row_maps = crate::table_gfni::RowMaps::new(field, &fold); + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] + let row_maps = crate::table_maps::RowMaps::new(field, &fold); // Rows of segments that do not exist are zero, and so is their product. table[self.rounds.len() * layout.live * ROW..layout.len() * ROW].fill(0); pads[self.rounds.len() * ROW..].fill(0); - // The GFNI rows need no wide scratch. - let gfni = cfg!(all( - target_arch = "x86_64", - target_feature = "avx512f", - target_feature = "avx512bw", - target_feature = "gfni" - )); - let scratch = || vec![0u64; if gfni { 0 } else { 6 * BITS }]; + // The byte-map rows need no wide scratch. + let maps = cfg!(all(target_arch = "x86_64", target_feature = "avx2")); + let scratch = || vec![0u64; if maps { 0 } else { 6 * BITS }]; // A block's rows: both groups', or the padding block's one. let build = |wide: &mut Vec, block: &[u64], rows: &mut [u64]| { let halves = rows.len() / ROW; cfg_select! { - all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx512bw", target_feature = "gfni") => { - // Each group's value by affine byte maps (`table_gfni`). + all(target_arch = "x86_64", target_feature = "avx2") => { + // Each group's value by byte maps (`table_maps`). let _ = wide; row_maps.rows(block, rows, halves); } _ => { // Both groups' unreduced sums, three words each and laid out by word, then reduced. for j in 0..BITS { - let sums = crate::kernels::fold_groups(&embed(words(block, j)), &fold); + let sums = crate::kernels::fold_groups(&crate::kernels::embed(words(block, j)), &fold); for (wide, sum) in wide.as_chunks_mut::<{ 3 * BITS }>().0[..halves].iter_mut().zip(&sums) { let sum: [u64; 3] = crate::basis::carry(sum); for (k, &word) in sum.iter().enumerate() { diff --git a/crates/experimental_prover/keccak_gkr/src/skip_clmul.rs b/crates/experimental_prover/keccak_gkr/src/skip_clmul.rs new file mode 100644 index 0000000000..92098dce4d --- /dev/null +++ b/crates/experimental_prover/keccak_gkr/src/skip_clmul.rs @@ -0,0 +1,226 @@ +//! The four bit-level rounds' sums over a block with carry-less products (x86 without GFNI): per +//! [`slot`], the polynomial [`skip_row`](crate::kernels::skip_row) sums over the block. +//! +//! A row's sum `sum over x of w[x] e[x + 1] e[x + 2]` is taken as `e[2] A + e[4] B + e[0] C`, with +//! the cofactors `A = w[0] e[1] + w[1] e[3]`, `B = w[2] e[3] + w[3] e[0]` and `C = w[4] e[1]`. +//! They are linear in the values, so they are multiplied out on the row's packs and extended like +//! them. Only a block's sums at `s` matter, so a cofactor is reduced modulo the minimal polynomial +//! of `s` to two words, and a slot takes six products of an embedding and a word where +//! [`skip_row`](crate::kernels::skip_row) takes fifteen of an embedding, a product and a weight's +//! word. + +use primitives::hash::positions::row_positions; + +use crate::clmul::{V, Vector}; +use crate::kernels::{GROUP, MIXED, PACKS, embed}; +use crate::power::PowerField; +use crate::witness::words; + +cfg_select! { + target_feature = "vpclmulqdq" => { + use core::arch::x86_64::*; + + /// Groups a register holds, a lane each. + const GROUPS: usize = 2; + + /// Packs `q` and `q + 1` of every group, a lane each. + #[inline(always)] + fn load_packs(e: &[u32; PACKS], q: usize) -> V { + // SAFETY: AVX2 is enabled at compile time (the module's cfg), and each load is of two + // packs of `e`. + unsafe { + let [low, high] = [0, 1].map(|g| _mm_loadl_epi64(e[GROUP * g + q..][..2].as_ptr().cast())); + _mm256_cvtepu32_epi64(_mm_unpacklo_epi64(low, high)) + } + } + + /// Where a sum's lanes hold output `i` of group `half`: a register's lanes are the + /// groups' outputs `i`. + const fn at(half: usize, i: usize) -> usize { + GROUPS * i + half + } + } + _ => { + /// Groups a register holds, a lane each. + const GROUPS: usize = 1; + + /// Packs `q` and `q + 1`. + #[inline(always)] + fn load_packs(e: &[u32; PACKS], q: usize) -> V { + // SAFETY: two packs of `e`. + unsafe { V::load_packs(e[q..][..2].as_ptr()) } + } + + /// Where a sum's lanes hold output `i` of group `half`: a group's outputs, then the + /// other's. + const fn at(half: usize, i: usize) -> usize { + MIXED * half + i + } + } +} +const _: () = assert!(GROUPS == V::N); + +/// The lanes of two words a slot's products read: `[e[2], e[4]]`, `[A0, B0]`, `[A1, B1]`, +/// `[C0, C1]` and `e[0]` in the low word, word `k` of a cofactor its coefficients from `64 k`. +const FIELDS: usize = 5; + +/// Per word of the cofactors, a block's sums of the products of the word, at [`at`] each output +/// of [`extend_group`](crate::kernels::extend_group) of each group. +type Acc = [[[u64; 2]; 2 * MIXED]; 2]; + +/// The pairs of lanes of a vector of four whose sums are its low mixed points, in the order of +/// [`extend_group`](crate::kernels::extend_group). +const LOW_MIXED: [(usize, usize); 4] = [(0, 1), (2, 3), (0, 2), (1, 3)]; + +/// [`skip_row`](crate::kernels::skip_row) over every row and slice of a block, a row at a time. +pub struct SkipBlock<'a> { + weights: &'a [[u64; 2]], + /// In every lane's low word: the high word of the Barrett constant of the minimal polynomial + /// of `s`, then the minimal polynomial's two words below its leading term. + reduction: [V; 3], + /// The embeddings of a row's positions. + e: [[u32; PACKS]; 5], + /// Per field, the nine vectors of four of [`extend_group`](crate::kernels::extend_group) of + /// [`GROUPS`] groups, a register each lane: the packs, then their sums. + vectors: [[[V; 4]; 9]; FIELDS], + /// The block's sums. + acc: Acc, +} + +impl<'a> SkipBlock<'a> { + /// For blocks summed at the position weights `weights`, as polynomials in the power basis of + /// `field`. + pub fn new(weights: &'a [[u64; 2]], field: &PowerField) -> Box { + Box::new(Self { + weights, + reduction: [field.barrett[1], field.modulus[0], field.modulus[1]].map(V::splat), + e: [[0; PACKS]; 5], + vectors: [[[V::zero(); 4]; 9]; FIELDS], + acc: [[[0; 2]; 2 * MIXED]; 2], + }) + } + + /// Sum a block. + pub fn run(&mut self, block: &[u64]) { + self.acc = [[[0; 2]; 2 * MIXED]; 2]; + for r in 0..5 { + for z in 0..64 { + let js = row_positions(r, z); + for (e, &j) in self.e.iter_mut().zip(&js) { + *e = embed(words(block, j)); + } + // Each lane a weight's two words. + let w = js.map(|j| { + let [low, high] = self.weights[j]; + V::splat(low).unpacklo(V::splat(high)) + }); + for pass in (0..2).step_by(GROUPS) { + self.cofactors(pass, w); + self.products(pass); + } + } + } + } + + /// The [`FIELDS`] of the packs of the groups from `pass`, at the row's weights `w`, and the + /// vectors their sums make. + #[inline(always)] + fn cofactors(&mut self, pass: usize, w: [V; 5]) { + for q in (0..GROUP).step_by(2) { + let e: [V; 5] = self.e.each_ref().map(|e| load_packs(e, GROUP * pass + q)); + // An embedding times a weight: per pack of the pair, the products of its words. + let product = |e: V, w: V| { + [ + [e.clmul::<0x00>(w), e.clmul::<0x10>(w)], + [e.clmul::<0x01>(w), e.clmul::<0x11>(w)], + ] + }; + let sum = |a: [[V; 2]; 2], b: [[V; 2]; 2]| -> [[V; 2]; 2] { + std::array::from_fn(|h| std::array::from_fn(|k| a[h][k].xor(b[h][k]))) + }; + let a = sum(product(e[1], w[0]), product(e[3], w[1])); + let b = sum(product(e[3], w[2]), product(e[0], w[3])); + let c = product(e[1], w[4]); + for h in 0..2 { + // Pack `q + h`'s embeddings in the low word. + let e = e.map(|e| if h == 0 { e } else { e.high() }); + let [a, b, c] = [a[h], b[h], c[h]].map(|x| reduce(x, self.reduction)); + let fields = [e[2].unpacklo(e[4]), a.unpacklo(b), a.high().unpacklo(b.high()), c, e[0]]; + for (vectors, field) in self.vectors.iter_mut().zip(fields) { + vectors[(q + h) / 4][(q + h) % 4] = field; + } + } + } + for v in &mut self.vectors { + for l in 0..4 { + let (v4, v5) = (v[0][l].xor(v[1][l]), v[2][l].xor(v[3][l])); + [v[4][l], v[5][l], v[6][l], v[7][l], v[8][l]] = + [v4, v5, v[0][l].xor(v[2][l]), v[1][l].xor(v[3][l]), v4.xor(v5)]; + } + } + } + + /// Adds the products of the groups from `pass` to their sums, the extension's low points + /// summed as the products read them. + #[inline(always)] + fn products(&mut self, pass: usize) { + let Self { vectors, acc, .. } = self; + for k in 0..9 { + let v = |l: usize| vectors.each_ref().map(|v| v[k][l]); + if k >= 4 { + for l in 0..4 { + accumulate(acc, pass, 4 * (k - 4) + l, v(l)); + } + } + for (j, (a, b)) in LOW_MIXED.into_iter().enumerate() { + let (a, b) = (v(a), v(b)); + accumulate(acc, pass, 20 + 4 * k + j, std::array::from_fn(|f| a[f].xor(b[f]))); + } + let all = vectors + .each_ref() + .map(|v| v[k][0].xor(v[k][1]).xor(v[k][2].xor(v[k][3]))); + accumulate(acc, pass, 56 + k, all); + } + } + + /// The sum at slot `i`: the products of word `k` of the cofactors from coefficient `64 k`. + pub fn get(&self, slot: usize) -> [u64; 3] { + let (half, i) = if slot < 128 { + (slot / 64, slot % 64) + } else { + (slot - 128, 64) + }; + let [s0, s1] = self.acc.each_ref().map(|acc| acc[at(half, i)]); + [s0[0], s0[1] ^ s1[0], s1[1]] + } +} + +/// A cofactor `low + high x^64`, `high`'s high word below 64 coefficients, modulo the minimal +/// polynomial of `s` by Barrett's reduction, `reduction` as [`SkipBlock`] holds it: its words in a +/// lane each. +#[inline(always)] +fn reduce([low, high]: [V; 2], [barrett, modulus_low, modulus_high]: [V; 3]) -> V { + // The quotient, from the cofactor's word 2 alone. + let top = high.high(); + let q = top.xor(top.clmul::<0x00>(barrett).high()); + // The low two words of the cofactor and of `q` times the modulus. + let shifted = |x: V| V::zero().unpacklo(x); + low.xor(shifted(high)) + .xor(q.clmul::<0x00>(modulus_low)) + .xor(shifted(q.clmul::<0x00>(modulus_high))) +} + +/// Adds the products of the [`FIELDS`] at output `i` of the groups from `pass` to their sums. +#[inline(always)] +fn accumulate(acc: &mut Acc, pass: usize, i: usize, [e, w0, w1, c, e0]: [V; FIELDS]) { + let sums = [ + e.clmul::<0x00>(w0).xor(e.clmul::<0x11>(w0)).xor(e0.clmul::<0x00>(c)), + e.clmul::<0x00>(w1).xor(e.clmul::<0x11>(w1)).xor(e0.clmul::<0x10>(c)), + ]; + for (acc, sum) in acc.iter_mut().zip(sums) { + // Lane `g` is group `pass + g`. + let acc = acc[at(pass, i)..].as_mut_ptr().cast::(); + // SAFETY: `GROUPS` lanes of `acc`. + unsafe { V::store(acc, V::load(acc).xor(sum)) }; + } +} diff --git a/crates/experimental_prover/keccak_gkr/src/skip_gfni.rs b/crates/experimental_prover/keccak_gkr/src/skip_gfni.rs index 1e41431080..4acd2c8ebd 100644 --- a/crates/experimental_prover/keccak_gkr/src/skip_gfni.rs +++ b/crates/experimental_prover/keccak_gkr/src/skip_gfni.rs @@ -369,8 +369,9 @@ pub struct SkipBlock { } impl SkipBlock { - /// For blocks summed at the position weights `weights`, as polynomials. - pub fn new(weights: &[[u64; 2]]) -> Box { + /// For blocks summed at the position weights `weights`, as polynomials. The sums are exact, + /// with no reduction modulo the minimal polynomial of the field's `s`. + pub fn new(weights: &[[u64; 2]], _field: &crate::power::PowerField) -> Box { let mut residues = vec![[splat(0); RESIDUES]; BITS / 8]; for (residues, weights) in residues.iter_mut().zip(weights[..BITS].as_chunks::<8>().0) { let bytes = transpose_bytes(weights.map(|w| [w[0], w[1], 0, 0])); diff --git a/crates/experimental_prover/keccak_gkr/src/table_gfni.rs b/crates/experimental_prover/keccak_gkr/src/table_gfni.rs deleted file mode 100644 index f07553ba2d..0000000000 --- a/crates/experimental_prover/keccak_gkr/src/table_gfni.rs +++ /dev/null @@ -1,243 +0,0 @@ -//! The table pass's rows by GFNI (x86-64 with AVX-512BW and GFNI): the same rows as embedding each -//! position's packs, weighting them by the folded challenges with carry-less products and reducing. -//! -//! A position's value in a group, `reduce(sum over packs q of embed(pack q) * fold[q])`, is -//! GF(2)-linear in the group's 256 instance bits, so it is a sum of one 8x8 bit matrix per (input -//! byte, output byte): byte `t` of the group's 32 bytes holds bits `8 (t % 2) ..` of pack `t / 2`, -//! and each of its bits maps to the reduced product of one unit embedding by that pack's weight. -//! A block is byte-transposed 64 positions at a time (register `b` = byte `b` of the 64 positions), -//! each group then costs `32 * 16` affine products per 64 positions, and the 16 output bytes are -//! transposed back to the row's words. -//! -//! The scheme is the table fill of hashmancer-keccak's GFNI kernels. - -use core::arch::x86_64::*; - -use primitives::field::gf2_64::mul_wide; -use verifier::keccak::BITS; - -use crate::kernels::GROUP; -use crate::power::PowerField; -use crate::rmfe::RMFE; -use crate::witness::BLOCK_WORDS; - -/// Bytes of a group of a position (16 packs of 16 bits), and of a reduced value. -const GROUP_BYTES: usize = 32; -const VALUE_BYTES: usize = 16; -/// Positions per register. -const REGISTER_POSITIONS: usize = 64; - -/// The affine maps of one layer's fold weights: `matrices[t][o]` sends byte `t` of a group to byte `o` -/// of its value. -pub struct RowMaps { - matrices: [[u64; VALUE_BYTES]; GROUP_BYTES], -} - -impl RowMaps { - /// From the fold weights `fold[k][q]` (word `k` of pack `q`'s weight, in the power basis). - pub fn new(field: &PowerField, fold: &[[u64; GROUP]; 2]) -> Self { - // The value of bit `i` of pack `q` alone, image `16 q + i`: its unit embedding times the pack's - // weight, reduced, a row of three-word products laid out by word as the carry-less path's. - const IMAGES: usize = 16 * GROUP; - let mut wide = vec![0u64; 3 * IMAGES]; - for q in 0..GROUP { - for (i, &e) in RMFE.emb.iter().enumerate() { - for (k, w) in fold.iter().enumerate() { - let x = mul_wide(u64::from(e), w[q]); - wide[k * IMAGES + 16 * q + i] ^= x as u64; - wide[(k + 1) * IMAGES + 16 * q + i] ^= (x >> 64) as u64; - } - } - } - let mut images = vec![0u64; 2 * IMAGES]; - field.reduce_row(&mut images, &wide, IMAGES); - let bit = |image: usize, n: usize| (images[(n / 64) * IMAGES + image] >> (n % 64)) as u8 & 1; - // An affine byte map reads output bit `k` off matrix byte `7 - k`: its bit `c` is bit `8 o + k` - // of the image of input bit `c`. - let matrices = std::array::from_fn(|t| { - std::array::from_fn(|o| { - (0..8).fold(0u64, |m, k| { - let row = (0..8).fold(0u8, |row, c| row | bit(16 * (t / 2) + 8 * (t % 2) + c, 8 * o + k) << c); - m | u64::from(row) << (8 * (7 - k)) - }) - }) - }); - Self { matrices } - } - - /// The rows of a block's first `halves` groups: row `h`, word `k` of position `j` at - /// `rows[h * 2 BITS + k * BITS + j]`. - pub fn rows(&self, block: &[u64], rows: &mut [u64], halves: usize) { - assert!(block.len() >= BITS * BLOCK_WORDS && halves <= 2 && rows.len() >= halves * 2 * BITS); - // SAFETY: AVX-512F/BW and GFNI are enabled at compile time (module cfg); every load reads - // positions inside `block` and every store writes words inside `rows` (asserted). - unsafe { - for g in 0..BITS / REGISTER_POSITIONS { - let y = transpose_position_bytes(block.as_ptr().add(REGISTER_POSITIONS * g * BLOCK_WORDS)); - for h in 0..halves { - let acc = self.accumulate(&y[GROUP_BYTES * h..][..GROUP_BYTES]); - for k in 0..2 { - let words = byte_planes_to_words(acc[8 * k..][..8].try_into().unwrap()); - let dst = rows.as_mut_ptr().add(h * 2 * BITS + k * BITS + REGISTER_POSITIONS * g); - for (r, w) in words.iter().enumerate() { - _mm512_storeu_si512(dst.add(8 * r).cast(), *w); - } - } - } - } - } - } - - /// Output byte `o` of the 64 positions: the sum over input bytes `t` of `matrices[t][o]` applied to `y[t]`. - #[inline] - #[target_feature(enable = "avx512f", enable = "avx512bw", enable = "gfni")] - fn accumulate(&self, y: &[__m512i]) -> [__m512i; VALUE_BYTES] { - let mut acc = [_mm512_setzero_si512(); VALUE_BYTES]; - for t in (0..GROUP_BYTES).step_by(2) { - let (x0, x1, m0, m1) = (y[t], y[t + 1], &self.matrices[t], &self.matrices[t + 1]); - for (o, a) in acc.iter_mut().enumerate() { - let g0 = _mm512_gf2p8affine_epi64_epi8::<0>(x0, _mm512_set1_epi64(m0[o] as i64)); - let g1 = _mm512_gf2p8affine_epi64_epi8::<0>(x1, _mm512_set1_epi64(m1[o] as i64)); - *a = _mm512_ternarylogic_epi64::<0x96>(*a, g0, g1); - } - } - acc - } -} - -/// 4x4 transpose of 128-bit lanes: `out[l]` lane `g` = `x[g]` lane `l`. -#[inline] -#[target_feature(enable = "avx512f")] -fn transpose_lanes(x: [__m512i; 4]) -> [__m512i; 4] { - let t0 = _mm512_shuffle_i64x2::<0b01_00_01_00>(x[0], x[1]); - let t1 = _mm512_shuffle_i64x2::<0b11_10_11_10>(x[0], x[1]); - let t2 = _mm512_shuffle_i64x2::<0b01_00_01_00>(x[2], x[3]); - let t3 = _mm512_shuffle_i64x2::<0b11_10_11_10>(x[2], x[3]); - [ - _mm512_shuffle_i64x2::<0b10_00_10_00>(t0, t2), - _mm512_shuffle_i64x2::<0b11_01_11_01>(t0, t2), - _mm512_shuffle_i64x2::<0b10_00_10_00>(t1, t3), - _mm512_shuffle_i64x2::<0b11_01_11_01>(t1, t3), - ] -} - -/// The 64 bytes of 64 consecutive positions (eight words each, at `p`), transposed: register `b` -/// byte `n` is byte `b` of position `n`. -/// -/// # Safety -/// `p` is valid for reading 64 positions; AVX-512F/BW. -#[inline] -#[target_feature(enable = "avx512f", enable = "avx512bw")] -unsafe fn transpose_position_bytes(p: *const u64) -> [__m512i; 64] { - let mut y = [[_mm512_setzero_si512(); 16]; 4]; - for (q, yq) in y.iter_mut().enumerate() { - for (n, v) in yq.iter_mut().enumerate() { - // SAFETY: position `16 q + n` lies inside the caller's 64. - *v = unsafe { _mm512_loadu_si512(p.add(BLOCK_WORDS * (16 * q + n)).cast()) }; - } - // yq[k] lane l byte n = byte 16 l + k of position 16 q + n. - primitives::bits::transpose_bytes16x16_per_lane512(yq); - } - let mut out = [_mm512_setzero_si512(); 64]; - for k in 0..16 { - // out[16 l + k] lane q = y[q][k] lane l. - for (l, v) in transpose_lanes([y[0][k], y[1][k], y[2][k], y[3][k]]) - .into_iter() - .enumerate() - { - out[16 * l + k] = v; - } - } - out -} - -/// Eight byte planes of 64 positions (plane `b` byte `n` = byte `b` of position `n`) as the -/// positions' words: register `r` holds positions `8 r .. 8 r + 8`. -#[inline] -#[target_feature(enable = "avx512f", enable = "avx512bw")] -fn byte_planes_to_words(a: &[__m512i; 8]) -> [__m512i; 8] { - // Per 128-bit lane `L` (positions 16 L ..): bytes, then 16-bit pairs, then 32-bit quads interleave. - let u: [__m512i; 8] = std::array::from_fn(|i| { - let (x, y) = (a[2 * (i / 2)], a[2 * (i / 2) + 1]); - if i % 2 == 0 { - _mm512_unpacklo_epi8(x, y) - } else { - _mm512_unpackhi_epi8(x, y) - } - }); - // u[2 m + s]: lane L, 16-bit element e = bytes of planes 2m, 2m + 1 for position 16 L + 8 s + e. - let v: [__m512i; 8] = std::array::from_fn(|i| { - let (s, half) = (i / 4, i % 4 / 2); - let (x, y) = (u[s], u[2 + s]); - let (x, y) = if i % 2 == 0 { (x, y) } else { (u[4 + s], u[6 + s]) }; - if half == 0 { - _mm512_unpacklo_epi16(x, y) - } else { - _mm512_unpackhi_epi16(x, y) - } - }); - // v[4 s + 2 h + c]: lane L, 32-bit element e = planes 4c .. 4c + 4 of position 16 L + 8 s + 4 h + e. - let w: [__m512i; 8] = std::array::from_fn(|i| { - let (j, hi) = (i / 2, i % 2); - let (s, h) = (j / 2, j % 2); - let (x, y) = (v[4 * s + 2 * h], v[4 * s + 2 * h + 1]); - if hi == 0 { - _mm512_unpacklo_epi32(x, y) - } else { - _mm512_unpackhi_epi32(x, y) - } - }); - // w[2 j + e2]: lane L, qword d = position 16 L + 4 j + 2 e2 + d, all eight planes: lane L of - // w[m] holds positions 16 L + 2 m, 16 L + 2 m + 1. - let lo = transpose_lanes([w[0], w[1], w[2], w[3]]); - let hi = transpose_lanes([w[4], w[5], w[6], w[7]]); - // lo[L] = positions 16 L .. 16 L + 8, hi[L] = 16 L + 8 .. 16 L + 16. - [lo[0], hi[0], lo[1], hi[1], lo[2], hi[2], lo[3], hi[3]] -} - -#[cfg(test)] -mod tests { - use primitives::test_rng::Rng; - - use super::*; - use crate::basis::carry; - use crate::kernels::{embed, fold_groups}; - use crate::power::Element; - use crate::witness::words; - - /// The rows the per-position path builds: embeddings weighted by carry-less products, reduced. - #[test] - fn rows_match_the_carry_less_path() { - let mut rng = Rng::new(0x7AB1E); - { - let field = PowerField::new(rng.ext()).expect("a generator"); - // The fold weights: field elements in the power basis, by word. - let elems: Vec = (0..GROUP).map(|_| field.to_poly(rng.ext())).collect(); - let fold: [[u64; GROUP]; 2] = std::array::from_fn(|k| std::array::from_fn(|q| elems[q][k])); - let block: Vec = (0..BITS * BLOCK_WORDS).map(|_| rng.next_u64()).collect(); - let mut want = vec![0u64; 2 * 2 * BITS]; - let mut wide = vec![0u64; 2 * 3 * BITS]; - for j in 0..BITS { - let sums = fold_groups(&embed(words(&block, j)), &fold); - for (h, sum) in sums.iter().enumerate() { - let sum: [u64; 3] = carry(sum); - for (k, &w) in sum.iter().enumerate() { - wide[h * 3 * BITS + k * BITS + j] = w; - } - } - } - for h in 0..2 { - field.reduce_row( - &mut want[h * 2 * BITS..][..2 * BITS], - &wide[h * 3 * BITS..][..3 * BITS], - BITS, - ); - } - for halves in [2, 1] { - let mut got = vec![0u64; 2 * 2 * BITS]; - RowMaps::new(&field, &fold).rows(&block, &mut got, halves); - assert_eq!(got[..halves * 2 * BITS], want[..halves * 2 * BITS], "{halves} halves"); - } - } - } -} diff --git a/crates/experimental_prover/keccak_gkr/src/table_maps.rs b/crates/experimental_prover/keccak_gkr/src/table_maps.rs new file mode 100644 index 0000000000..ab97a63000 --- /dev/null +++ b/crates/experimental_prover/keccak_gkr/src/table_maps.rs @@ -0,0 +1,422 @@ +//! The table pass's rows by byte maps (x86-64 with AVX2): the same rows as embedding each +//! position's packs, weighting them by the folded challenges with carry-less products and reducing. +//! +//! A position's value in a group, `reduce(sum over packs q of embed(pack q) * fold[q])`, is +//! GF(2)-linear in the group's 256 instance bits, so it is a sum of one 8x8 bit matrix per (input +//! byte, output byte): byte `t` of the group's 32 bytes holds bits `8 (t % 2) ..` of pack `t / 2`, +//! and each of its bits maps to the reduced product of one unit embedding by that pack's weight. +//! A block is byte-transposed a register of positions at a time (register `b` = byte `b` of the +//! register's positions), each group then costs `32 * 16` byte maps per register, and the 16 +//! output bytes are transposed back to the row's words. A byte map is one GFNI affine product, 512 +//! bits wide with AVX-512BW, or without GFNI two `vpshufb` nibble lookups. +//! +//! The scheme is the table fill of hashmancer-keccak's GFNI kernels. + +use core::arch::x86_64::*; + +use primitives::field::gf2_64::mul_wide; +use verifier::keccak::BITS; + +use crate::kernels::GROUP; +use crate::power::PowerField; +use crate::rmfe::RMFE; +use crate::witness::BLOCK_WORDS; + +/// Bytes of a group of a position (16 packs of 16 bits), and of a reduced value. +const GROUP_BYTES: usize = 32; +const VALUE_BYTES: usize = 16; + +/// The byte maps of one layer's fold weights: `matrices[t][o]` sends byte `t` of a group to byte `o` +/// of its value. +pub struct RowMaps { + matrices: [[Matrix; VALUE_BYTES]; GROUP_BYTES], +} + +impl RowMaps { + /// From the fold weights `fold[k][q]` (word `k` of pack `q`'s weight, in the power basis). + pub fn new(field: &PowerField, fold: &[[u64; GROUP]; 2]) -> Self { + // The value of bit `i` of pack `q` alone, image `16 q + i`: its unit embedding times the pack's + // weight, reduced, a row of three-word products laid out by word as the carry-less path's. + const IMAGES: usize = 16 * GROUP; + let mut wide = vec![0u64; 3 * IMAGES]; + for q in 0..GROUP { + for (i, &e) in RMFE.emb.iter().enumerate() { + for (k, w) in fold.iter().enumerate() { + let x = mul_wide(u64::from(e), w[q]); + wide[k * IMAGES + 16 * q + i] ^= x as u64; + wide[(k + 1) * IMAGES + 16 * q + i] ^= (x >> 64) as u64; + } + } + } + let mut images = vec![0u64; 2 * IMAGES]; + field.reduce_row(&mut images, &wide, IMAGES); + let bit = |image: usize, n: usize| (images[(n / 64) * IMAGES + image] >> (n % 64)) as u8 & 1; + // An affine byte map reads output bit `k` off matrix byte `7 - k`: its bit `c` is bit `8 o + k` + // of the image of input bit `c`. + let matrices = std::array::from_fn(|t| { + std::array::from_fn(|o| { + (0..8).fold(0u64, |m, k| { + let row = (0..8).fold(0u8, |row, c| row | bit(16 * (t / 2) + 8 * (t % 2) + c, 8 * o + k) << c); + m | u64::from(row) << (8 * (7 - k)) + }) + }) + }); + Self { + matrices: matrices.map(|m| m.map(matrix)), + } + } +} + +/// The three in-lane unpack rounds from eight byte planes `$a` (plane `b` byte `n` of a 128-bit lane +/// `L` = byte `b` of position `16 L + n`) to the positions' words: register `m` lane `L` holds positions +/// `16 L + 2 m` and `16 L + 2 m + 1`. +macro_rules! interleave_planes { + ($a:expr, $lo8:ident, $hi8:ident, $lo16:ident, $hi16:ident, $lo32:ident, $hi32:ident) => {{ + let a = $a; + // Per lane: bytes, then 16-bit pairs, then 32-bit quads interleave. + let u: [_; 8] = std::array::from_fn(|i| { + let (x, y) = (a[2 * (i / 2)], a[2 * (i / 2) + 1]); + if i % 2 == 0 { $lo8(x, y) } else { $hi8(x, y) } + }); + // u[2 m + s]: lane L, 16-bit element e = bytes of planes 2m, 2m + 1 for position 16 L + 8 s + e. + let v: [_; 8] = std::array::from_fn(|i| { + let (s, half) = (i / 4, i % 4 / 2); + let (x, y) = if i % 2 == 0 { + (u[s], u[2 + s]) + } else { + (u[4 + s], u[6 + s]) + }; + if half == 0 { $lo16(x, y) } else { $hi16(x, y) } + }); + // v[4 s + 2 h + c]: lane L, 32-bit element e = planes 4c .. 4c + 4 of position 16 L + 8 s + 4 h + e. + let w: [_; 8] = std::array::from_fn(|i| { + let (j, hi) = (i / 2, i % 2); + let (s, h) = (j / 2, j % 2); + let (x, y) = (v[4 * s + 2 * h], v[4 * s + 2 * h + 1]); + if hi == 0 { $lo32(x, y) } else { $hi32(x, y) } + }); + w + }}; +} + +cfg_select! { + all(target_feature = "avx512f", target_feature = "avx512bw", target_feature = "gfni") => { + /// Positions per register. + const REGISTER_POSITIONS: usize = 64; + + /// A matrix, broadcast by the affine product's memory operand. + type Matrix = u64; + + #[inline] + fn matrix(m: u64) -> Matrix { + m + } + + impl RowMaps { + /// The rows of a block's first `halves` groups: row `h`, word `k` of position `j` at + /// `rows[h * 2 BITS + k * BITS + j]`. + pub fn rows(&self, block: &[u64], rows: &mut [u64], halves: usize) { + assert!(block.len() >= BITS * BLOCK_WORDS && halves <= 2 && rows.len() >= halves * 2 * BITS); + // SAFETY: AVX-512F/BW and GFNI are enabled at compile time (module cfg); every load reads + // positions inside `block` and every store writes words inside `rows` (asserted). + unsafe { + for g in 0..BITS / REGISTER_POSITIONS { + let y = transpose_position_bytes(block.as_ptr().add(REGISTER_POSITIONS * g * BLOCK_WORDS)); + for h in 0..halves { + let acc = self.accumulate(&y[GROUP_BYTES * h..][..GROUP_BYTES]); + for k in 0..2 { + let words = byte_planes_to_words(acc[8 * k..][..8].try_into().unwrap()); + let dst = rows.as_mut_ptr().add(h * 2 * BITS + k * BITS + REGISTER_POSITIONS * g); + for (r, w) in words.iter().enumerate() { + _mm512_storeu_si512(dst.add(8 * r).cast(), *w); + } + } + } + } + } + } + + /// Output byte `o` of the 64 positions: the sum over input bytes `t` of `matrices[t][o]` applied to `y[t]`. + #[inline] + #[target_feature(enable = "avx512f", enable = "avx512bw", enable = "gfni")] + fn accumulate(&self, y: &[__m512i]) -> [__m512i; VALUE_BYTES] { + let mut acc = [_mm512_setzero_si512(); VALUE_BYTES]; + for t in (0..GROUP_BYTES).step_by(2) { + let (x0, x1, m0, m1) = (y[t], y[t + 1], &self.matrices[t], &self.matrices[t + 1]); + for (o, a) in acc.iter_mut().enumerate() { + let g0 = _mm512_gf2p8affine_epi64_epi8::<0>(x0, _mm512_set1_epi64(m0[o] as i64)); + let g1 = _mm512_gf2p8affine_epi64_epi8::<0>(x1, _mm512_set1_epi64(m1[o] as i64)); + *a = _mm512_ternarylogic_epi64::<0x96>(*a, g0, g1); + } + } + acc + } + } + + /// 4x4 transpose of 128-bit lanes: `out[l]` lane `g` = `x[g]` lane `l`. + #[inline] + #[target_feature(enable = "avx512f")] + fn transpose_lanes(x: [__m512i; 4]) -> [__m512i; 4] { + let t0 = _mm512_shuffle_i64x2::<0b01_00_01_00>(x[0], x[1]); + let t1 = _mm512_shuffle_i64x2::<0b11_10_11_10>(x[0], x[1]); + let t2 = _mm512_shuffle_i64x2::<0b01_00_01_00>(x[2], x[3]); + let t3 = _mm512_shuffle_i64x2::<0b11_10_11_10>(x[2], x[3]); + [ + _mm512_shuffle_i64x2::<0b10_00_10_00>(t0, t2), + _mm512_shuffle_i64x2::<0b11_01_11_01>(t0, t2), + _mm512_shuffle_i64x2::<0b10_00_10_00>(t1, t3), + _mm512_shuffle_i64x2::<0b11_01_11_01>(t1, t3), + ] + } + + /// The 64 bytes of 64 consecutive positions (eight words each, at `p`), transposed: register `b` + /// byte `n` is byte `b` of position `n`. + /// + /// # Safety + /// `p` is valid for reading 64 positions; AVX-512F/BW. + #[inline] + #[target_feature(enable = "avx512f", enable = "avx512bw")] + unsafe fn transpose_position_bytes(p: *const u64) -> [__m512i; 64] { + let mut y = [[_mm512_setzero_si512(); 16]; 4]; + for (q, yq) in y.iter_mut().enumerate() { + for (n, v) in yq.iter_mut().enumerate() { + // SAFETY: position `16 q + n` lies inside the caller's 64. + *v = unsafe { _mm512_loadu_si512(p.add(BLOCK_WORDS * (16 * q + n)).cast()) }; + } + // yq[k] lane l byte n = byte 16 l + k of position 16 q + n. + primitives::bits::transpose_bytes16x16_per_lane512(yq); + } + let mut out = [_mm512_setzero_si512(); 64]; + for k in 0..16 { + // out[16 l + k] lane q = y[q][k] lane l. + for (l, v) in transpose_lanes([y[0][k], y[1][k], y[2][k], y[3][k]]) + .into_iter() + .enumerate() + { + out[16 * l + k] = v; + } + } + out + } + + /// Eight byte planes of 64 positions (plane `b` byte `n` = byte `b` of position `n`) as the + /// positions' words: register `r` holds positions `8 r .. 8 r + 8`. + #[inline] + #[target_feature(enable = "avx512f", enable = "avx512bw")] + fn byte_planes_to_words(a: &[__m512i; 8]) -> [__m512i; 8] { + let w = interleave_planes!( + a, + _mm512_unpacklo_epi8, + _mm512_unpackhi_epi8, + _mm512_unpacklo_epi16, + _mm512_unpackhi_epi16, + _mm512_unpacklo_epi32, + _mm512_unpackhi_epi32 + ); + let lo = transpose_lanes([w[0], w[1], w[2], w[3]]); + let hi = transpose_lanes([w[4], w[5], w[6], w[7]]); + // lo[L] = positions 16 L .. 16 L + 8, hi[L] = 16 L + 8 .. 16 L + 16. + [lo[0], hi[0], lo[1], hi[1], lo[2], hi[2], lo[3], hi[3]] + } + } + _ => { + /// Positions per register. + const REGISTER_POSITIONS: usize = 32; + + cfg_select! { + target_feature = "gfni" => { + /// A matrix in every lane, so that the affine product reads it from memory. + type Matrix = __m256i; + + fn matrix(m: u64) -> Matrix { + // SAFETY: AVX is enabled at compile time (module cfg). + unsafe { _mm256_set1_epi64x(m as i64) } + } + + /// An input register as [`product`] reads it. + type Input = __m256i; + + #[inline(always)] + fn input(x: __m256i) -> Input { + x + } + + #[inline(always)] + fn product(x: Input, m: &Matrix) -> __m256i { + // SAFETY: GFNI and AVX are enabled at compile time (module cfg); registers only. + unsafe { _mm256_gf2p8affine_epi64_epi8::<0>(x, *m) } + } + } + _ => { + /// A matrix as two `vpshufb` tables: its images of the low nibbles, then of the high ones. + type Matrix = [u8; 32]; + + fn matrix(m: u64) -> Matrix { + // Output bit `k` is the parity of the input against matrix byte `7 - k`. + let image = |x: u8| (0..8).fold(0u8, |r, k| r | (((m >> (8 * (7 - k))) as u8 & x).count_ones() as u8 & 1) << k); + std::array::from_fn(|i| image(((i % 16) as u8) << (4 * (i / 16)))) + } + + /// An input register as [`product`] reads it: its low nibbles, then its high ones. + type Input = [__m256i; 2]; + + #[inline(always)] + fn input(x: __m256i) -> Input { + // SAFETY: AVX2 is enabled at compile time (module cfg); registers only. + unsafe { + let nibble = _mm256_set1_epi8(0x0F); + [_mm256_and_si256(x, nibble), _mm256_and_si256(_mm256_srli_epi16::<4>(x), nibble)] + } + } + + #[inline(always)] + fn product(x: Input, m: &Matrix) -> __m256i { + // SAFETY: AVX2 is enabled at compile time (module cfg), and each load reads 16 of the + // matrix's bytes. + unsafe { + let lo = _mm256_broadcastsi128_si256(_mm_loadu_si128(m.as_ptr().cast())); + let hi = _mm256_broadcastsi128_si256(_mm_loadu_si128(m.as_ptr().add(16).cast())); + _mm256_xor_si256(_mm256_shuffle_epi8(lo, x[0]), _mm256_shuffle_epi8(hi, x[1])) + } + } + } + } + + impl RowMaps { + /// The rows of a block's first `halves` groups: row `h`, word `k` of position `j` at + /// `rows[h * 2 BITS + k * BITS + j]`. + pub fn rows(&self, block: &[u64], rows: &mut [u64], halves: usize) { + assert!(block.len() >= BITS * BLOCK_WORDS && halves <= 2 && rows.len() >= halves * 2 * BITS); + // SAFETY: AVX2 is enabled at compile time (module cfg); every load reads + // positions inside `block` and every store writes words inside `rows` (asserted). + unsafe { + let mut y = [_mm256_setzero_si256(); GROUP_BYTES]; + for g in 0..BITS / REGISTER_POSITIONS { + for h in 0..halves { + let at = BLOCK_WORDS * REGISTER_POSITIONS * g + GROUP_BYTES / 8 * h; + transpose_group_bytes(block.as_ptr().add(at), &mut y); + for k in 0..2 { + let words = byte_planes_to_words(&self.accumulate(&y, k)); + let dst = rows.as_mut_ptr().add(h * 2 * BITS + k * BITS + REGISTER_POSITIONS * g); + for (r, w) in words.iter().enumerate() { + _mm256_storeu_si256(dst.add(4 * r).cast(), *w); + } + } + } + } + } + } + + /// Output bytes `8 k .. 8 k + 8` of the 32 positions, word `k` of their values: the sums over + /// input bytes `t` of `matrices[t][o]` applied to `y[t]`. Eight outputs at a time keep the sums + /// in registers. + #[inline] + #[target_feature(enable = "avx2")] + fn accumulate(&self, y: &[__m256i; GROUP_BYTES], k: usize) -> [__m256i; 8] { + let mut acc = [_mm256_setzero_si256(); 8]; + for (&x, m) in y.iter().zip(&self.matrices) { + let x = input(x); + for (a, m) in acc.iter_mut().zip(&m[8 * k..][..8]) { + *a = _mm256_xor_si256(*a, product(x, m)); + } + } + acc + } + } + + /// The 32 bytes of a group of 32 consecutive positions (eight words apart, from `p`), transposed: + /// `y[t]` byte `n` is byte `t` of the group of position `n`. + /// + /// # Safety + /// `p` is valid for reading the 32 positions' groups; AVX2. + #[inline] + #[target_feature(enable = "avx2")] + unsafe fn transpose_group_bytes(p: *const u64, y: &mut [__m256i; GROUP_BYTES]) { + for (l, y) in y.as_chunks_mut::<16>().0.iter_mut().enumerate() { + // Register `n`: bytes `16 l ..` of positions `n` and `16 + n`. + *y = std::array::from_fn(|n| { + // SAFETY: positions `n` and `16 + n` lie inside the caller's 32. + unsafe { + _mm256_loadu2_m128i( + p.add(BLOCK_WORDS * (16 + n) + 2 * l).cast(), + p.add(BLOCK_WORDS * n + 2 * l).cast(), + ) + } + }); + // y[k] lane i byte n = byte 16 l + k of position 16 i + n. + primitives::bits::transpose_bytes16x16_per_lane256(y); + } + } + + /// Eight byte planes of 32 positions (plane `b` byte `n` = byte `b` of position `n`) as the + /// positions' words: register `r` holds positions `4 r .. 4 r + 4`. + #[inline] + #[target_feature(enable = "avx2")] + fn byte_planes_to_words(a: &[__m256i; 8]) -> [__m256i; 8] { + let w = interleave_planes!( + a, + _mm256_unpacklo_epi8, + _mm256_unpackhi_epi8, + _mm256_unpacklo_epi16, + _mm256_unpackhi_epi16, + _mm256_unpacklo_epi32, + _mm256_unpackhi_epi32 + ); + std::array::from_fn(|r| { + let (x, y) = (w[2 * (r % 4)], w[2 * (r % 4) + 1]); + if r < 4 { + _mm256_permute2x128_si256::<0x20>(x, y) + } else { + _mm256_permute2x128_si256::<0x31>(x, y) + } + }) + } + } +} + +#[cfg(test)] +mod tests { + use primitives::test_rng::Rng; + + use super::*; + use crate::basis::carry; + use crate::kernels::{embed, fold_groups}; + use crate::power::Element; + use crate::witness::words; + + /// The rows the per-position path builds: embeddings weighted by carry-less products, reduced. + #[test] + fn rows_match_the_carry_less_path() { + let mut rng = Rng::new(0x7AB1E); + { + let field = PowerField::new(rng.ext()).expect("a generator"); + // The fold weights: field elements in the power basis, by word. + let elems: Vec = (0..GROUP).map(|_| field.to_poly(rng.ext())).collect(); + let fold: [[u64; GROUP]; 2] = std::array::from_fn(|k| std::array::from_fn(|q| elems[q][k])); + let block: Vec = (0..BITS * BLOCK_WORDS).map(|_| rng.next_u64()).collect(); + let mut want = vec![0u64; 2 * 2 * BITS]; + let mut wide = vec![0u64; 2 * 3 * BITS]; + for j in 0..BITS { + let sums = fold_groups(&embed(words(&block, j)), &fold); + for (h, sum) in sums.iter().enumerate() { + let sum: [u64; 3] = carry(sum); + for (k, &w) in sum.iter().enumerate() { + wide[h * 3 * BITS + k * BITS + j] = w; + } + } + } + for h in 0..2 { + field.reduce_row( + &mut want[h * 2 * BITS..][..2 * BITS], + &wide[h * 3 * BITS..][..3 * BITS], + BITS, + ); + } + for halves in [2, 1] { + let mut got = vec![0u64; 2 * 2 * BITS]; + RowMaps::new(&field, &fold).rows(&block, &mut got, halves); + assert_eq!(got[..halves * 2 * BITS], want[..halves * 2 * BITS], "{halves} halves"); + } + } + } +} diff --git a/crates/experimental_prover/lean_vm_core/src/gkr.rs b/crates/experimental_prover/lean_vm_core/src/gkr.rs index a5922c160c..1038404216 100644 --- a/crates/experimental_prover/lean_vm_core/src/gkr.rs +++ b/crates/experimental_prover/lean_vm_core/src/gkr.rs @@ -6,8 +6,9 @@ //! Leaves and every layer are `E`-valued (the bus fingerprints mix `K`-columns //! into `E` upstream, [`crate::leaf`]). //! -//! On AVX-512 the large rounds run eight pairs of rows at a time in transposed form -//! (`vertical`), and skip the runs of rows known to be the identity. +//! On x86 with AVX2 and a carry-less multiplier (VPCLMULQDQ with AVX-512) the large rounds run +//! eight pairs of rows at a time in transposed form (`vertical`), and skip the runs of rows known +//! to be the identity. use std::ops::Range; @@ -41,9 +42,19 @@ fn sum_tasks(tasks: usize, parallel: bool, f: impl Fn(usize) -> [F128; 4] + Sync /// `out[i] = ∏ current[4i..4i+4]`, for `out.len() == current.len() / 4`. pub(crate) fn products_into(current: &[F128], out: &mut [F128]) { debug_assert_eq!(out.len(), current.len() / 4); - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] let done = vertical::products_into(current, out); - #[cfg(not(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq")))] + #[cfg(not(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + )))] let done = 0; for (row, slot) in out.iter_mut().enumerate().skip(done) { let [left, right] = mul2( @@ -161,8 +172,13 @@ struct SplitEq { hi: Vec, lo_bits: usize, /// `lo` eight entries at a time, for the vertical kernels. - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] - lo8: Vec, + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] + lo8: Vec, } impl SplitEq { @@ -171,13 +187,13 @@ impl SplitEq { let (low, high) = vars.split_at(lo_bits); let lo = eq_table(low); Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] - lo8: lo - .as_chunks::<8>() - .0 - .iter() - .map(primitives::field::x8::F128x8::load) - .collect(), + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] + lo8: lo.as_chunks::<8>().0.iter().map(vertical::load_eight).collect(), lo, hi: eq_table(high), lo_bits, @@ -264,7 +280,12 @@ fn runs<'a>(range: Range, ones: &'a [Range]) -> impl Iterator, - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] spare: Vec, /// Runs of rows known to be the identity, sorted: the kernels skip them, and /// the vertical ones leave their groups unwritten. @@ -295,9 +326,19 @@ impl QuaternaryLayerState { Self { values, logical_rows: width, - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] groups: Vec::new(), - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] spare: Vec::new(), ones: Vec::new(), } @@ -305,7 +346,12 @@ impl QuaternaryLayerState { /// `(q(0)+q(1), [X²]q, [X³]q, [X⁴]q)`. fn round_message(&self, eq: &SplitEq) -> [F128; 4] { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] if self.logical_rows >= VERTICAL_ROWS { return vertical::message(&self.values, &self.ones, eq); } @@ -313,13 +359,23 @@ impl QuaternaryLayerState { } fn fold(&mut self, challenge: F128) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] self.leave_groups(); self.fold_scalar(challenge); } /// The vertical kernels' view of the rows, and the buffer the fold writes. - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] fn split(&mut self) -> (vertical::Input<'_>, &mut Vec) { let data = if self.groups.is_empty() { vertical::Data::Rows(&self.values) @@ -330,7 +386,12 @@ impl QuaternaryLayerState { } /// Move the rows back into `values` for the scalar kernels. - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] fn leave_groups(&mut self) { if !self.groups.is_empty() { self.values = vertical::to_rows(&self.groups, &self.ones); @@ -515,7 +576,12 @@ fn fold_and_message( challenge: F128, eq: &SplitEq, ) -> [[F128; 4]; N_BUS_TREES] { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] { if trees[0].logical_rows >= VERTICAL_ROWS { let [a, b, c] = trees.each_mut().map(QuaternaryLayerState::split); @@ -535,18 +601,43 @@ fn fold_and_message( trees.each_mut().map(|tree| tree.fold_and_message_scalar(challenge, eq)) } -/// The AVX-512 kernels: eight pairs of rows at a time, transposed so that every -/// product is lane-wise ([`primitives::field::x8`]). After its first fold a layer -/// keeps its rows transposed, in [`Group`]s, until it is small enough for the scalar -/// kernels. -#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] +/// The vector kernels: eight pairs of rows at a time, transposed so that every +/// product is lane-wise (`primitives::field::x8` on AVX-512, `f128x8`'s two halves +/// of `x4` lanes on AVX2). After its first fold a layer keeps its rows transposed, +/// in [`Group`]s, until it is small enough for the scalar kernels. +#[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) +))] mod vertical { use super::{N_BUS_TREES, SplitEq, add4, coarsen, sum_tasks}; use primitives::field::F128; - use primitives::field::x8::{F128x8, Sum8}; + // A message runs on `Lane`s: all eight lanes on AVX-512, and on AVX2 a four-lane half at a time, which + // keeps its live set within the sixteen YMM registers. + cfg_select! { + target_feature = "avx512f" => { + use primitives::field::x8::{F128x8 as Lane, Sum8 as Sum}; + pub(super) use primitives::field::x8::F128x8; + } + _ => { + use primitives::field::x4::{F128x4 as Lane, Sum4 as Sum}; + pub(super) use primitives::field::f128x8::F128x8; + } + } use primitives::stream::Stream; use std::ops::Range; + /// Eight consecutive elements. + #[inline(always)] + pub(super) fn load_eight(src: &[F128; 8]) -> F128x8 { + cfg_select! { + target_feature = "avx512f" => F128x8::load(src), + _ => F128x8::from_array(src), + } + } + /// Whether `ones` covers `rows`. fn covered(ones: &[Range], rows: Range) -> bool { let i = ones.partition_point(|r| r.start <= rows.start); @@ -573,10 +664,10 @@ mod vertical { } } - /// Add eight pairs' `eq`-weighted summands. The weight scales child 0's line, and - /// with it every product, so it costs two products rather than one per coefficient. + /// Add the `eq`-weighted summands of a group's pairs, in `Lane`s. The weight scales child 0's line, + /// and with it every product, so it costs two products rather than one per coefficient. #[inline(always)] - fn add_message(g: &Group, eq: F128x8, acc: &mut [Sum8; 4]) { + fn add_message(g: [Lane; 8], eq: Lane, acc: &mut [Sum; 4]) { let a0 = eq * g[0]; let d0 = eq * (g[0] + g[4]); let left0 = a0 * g[1]; @@ -601,7 +692,7 @@ mod vertical { struct Message<'a> { eq: &'a SplitEq, block: usize, - sums: [Sum8; 4], + sums: [Sum; 4], total: [F128; 4], } @@ -610,7 +701,7 @@ mod vertical { Self { eq, block: first >> (eq.lo_bits - 3), - sums: [Sum8::ZERO; 4], + sums: [Sum::ZERO; 4], total: [F128::ZERO; 4], } } @@ -622,11 +713,20 @@ mod vertical { self.flush(); self.block = g >> shift; } - add_message(group, self.eq.lo8[g & ((1 << shift) - 1)], &mut self.sums); + let eq = self.eq.lo8[g & ((1 << shift) - 1)]; + cfg_select! { + target_feature = "avx512f" => add_message(*group, eq, &mut self.sums), + _ => { + // One half after the other, unrolled: a loop over them runs slower. + let (e, h) = (eq.halves(), group.map(F128x8::halves)); + add_message(h.map(|lanes| lanes[0]), e[0], &mut self.sums); + add_message(h.map(|lanes| lanes[1]), e[1], &mut self.sums); + } + } } fn flush(&mut self) { - let sums = std::mem::replace(&mut self.sums, [Sum8::ZERO; 4]).map(Sum8::reduce); + let sums = std::mem::replace(&mut self.sums, [Sum::ZERO; 4]).map(Sum::reduce); let scaled = primitives::field::mul4([self.eq.hi[self.block]; 4], sums); self.total = add4(self.total, scaled); } @@ -658,7 +758,10 @@ mod vertical { let fold = |g: &Group, c: usize| g[c] + r * (g[c] + g[4 + c]); let mut out = [F128x8::splat(F128::ZERO); 8]; for c in 0..4 { - (out[c], out[4 + c]) = F128x8::deinterleave(fold(a, c), fold(b, c)); + (out[c], out[4 + c]) = cfg_select! { + target_feature = "avx512f" => F128x8::deinterleave(fold(a, c), fold(b, c)), + _ => F128x8::even_odd(fold(a, c), fold(b, c)), + }; } out } @@ -775,10 +878,16 @@ mod vertical { let (chunks, _) = out.as_chunks_mut::<16>(); for (slot, run) in chunks.iter_mut().zip(current.as_chunks::<64>().0) { let g = F128x8::load_runs(run); - let (lo, hi) = F128x8::interleave(product(g[0], g[1], g[2], g[3]), product(g[4], g[5], g[6], g[7])); - let (first, second) = slot.split_at_mut(8); - lo.store(first.try_into().unwrap()); - hi.store(second.try_into().unwrap()); + let (a, b) = (product(g[0], g[1], g[2], g[3]), product(g[4], g[5], g[6], g[7])); + cfg_select! { + target_feature = "avx512f" => { + let (lo, hi) = F128x8::interleave(a, b); + let (first, second) = slot.split_at_mut(8); + lo.store(first.try_into().unwrap()); + hi.store(second.try_into().unwrap()); + } + _ => F128x8::store_interleaved(a, b, slot), + } } 16 * chunks.len() } diff --git a/crates/experimental_prover/lean_vm_core/src/leaf.rs b/crates/experimental_prover/lean_vm_core/src/leaf.rs index 0c54a64a1c..cfaef01e12 100644 --- a/crates/experimental_prover/lean_vm_core/src/leaf.rs +++ b/crates/experimental_prover/lean_vm_core/src/leaf.rs @@ -219,9 +219,19 @@ impl LeafBlock { let (real_out, padding) = out.split_at_mut(real); // A padding row's leaf is the identity, so it leaves the product alone. padding.fill(F128::ONE); - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] let done = self.fill_lanes(z0, real_out, cols); - #[cfg(not(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq")))] + #[cfg(not(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + )))] let done = 0; let mut index = g_pow(z0 + done); for (i, slot) in real_out.iter_mut().enumerate().skip(done) { @@ -233,7 +243,12 @@ impl LeafBlock { /// [`fill`](Self::fill) [`lanes::N`] rows at a time, every term one lane-wise mixed /// product; returns how many rows it wrote. - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] fn fill_lanes(&self, z0: usize, out: &mut [F128], cols: &[&[F64]]) -> usize { use lanes::{E, K, Mixed, N}; let load = |source: &[F64], z: usize| K::load(source[z..z + N].try_into().unwrap()); @@ -271,7 +286,7 @@ mod lanes { #[cfg(all( target_arch = "x86_64", target_feature = "avx2", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", not(target_feature = "avx512f") ))] mod lanes { diff --git a/crates/experimental_prover/pcs/src/merkle.rs b/crates/experimental_prover/pcs/src/merkle.rs index 9af7b2ee28..2108ec0e47 100644 --- a/crates/experimental_prover/pcs/src/merkle.rs +++ b/crates/experimental_prover/pcs/src/merkle.rs @@ -38,7 +38,7 @@ const STAGE_TILE_BYTES: usize = 16 << 10; /// A Merkle tree filled one aligned block of leaves at a time, from any thread. /// -/// Leaf `i` is `zeros(leaf_words - row_words) ‖ row_i`. +/// Leaf `i` is `zeros(leaf_words - row_words) ‖ row_i`, and a row may lead with zero words of its own. /// /// Blocks must all have the same power-of-two size, and together cover every leaf once. pub(crate) struct MerkleBuilder { @@ -50,20 +50,21 @@ pub(crate) struct MerkleBuilder { } impl MerkleBuilder { - /// An empty tree of `num_leaves` leaves of `leaf_words`, each committing `row_words`. + /// An empty tree of `num_leaves` leaves of `leaf_words`, each committing `row_words`, of which all but the last + /// `data_words` are zero. /// /// A tree of `CUT_FROM` leaves or more is cut, so its blocks must hold whole leaf groups. - pub fn new(num_leaves: usize, row_words: usize, leaf_words: usize) -> Self { + pub fn new(num_leaves: usize, row_words: usize, data_words: usize, leaf_words: usize) -> Self { let cut = if num_leaves >= CUT_FROM { CUT } else { 0 }; - Self::with_bytes(num_leaves, 8 * row_words, 8 * leaf_words, cut) + Self::with_bytes(num_leaves, 8 * row_words, 8 * data_words, 8 * leaf_words, cut) } - fn with_bytes(num_leaves: usize, row_bytes: usize, leaf_bytes: usize, cut: usize) -> Self { + fn with_bytes(num_leaves: usize, row_bytes: usize, data_bytes: usize, leaf_bytes: usize, cut: usize) -> Self { assert!(num_leaves.is_power_of_two(), "num_leaves must be power of 2"); Self { nodes: Nodes::new(num_leaves, cut), cut, - leaves: LeafHasher::new(row_bytes, leaf_bytes), + leaves: LeafHasher::new(row_bytes, data_bytes, leaf_bytes), progress: Progress::new(num_leaves), } } @@ -341,12 +342,13 @@ impl CutTree { /// The paths of `positions`, with `row(q)` the row at `q`. /// - /// `group_rows` are the full rows of [`Self::group_positions`], `row_words` each, back to back, hashed - /// as the leaves `zeros(leaf_words - row_words) ‖ row`. + /// `group_rows` are the full rows of [`Self::group_positions`], `row_words` each, all but the last `data_words` + /// of each zero, back to back, hashed as the leaves `zeros(leaf_words - row_words) ‖ row`. pub fn open<'a>( &self, positions: &[usize], row_words: usize, + data_words: usize, leaf_words: usize, group_rows: &[F64], row: impl Fn(usize) -> &'a [F64], @@ -362,7 +364,8 @@ impl CutTree { let mut nodes = Nodes::new(n, 0); // SAFETY: the nodes are ours alone, the climb writes every level it reads, and the levels kept are written. unsafe { - LeafHasher::new(8 * row_words, 8 * leaf_words).hash(cast_slice(group_rows), nodes.level_mut(0, 0, n)); + LeafHasher::new(8 * row_words, 8 * data_words, 8 * leaf_words) + .hash(cast_slice(group_rows), nodes.level_mut(0, 0, n)); nodes.climb(0, 0, n, self.cut); nodes.tree.truncate(Self::start(n, self.cut)); nodes.assume_init() @@ -385,14 +388,15 @@ impl CutTree { /// How rows become leaf digests, chosen once from the leaf shape. /// -/// Whole blocks of leading zeros are one sponge state, absorbed once for every leaf. +/// Whole blocks of leading zeros, the row's own included, are one sponge state, absorbed once for every leaf. enum LeafHasher { /// Past the shared prefix, each row is its own image. Direct { row_bytes: usize, state: [u64; STATE_LANES], }, - /// The rest of the padding does not fill a block, so rows are zero-extended in a tile first. + /// The rest of the padding does not fill a block, or the row leads with more zeros than the image, so each row's + /// image is laid out in a tile first. Staged { row_bytes: usize, image: usize, @@ -401,9 +405,11 @@ enum LeafHasher { } impl LeafHasher { - fn new(row_bytes: usize, leaf_bytes: usize) -> Self { - assert!(0 < row_bytes && row_bytes <= leaf_bytes, "a leaf holds its row"); - let zero_blocks = (leaf_bytes - row_bytes) / RATE; + /// Rows of `row_bytes`, zero but for their last `data_bytes`, hashed as leaves of `leaf_bytes`. + fn new(row_bytes: usize, data_bytes: usize, leaf_bytes: usize) -> Self { + assert!(0 < data_bytes && data_bytes <= row_bytes, "a row holds its data"); + assert!(row_bytes <= leaf_bytes, "a leaf holds its row"); + let zero_blocks = (leaf_bytes - data_bytes) / RATE; let state = zero_prefix_state(zero_blocks); let image = leaf_bytes - zero_blocks * RATE; if image == row_bytes { @@ -460,14 +466,16 @@ impl Tile { Self([0; STAGE_TILE_BYTES / 8]) } - /// Lay each `row_bytes` row at the end of its own `image`-byte slot, and return the slots. + /// Lay each `row_bytes` row, cut to its last `image` bytes when longer (the rest is zeros), at the end of its own + /// `image`-byte slot, and return the slots. /// - /// The bytes before each row are never written, so they stay zero. + /// A slot's bytes before its row are never written, so they stay zero. fn extend(&mut self, rows: &[u8], row_bytes: usize, image: usize) -> &[u8] { let tile: &mut [u8] = cast_slice_mut(&mut self.0); let n = rows.len() / row_bytes; + let take = row_bytes.min(image); for (slot, row) in tile.chunks_exact_mut(image).zip(rows.chunks_exact(row_bytes)) { - slot[image - row_bytes..].copy_from_slice(row); + slot[image - take..].copy_from_slice(&row[row_bytes - take..]); } &tile[..n * image] } @@ -486,15 +494,22 @@ mod tests { /// The tree over `num_leaves` equal byte leaves, each hashed as [`primitives::hash::hash`] of its bytes. fn merkle_tree(data: &[u8], num_leaves: usize) -> Vec { let leaf_bytes = data.len() / num_leaves; - let builder = MerkleBuilder::with_bytes(num_leaves, leaf_bytes, leaf_bytes, 0); + let builder = MerkleBuilder::with_bytes(num_leaves, leaf_bytes, leaf_bytes, leaf_bytes, 0); builder.absorb_all(data); builder.finish().nodes } - /// The tree over `num_leaves` rows of `row_words`, each hashed as `zeros(leaf_words - row_words) ‖ row`. - fn merkle_tree_padded_rows(data: &[F64], num_leaves: usize, row_words: usize, leaf_words: usize) -> Vec { + /// The tree over `num_leaves` rows of `row_words`, all but the last `data_words` of each zero, each hashed as + /// `zeros(leaf_words - row_words) ‖ row`. + fn merkle_tree_padded_rows( + data: &[F64], + num_leaves: usize, + row_words: usize, + data_words: usize, + leaf_words: usize, + ) -> Vec { assert_eq!(data.len(), row_words * num_leaves); - let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * leaf_words, 0); + let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * data_words, 8 * leaf_words, 0); builder.absorb_all(cast_slice(data)); builder.finish().nodes } @@ -549,26 +564,37 @@ mod tests { #[test] fn padded_rows_tree_matches_full_width() { // Invariant: each leaf hashes as its zero-extended row, however the prefix splits. - for (num_leaves, row_words, leaf_words) in [ + for (num_leaves, row_words, data_words, leaf_words) in [ // Nothing to pad. - (8usize, 64usize, 64usize), - (1024, 8, 8), + (8usize, 64usize, 64usize, 64usize), + (1024, 8, 8, 8), // A shared prefix plus a partial block, staged. - (2048, 37, 64), - (256, 27, 64), - (64, 1, 64), + (2048, 37, 37, 64), + (256, 27, 27, 64), + (64, 1, 1, 64), // A prefix of whole blocks: the rows are hashed where they lie. - (256, 30, 64), - (1 << 14, 47, 64), + (256, 30, 30, 64), + (1 << 14, 47, 47, 64), // A prefix shorter than one block: nothing shared, still staged. - (32, 5, 8), - (256, 61, 64), + (32, 5, 5, 8), + (256, 61, 61, 64), // Leaves that are not a whole number of blocks. - (16, 3, 5), - (64, 7, 40), + (16, 3, 3, 5), + (64, 7, 7, 40), + // Rows that lead with zeros: a shared prefix that reaches into them, or stops short of them. + (1 << 10, 48, 46, 64), + (256, 32, 30, 64), + (256, 44, 42, 64), + (64, 8, 5, 8), ] { let data: Vec = (0..row_words * num_leaves) - .map(|i| F64(i.wrapping_mul(0x9E37_79B9_7F4A_7C15) as u64 | 1)) + .map(|i| { + if i % row_words < row_words - data_words { + F64::ZERO + } else { + F64(i.wrapping_mul(0x9E37_79B9_7F4A_7C15) as u64 | 1) + } + }) .collect(); let want = merkle_tree_sequential( data.chunks(row_words).map(|row| { @@ -579,9 +605,9 @@ mod tests { num_leaves, ); assert_eq!( - &merkle_tree_padded_rows(&data, num_leaves, row_words, leaf_words)[..], + &merkle_tree_padded_rows(&data, num_leaves, row_words, data_words, leaf_words)[..], &want[..], - "num_leaves={num_leaves} row_words={row_words} leaf_words={leaf_words}" + "num_leaves={num_leaves} row_words={row_words} data_words={data_words} leaf_words={leaf_words}" ); } } @@ -589,13 +615,24 @@ mod tests { #[test] fn cut_tree_opens_like_the_full_tree() { // Invariant: opening through the cut tree gives the paths of the full tree. - for (num_leaves, row_words, leaf_words) in [(256usize, 8usize, 16usize), (1 << 10, 24, 24), (32, 8, 8)] { + for (num_leaves, row_words, data_words, leaf_words) in [ + (256usize, 8usize, 8usize, 16usize), + (1 << 10, 24, 24, 24), + (32, 8, 8, 8), + (256, 48, 45, 64), + ] { let data: Vec = (0..row_words * num_leaves) - .map(|i| F64(i as u64 * 0x9E37_79B9 + 1)) + .map(|i| { + if i % row_words < row_words - data_words { + F64::ZERO + } else { + F64(i as u64 * 0x9E37_79B9 + 1) + } + }) .collect(); - let full = merkle_tree_padded_rows(&data, num_leaves, row_words, leaf_words); + let full = merkle_tree_padded_rows(&data, num_leaves, row_words, data_words, leaf_words); let row = |q: usize| &data[q * row_words..][..row_words]; - let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * leaf_words, CUT); + let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * data_words, 8 * leaf_words, CUT); builder.absorb_all(cast_slice(&data)); let cut = builder.finish(); assert_eq!(cut.root(), *full.last().unwrap()); @@ -606,7 +643,7 @@ mod tests { ] { let group = cut.group_positions(&positions); let rows: Vec = group.iter().flat_map(|&q| row(q)).copied().collect(); - let got = cut.open(&positions, row_words, leaf_words, &rows, row); + let got = cut.open(&positions, row_words, data_words, leaf_words, &rows, row); let level_start = |level: usize| 2 * num_leaves - ((2 * num_leaves) >> level); let node = |level: usize, i: usize| full[level_start(level) + i]; let row = |q| row(q).iter().map(|word| word.0); @@ -623,9 +660,9 @@ mod tests { // Invariant: the tree does not depend on the block size the encode picks. let (num_leaves, row_words) = (1usize << 12, 6usize); let data: Vec = (0..row_words * num_leaves).map(|i| F64(i as u64 * 7 + 1)).collect(); - let want = merkle_tree_padded_rows(&data, num_leaves, row_words, row_words); + let want = merkle_tree_padded_rows(&data, num_leaves, row_words, row_words, row_words); for log_block in [0usize, 3, 5, 6, 9, 12] { - let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * row_words, 0); + let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * row_words, 8 * row_words, 0); // Blocks arrive out of order, as the encode's tasks finish. for b in (0..num_leaves >> log_block).rev() { let rows = row_words << log_block; @@ -635,7 +672,7 @@ mod tests { } // A cut tree is the full tree without its lowest levels, whatever the block size. for log_block in [4usize, 5, 9, 12] { - let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * row_words, CUT); + let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * row_words, 8 * row_words, CUT); for b in (0..num_leaves >> log_block).rev() { let rows = row_words << log_block; builder.absorb(b << log_block, &data[b * rows..][..rows]); @@ -652,7 +689,7 @@ mod tests { #[should_panic(expected = "every leaf absorbed")] fn a_missing_block_is_rejected() { let data = vec![F64::ZERO; 8 * 16]; - let builder = MerkleBuilder::new(16, 8, 8); + let builder = MerkleBuilder::new(16, 8, 8, 8); builder.absorb(0, &data[..64]); let _ = builder.finish(); } @@ -661,7 +698,7 @@ mod tests { #[should_panic(expected = "a block absorbed twice")] fn a_repeated_block_is_rejected() { let data = vec![F64::ZERO; 8 * 16]; - let builder = MerkleBuilder::new(16, 8, 8); + let builder = MerkleBuilder::new(16, 8, 8, 8); builder.absorb(8, &data[..64]); builder.absorb(8, &data[..64]); } diff --git a/crates/experimental_prover/pcs/src/ntt.rs b/crates/experimental_prover/pcs/src/ntt.rs index 23b9c83907..b2d322e246 100644 --- a/crates/experimental_prover/pcs/src/ntt.rs +++ b/crates/experimental_prover/pcs/src/ntt.rs @@ -69,5 +69,5 @@ mod twiddles; mod window; pub(crate) use encode::{Codeword, Keep}; -pub(crate) use kernels::transpose_butterfly_lanes; +pub(crate) use kernels::{ROW_LANES, transpose_butterfly_lanes}; pub use twiddles::AdditiveNttF64; diff --git a/crates/experimental_prover/pcs/src/ntt/gathered.rs b/crates/experimental_prover/pcs/src/ntt/gathered.rs index 3ab5e7b70f..c7863fa34d 100644 --- a/crates/experimental_prover/pcs/src/ntt/gathered.rs +++ b/crates/experimental_prover/pcs/src/ntt/gathered.rs @@ -6,7 +6,7 @@ use primitives::stream::Stream; use super::plan::Plan; use super::scratch::{Line, with_scratch}; use super::source::Source; -use super::stage::{BATCH, Stage}; +use super::stage::{BATCH, Stage, lane_groups}; use super::twiddles::AdditiveNttF64; use super::window::Window; @@ -47,9 +47,10 @@ use super::window::Window; /// /// - Word `w` of message row `j` is word `j` of lane block `num_ntts - 1 - w`, and zero below the padding. /// - A lane block's cache line holds [`BATCH`] consecutive rows, so a task takes that many residues. -/// - It goes one octet of lanes at a time, and the octet of the residues is 64 lanes of one transform. -/// - It transposes those lanes into a stage of `2^g` rows of 64 words, one line per residue. -/// - Each block transforms a copy of the stage and scatters it line by line. +/// - It goes one group of lanes at a time, whole octets and then a quad where the row ends in one ([`lane_groups`]). +/// - With the residues, a group of `lanes` lanes is `BATCH * lanes` lanes of one transform. +/// - It transposes those lanes into a stage of `2^g` rows of `BATCH * lanes` words, `lanes` words per residue. +/// - Each block transforms a copy of the stage and scatters it residue by residue. pub(super) struct GatheredPass<'a> { src: Source<'a>, /// The transpose of a lane-major message, if the source is one. @@ -63,13 +64,14 @@ pub(super) struct GatheredPass<'a> { /// Rows of a layer-`layer` block, and the rows between the rows of a group. block_rows: usize, step: usize, - /// Words of a group's row, and of the group. + /// Words of a group's row, and of the group: for a lane-major message, those of an octet's. width: usize, group_len: usize, /// Residues one task step takes, and the words of its stage. batch: usize, stage_len: usize, - /// How a finished group drains: in pieces of `piece` words, `piece_stride` apart. + /// How a finished group drains: in pieces of `piece` words, `piece_stride` apart. A lane-major group's pieces + /// are its lanes of each residue. piece: usize, piece_stride: usize, stream: bool, @@ -85,7 +87,7 @@ impl<'a> GatheredPass<'a> { // The blocks the buffer holds. let first_block = win.first_row() / block_rows; let blocks = first_block..first_block + (win.len() / num_ntts) / block_rows; - // A lane-major group is 64 lanes of a batch of residues, transposed into a stage: one line per residue in each row. + // A lane-major group is at most an octet of lanes of a batch of residues, transposed into a stage. let staged = matches!(src, Source::Lanes(_)); let batch = if staged { BATCH } else { 1 }; let width = if staged { BATCH * 8 } else { num_ntts }; @@ -135,11 +137,14 @@ impl<'a> GatheredPass<'a> { match &self.staging { Some(staging) => { let r = t * self.batch; - for octet in 0..self.num_ntts / 8 { - staging.fill(stage, r, octet); + for (lane0, lanes) in lane_groups(self.num_ntts) { + let width = BATCH * lanes; + let stage = &mut stage[..width << self.g]; + staging.fill(stage, r, lane0, lanes); for block in self.blocks.clone().rev() { - let dst = win.row(self.block_rows, block, r) + 8 * octet; - self.group(ntt, scratch, stream, win, base, block, r, Some(&*stage), dst); + let dst = win.row(self.block_rows, block, r) + lane0; + let (from, piece) = (Some(&*stage), lanes); + self.group(ntt, scratch, stream, win, base, block, r, from, dst, width, piece); } } } @@ -151,7 +156,9 @@ impl<'a> GatheredPass<'a> { for block in self.blocks.clone().rev() { let dst = win.row(self.block_rows, block, t); let from = msg.map(|_| &*stage); - self.group(ntt, scratch, stream, win, base, block, t, from, dst); + self.group( + ntt, scratch, stream, win, base, block, t, from, dst, self.width, self.piece, + ); } } } @@ -185,7 +192,8 @@ impl<'a> GatheredPass<'a> { /// Transform residue `r`'s group as layer-`layer` block `block`, and send it back to word `dst` on. /// - /// The rows are `from` the stage, or gathered from the block's own rows of the buffer. + /// The rows, `width` words each, are `from` the stage, or gathered from the block's own rows of the buffer. + /// They drain in pieces of `piece` words. #[inline(always)] fn group( &self, @@ -198,8 +206,11 @@ impl<'a> GatheredPass<'a> { r: usize, from: Option<&[F64]>, dst: usize, + width: usize, + piece: usize, ) { let (layer, g, num_ntts, step) = (self.layer, self.g, self.num_ntts, self.step); + let scratch = &mut scratch[..width << g]; if from.is_none() { // Replicas: each block transforms its own rows of the buffer. self.gather( @@ -207,11 +218,11 @@ impl<'a> GatheredPass<'a> { base.0.cast_const().wrapping_add(win.row(self.block_rows, block, r)), ); } - ntt.run_layers(scratch, from, layer + g, self.width, layer, layer, block, None); + ntt.run_layers(scratch, from, layer + g, width, layer, layer, block, None); // Write the group back: scratch row `i` lands `i * step` rows after word `dst`, in pieces of `piece` words // `piece_stride` apart. A streaming write moves whole aligned cache lines: `piece` is then 8. - for (i, row) in scratch.chunks_exact(self.width).enumerate() { - for (k, piece) in row.chunks_exact(self.piece).enumerate() { + for (i, row) in scratch.chunks_exact(width).enumerate() { + for (k, piece) in row.chunks_exact(piece).enumerate() { // SAFETY: // - The group lies inside the buffer, so every piece is in bounds, and line-aligned when streamed. // - Only this group's task writes these rows of the buffer. @@ -219,7 +230,7 @@ impl<'a> GatheredPass<'a> { let at = base.add(dst + i * step * num_ntts + k * self.piece_stride); match stream { Some(s) => s.copy_lines(at.cast::(), &primitives::cast_slice::(piece)[0]), - None => at.copy_from_nonoverlapping(piece.as_ptr(), self.piece), + None => at.copy_from_nonoverlapping(piece.as_ptr(), piece.len()), } } } diff --git a/crates/experimental_prover/pcs/src/ntt/kernels.rs b/crates/experimental_prover/pcs/src/ntt/kernels.rs index ebce4cfcf1..827d9a40aa 100644 --- a/crates/experimental_prover/pcs/src/ntt/kernels.rs +++ b/crates/experimental_prover/pcs/src/ntt/kernels.rs @@ -4,6 +4,17 @@ use core::arch::x86_64::__m512i; use primitives::field::F64; +/// The lanes a codeword row is padded to a multiple of, with zero lanes: whole vectors of the kernels, or quads. +/// +/// - AVX2 runs four lanes to a vector, and a last chunk of one vector multiplies only it. +/// - AVX-512 runs eight, but masks its last vector to the row: a narrower row still saves memory traffic and the +/// Merkle leaf's hashing, which absorbs whole blocks of leading zero lanes as one shared state. +/// - NEON runs eight, and the rest word by word. +#[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq", target_feature = "avx2"))] +pub(crate) const ROW_LANES: usize = 4; +#[cfg(not(all(target_arch = "x86_64", target_feature = "pclmulqdq", target_feature = "avx2")))] +pub(crate) const ROW_LANES: usize = 8; + /// Visit every row group of a fused multi-layer block. /// /// ```text @@ -130,13 +141,14 @@ pub(super) fn butterfly_interleaved_fused_3layer( } #[cfg(all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ))] { #[inline] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + #[target_feature(enable = "pclmulqdq", enable = "avx2")] + #[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] fn sweep(block: &mut [F64], src: Option<&[F64]>, t: &[F64; 7], eighth: usize, num_ntts: usize) { avx2::fused_sweep::<8, 7>( block, @@ -145,24 +157,24 @@ pub(super) fn butterfly_interleaved_fused_3layer( num_ntts, t, #[inline(always)] - |v, t| { + |v, t, vectors| { radix8_schedule( #[inline(always)] - |top, bot, k| avx2::butterfly_regs(v, top, bot, t[k]), + |top, bot, k| avx2::butterfly_regs(v, top, bot, t[k], vectors), ); }, ); } - // SAFETY: this code is compiled only where VPCLMULQDQ and AVX2 are enabled. + // SAFETY: this code is compiled only where PCLMULQDQ and AVX2 are enabled, with VPCLMULQDQ when it is. unsafe { sweep(block, src, t, eighth, num_ntts) } } #[cfg(not(any( all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ) )))] fused_rows::<8>(block, src, eighth, num_ntts, |src, rows| { @@ -198,9 +210,9 @@ fn fused_sweep_avx512( all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ) )))] #[inline(always)] @@ -214,9 +226,9 @@ fn radix8_butterflies(src: Option<[&[F64]; 8]>, rows: &mut [&mut [F64]; 8], t: & all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ) )))] #[inline(always)] @@ -230,9 +242,9 @@ fn butterfly_rows(rows: &mut [&mut [F64]; N], top: usize, bot: u all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ) )))] #[inline(always)] @@ -278,13 +290,14 @@ pub(super) fn butterfly_interleaved_fused_2layer( } #[cfg(all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ))] { #[inline] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + #[target_feature(enable = "pclmulqdq", enable = "avx2")] + #[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] fn sweep(block: &mut [F64], src: Option<&[F64]>, t: &[F64; 3], quarter: usize, num_ntts: usize) { avx2::fused_sweep::<4, 3>( block, @@ -293,24 +306,24 @@ pub(super) fn butterfly_interleaved_fused_2layer( num_ntts, t, #[inline(always)] - |v, t| { + |v, t, vectors| { radix4_schedule( #[inline(always)] - |top, bot, k| avx2::butterfly_regs(v, top, bot, t[k]), + |top, bot, k| avx2::butterfly_regs(v, top, bot, t[k], vectors), ); }, ); } - // SAFETY: this code is compiled only where VPCLMULQDQ and AVX2 are enabled. + // SAFETY: as above. unsafe { sweep(block, src, t, quarter, num_ntts) } } #[cfg(not(any( all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ) )))] fused_rows::<4>(block, src, quarter, num_ntts, |src, rows| { @@ -323,9 +336,9 @@ pub(super) fn butterfly_interleaved_fused_2layer( all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ) )))] #[inline(always)] @@ -470,8 +483,7 @@ fn butterfly_lanes_avx2(top: &mut [F64], bot: &mut [F64], twiddle: u64) { } } -/// Four butterflies `u' = u + v * t`, `v' = v + u'` in registers: the products one per 128-bit -/// lane, even lanes then odd lanes as in `mul_xor_avx512`, then one shift-and-XOR reduction. +/// Four butterflies `u' = u + v * t`, `v' = v + u'` in registers, by [`mul_xor_ymm`]. #[cfg(all( target_arch = "x86_64", target_feature = "pclmulqdq", @@ -487,20 +499,39 @@ fn butterfly_ymm( twiddle: u64, ) -> (core::arch::x86_64::__m256i, core::arch::x86_64::__m256i) { use core::arch::x86_64::*; + + let new_u = mul_xor_ymm(u, v, _mm256_set1_epi64x(twiddle as i64)); + (new_u, _mm256_xor_si256(v, new_u)) +} + +/// `u ^ v * tw` in each of the four lanes, `tw` broadcast: the products one per 128-bit lane, even lanes then +/// odd lanes as in `mul_xor_avx512`, then one shift-and-XOR reduction. +#[cfg(all( + target_arch = "x86_64", + target_feature = "pclmulqdq", + target_feature = "avx2", + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) +))] +#[inline] +#[target_feature(enable = "pclmulqdq", enable = "avx2")] +#[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] +fn mul_xor_ymm( + u: core::arch::x86_64::__m256i, + v: core::arch::x86_64::__m256i, + tw: core::arch::x86_64::__m256i, +) -> core::arch::x86_64::__m256i { + use core::arch::x86_64::*; use primitives::field::gf2_64x2::x86_64::reduce_halves256; #[cfg(target_feature = "vpclmulqdq")] - let (even, odd) = { - let tw = _mm256_set1_epi64x(twiddle as i64); - ( - _mm256_clmulepi64_epi128::<0x00>(v, tw), - _mm256_clmulepi64_epi128::<0x11>(v, tw), - ) - }; - // Four 128-bit multiplies; immediate bit 0 picks the qword of `v`. + let (even, odd) = ( + _mm256_clmulepi64_epi128::<0x00>(v, tw), + _mm256_clmulepi64_epi128::<0x11>(v, tw), + ); + // Four 128-bit multiplies by the twiddle in the low qword; immediate bit 0 picks the qword of `v`. #[cfg(not(target_feature = "vpclmulqdq"))] let (even, odd) = { - let tw = _mm_cvtsi64_si128(twiddle as i64); + let tw = _mm256_castsi256_si128(tw); let (v01, v23) = (_mm256_castsi256_si128(v), _mm256_extracti128_si256::<1>(v)); ( _mm256_set_m128i( @@ -513,8 +544,7 @@ fn butterfly_ymm( ), ) }; - let new_u = _mm256_xor_si256(u, reduce_halves256(even, odd)); - (new_u, _mm256_xor_si256(v, new_u)) + _mm256_xor_si256(u, reduce_halves256(even, odd)) } /// The butterflies of one row pair that the NEON kernel covers: whole groups of eight lanes. @@ -744,24 +774,27 @@ fn transpose_butterfly_lanes_avx512(top: &mut [F64], bot: &mut [F64], twiddle: u ); } -/// The AVX-512 register kernels at half the width, for a machine with VPCLMULQDQ but no AVX-512. +/// The AVX-512 register kernels at half the width, for a machine with AVX2 but not both VPCLMULQDQ and AVX-512. #[cfg(all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ))] mod avx2 { use core::arch::x86_64::*; use primitives::field::F64; - use primitives::field::gf2_64x2::x86_64::reduce_halves256; use primitives::{load_at, store_at}; - use super::fused_rows; + use super::{fused_rows, mul_xor_ymm}; - /// Four-lane vectors of every row that a register kernel keeps in flight, as on AVX-512. + /// Four-lane vectors of every row that a register kernel keeps in flight: two as on AVX-512, but one with + /// 128-bit multiplies, where a vector's four products already hide each other's latency. + #[cfg(target_feature = "vpclmulqdq")] const LANE_VECTORS: usize = 2; + #[cfg(not(target_feature = "vpclmulqdq"))] + const LANE_VECTORS: usize = 1; /// A broadcast twiddle, and whether it is zero: the butterfly is then `v' = v + u`, with no product. pub(super) type Twiddle = (__m256i, bool); @@ -769,15 +802,6 @@ mod avx2 { /// The vectors of `N` rows held in registers: `v[row][k]` is vector `k` of the row. pub(super) type RowVectors = [[__m256i; LANE_VECTORS]; N]; - /// `u ^ v * tw` in each of the four lanes, as `mul_xor_avx512`. - #[inline] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] - fn mul_xor(u: __m256i, v: __m256i, tw: __m256i) -> __m256i { - let even = _mm256_clmulepi64_epi128::<0x00>(v, tw); - let odd = _mm256_clmulepi64_epi128::<0x11>(v, tw); - _mm256_xor_si256(u, reduce_halves256(even, odd)) - } - /// Words `4i..4i + n` of `src`, then zeros. /// /// A short tail goes through a padded copy rather than `vpmaskmovq`, which AMD lets fault on masked-out words. @@ -812,13 +836,20 @@ mod avx2 { dst[4 * i..4 * i + n].copy_from_slice(&words[..n]); } - /// Butterfly rows `top` and `bot` of every vector in flight. + /// Butterfly rows `top` and `bot` of the first `vectors` vectors in flight. #[inline] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] - pub(super) fn butterfly_regs(v: &mut RowVectors, top: usize, bot: usize, (tw, zero): Twiddle) { - for k in 0..LANE_VECTORS { + #[target_feature(enable = "pclmulqdq", enable = "avx2")] + #[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] + pub(super) fn butterfly_regs( + v: &mut RowVectors, + top: usize, + bot: usize, + (tw, zero): Twiddle, + vectors: usize, + ) { + for k in 0..vectors { let (u, w) = (v[top][k], v[bot][k]); - let new_u = if zero { u } else { mul_xor(u, w, tw) }; + let new_u = if zero { u } else { mul_xor_ymm(u, w, tw) }; v[top][k] = new_u; v[bot][k] = _mm256_xor_si256(w, new_u); } @@ -826,14 +857,15 @@ mod avx2 { /// The sweep of [`fused_rows`] with `kernel` on registers: [`fused_lanes`] for each row group. #[inline] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + #[target_feature(enable = "pclmulqdq", enable = "avx2")] + #[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] pub(super) fn fused_sweep( block: &mut [F64], src: Option<&[F64]>, stride_rows: usize, num_ntts: usize, twiddles: &[F64; T], - kernel: impl Fn(&mut RowVectors, &[Twiddle; T]), + kernel: impl Fn(&mut RowVectors, &[Twiddle; T], usize), ) { fused_rows::(block, src, stride_rows, num_ntts, |src, rows| { fused_lanes( @@ -841,19 +873,21 @@ mod avx2 { rows, twiddles, #[inline(always)] - |v, t| kernel(v, t), + |v, t, vectors| kernel(v, t, vectors), ) }); } - /// `fused_lanes_avx512` at half the width: [`LANE_VECTORS`] four-lane vectors of every row at a time. + /// `fused_lanes_avx512` at half the width: [`LANE_VECTORS`] four-lane vectors of every row at a time, and in the + /// last chunk only those that hold lanes. #[inline] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + #[target_feature(enable = "pclmulqdq", enable = "avx2")] + #[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] fn fused_lanes( src: Option<[&[F64]; N]>, rows: &mut [&mut [F64]; N], twiddles: &[F64; T], - kernel: impl Fn(&mut RowVectors, &[Twiddle; T]), + kernel: impl Fn(&mut RowVectors, &[Twiddle; T], usize), ) { const CHUNK: usize = 4 * LANE_VECTORS; let len = rows[0].len(); @@ -872,7 +906,7 @@ mod avx2 { }), None => std::array::from_fn(|i| std::array::from_fn(|k| load_at(&*chunks[i], 4 * k))), }; - kernel(&mut v, &tw); + kernel(&mut v, &tw, LANE_VECTORS); for i in 0..N { for k in 0..LANE_VECTORS { store_at(chunks[i], 4 * k, v[i][k]); @@ -890,7 +924,11 @@ mod avx2 { None => load_prefix(&*rows[i], reg + k, n(k)), }) }); - kernel(&mut v, &tw); + // Each count is its own call, so the kernel's loops over the vectors unroll. + match (len - lane).div_ceil(4) { + 1 => kernel(&mut v, &tw, 1), + _ => kernel(&mut v, &tw, LANE_VECTORS), + } for i in 0..N { for k in 0..LANE_VECTORS { store_prefix(rows[i], reg + k, n(k), v[i][k]); @@ -900,6 +938,7 @@ mod avx2 { } /// `transpose_butterfly_lanes_avx512` at half the width: `s = u + v; u' = s; v' = s * t + v`. + #[cfg(target_feature = "vpclmulqdq")] #[inline] #[target_feature(enable = "vpclmulqdq", enable = "avx2")] pub(super) fn transpose_butterfly_lanes(top: &mut [F64], bot: &mut [F64], twiddle: u64) { @@ -909,14 +948,14 @@ mod avx2 { let (u, v): (__m256i, __m256i) = (load_at(top, 4 * i), load_at(bot, 4 * i)); let s = _mm256_xor_si256(u, v); store_at(top, 4 * i, s); - store_at(bot, 4 * i, mul_xor(v, s, tw)); + store_at(bot, 4 * i, mul_xor_ymm(v, s, tw)); } let tail = top.len() - 4 * vectors; if tail > 0 { let (u, v) = (load_prefix(top, vectors, tail), load_prefix(bot, vectors, tail)); let s = _mm256_xor_si256(u, v); store_prefix(top, vectors, tail, s); - store_prefix(bot, vectors, tail, mul_xor(v, s, tw)); + store_prefix(bot, vectors, tail, mul_xor_ymm(v, s, tw)); } } } diff --git a/crates/experimental_prover/pcs/src/ntt/openings.rs b/crates/experimental_prover/pcs/src/ntt/openings.rs index 1176d145e8..e0afd2f04a 100644 --- a/crates/experimental_prover/pcs/src/ntt/openings.rs +++ b/crates/experimental_prover/pcs/src/ntt/openings.rs @@ -6,7 +6,7 @@ use super::encode::Codeword; use super::plan::Plan; use super::scratch::with_scratch; use super::source::Lanes; -use super::stage::{BATCH, Stage}; +use super::stage::{BATCH, Stage, lane_groups}; use super::twiddles::AdditiveNttF64; /// The rows at the sorted `positions` of a codeword kept whole, `row_words` words each, back to back. @@ -177,39 +177,27 @@ impl<'a> Rebuild<'a> { // A group writes row `r` of every queried sub-block, and the groups cover every residue `r`. let out = parallel::SendPtr(built.as_mut_ptr()); parallel::for_each_chunk((1usize << log_step) / BATCH, |lo, hi| { - // One octet of lanes of a batch of residues is 64 lanes of one transform. + // An octet of lanes of a batch of residues is 64 lanes of one transform, a quad 32. let stage_len = rows * BATCH * 8; // `built` is far larger than a node's L3 share and is read only after every task: its lines stream. let stream = primitives::stream::Stream::new(); with_scratch(2 * stage_len, |all| { let (stage, scratch) = all.split_at_mut(stage_len); for t in lo..hi { - for octet in 0..num_ntts / 8 { - self.stage.fill(stage, t * BATCH, octet); + for (lane0, lanes) in lane_groups(num_ntts) { + let width = BATCH * lanes; + let (stage, scratch) = (&mut stage[..rows * width], &mut scratch[..rows * width]); + self.stage.fill(stage, t * BATCH, lane0, lanes); for (coset, run, live) in &self.cosets { let live_rows = Some(&live[..]); - ntt.run_layers( - scratch, - Some(stage), - deep_start, - BATCH * 8, - start, - start, - *coset, - live_rows, - ); + ntt.run_layers(scratch, Some(stage), deep_start, width, start, start, *coset, live_rows); for (pos, &i) in run.clone().zip(live) { - for (s, line) in scratch[i * BATCH * 8..][..BATCH * 8] - .as_chunks::<8>() - .0 - .iter() - .enumerate() - { + for (s, lanes_of_s) in scratch[i * width..][..width].chunks_exact(lanes).enumerate() { // SAFETY: this task alone owns row `t * BATCH + s` of every sub-block, // and `built` outlives the tasks. let dst = - unsafe { out.slice(pos * sub_len + (t * BATCH + s) * num_ntts + 8 * octet, 8) }; - stream.copy(dst, line); + unsafe { out.slice(pos * sub_len + (t * BATCH + s) * num_ntts + lane0, lanes) }; + stream.copy(dst, lanes_of_s); } } } diff --git a/crates/experimental_prover/pcs/src/ntt/plan.rs b/crates/experimental_prover/pcs/src/ntt/plan.rs index 569f980dbc..d48871d360 100644 --- a/crates/experimental_prover/pcs/src/ntt/plan.rs +++ b/crates/experimental_prover/pcs/src/ntt/plan.rs @@ -34,11 +34,11 @@ impl Plan { } } - /// Whether the first gathered pass can read a lane-major message itself: it has a layer to run, whole cache - /// lines in a row, and enough residues to batch. + /// Whether the first gathered pass can read a lane-major message itself: it has a layer to run, whole quads of + /// lanes in a row, and enough residues to batch. pub(super) fn reads_lanes(&self) -> bool { let g = (self.deep_start - self.start).min(self.max_g); - self.start < self.deep_start && self.num_ntts.is_multiple_of(8) && self.log_d - self.start - g >= LOG_BATCH + self.start < self.deep_start && self.num_ntts.is_multiple_of(4) && self.log_d - self.start - g >= LOG_BATCH } /// Whether an opening can rebuild a row from a lane-major message alone: one gathered pass reads it and diff --git a/crates/experimental_prover/pcs/src/ntt/stage.rs b/crates/experimental_prover/pcs/src/ntt/stage.rs index d6e2312605..ff27220a80 100644 --- a/crates/experimental_prover/pcs/src/ntt/stage.rs +++ b/crates/experimental_prover/pcs/src/ntt/stage.rs @@ -19,7 +19,7 @@ use super::window::STREAM_MIN_WORDS; /// - A row is `row_words >= n` words: the first `row_words - n` lanes are zero padding. /// - Codeword lane `t` past the padding takes message block `row_words - 1 - t`. /// - Any lane count works: a commitment encodes only the lanes that carry data, and a caller pads -/// the row to the vector width. +/// the row to a multiple of [`ROW_LANES`](super::ROW_LANES). /// /// # Why descending /// @@ -83,10 +83,19 @@ pub(super) fn transpose_lane_major(out: &mut [F64], msg: &[F64], n_lanes: usize, pub(super) const LOG_BATCH: usize = 3; pub(super) const BATCH: usize = 1 << LOG_BATCH; +/// The lane groups a stage moves across a row of `row_words` lanes, as `(first lane, lanes)`: whole octets, then a +/// quad where the row ends four lanes past one. +pub(super) fn lane_groups(row_words: usize) -> impl Iterator { + assert!(row_words.is_multiple_of(4), "a staged row is whole quads of lanes"); + (0..row_words) + .step_by(8) + .map(move |lane| (lane, (row_words - lane).min(8))) +} + /// The transpose of a lane-major message through a stage, for one gathered pass. /// /// - It owns the message and the geometry of the fill: the step, the message rows and the row width. -/// - [`Stage::fill`] moves one octet of lanes of a batch of residues into a stage buffer. +/// - [`Stage::fill`] moves one group of lanes of a batch of residues into a stage buffer. /// - A task fills the stage, then each block of the pass transforms a copy of it. pub(super) struct Stage<'a> { lanes: Lanes<'a>, @@ -106,17 +115,27 @@ impl<'a> Stage<'a> { } } - /// Transpose one octet of lanes of a batch of [`BATCH`] consecutive residues from `t0` into `stage`. + /// Transpose lanes `lane0..lane0 + lanes`, a group of [`lane_groups`], of a batch of [`BATCH`] consecutive + /// residues from `t0` into `stage`. /// /// - Message row `j` is `msg_rows` words apart per lane block, and word `w` of a row of `row_words` words /// is lane block `row_words - 1 - w` of the message, or zero while `w` is below the padding. /// - Residue `t0 + s` takes rows `t0 + s + i * step` for `i < rows`, `step = 2^log_step`. - /// - The octet is lanes `8 * octet..8 * octet + 8`. Together with the residues, they are 64 lanes of one transform. - /// - `stage` has `rows` rows of 64 words. - /// Lane `8 * octet + k` of row `t0 + s + i * step` is word `(i * BATCH + s) * 8 + k`. - /// - One tile is 8 words of 8 lane blocks, so each 64 byte line of a lane block is read once, whole. + /// - Together with the residues, the group's lanes are `BATCH * lanes` lanes of one transform. + /// - `stage` has `rows` rows of `BATCH * lanes` words. + /// Lane `lane0 + k` of row `t0 + s + i * step` is word `(i * BATCH + s) * lanes + k`. + /// - One tile is 8 words of each lane block of the group, so each 64 byte line of a lane block is read once, whole. #[inline] - pub(super) fn fill(&self, stage: &mut [F64], t0: usize, octet: usize) { + pub(super) fn fill(&self, stage: &mut [F64], t0: usize, lane0: usize, lanes: usize) { + match lanes { + 8 => self.fill_group::<8>(stage, t0, lane0), + 4 => self.fill_group::<4>(stage, t0, lane0), + _ => unreachable!("a stage moves an octet or a quad of lanes"), + } + } + + #[inline(always)] + fn fill_group(&self, stage: &mut [F64], t0: usize, lane0: usize) { let Self { lanes: Lanes { words: msg, n_lanes }, log_step, @@ -126,22 +145,23 @@ impl<'a> Stage<'a> { // SAFETY: AVX-512F is enabled at compile time. #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] unsafe { - fill_stage_tiles(stage, msg, t0, log_step, msg_rows, n_lanes, row_words, octet) + fill_stage_tiles::(stage, msg, t0, log_step, msg_rows, n_lanes, row_words, lane0) } // SAFETY: AVX2 is enabled at compile time. #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] unsafe { - fill_stage_tiles(stage, msg, t0, log_step, msg_rows, n_lanes, row_words, octet) + fill_stage_tiles::(stage, msg, t0, log_step, msg_rows, n_lanes, row_words, lane0) } #[cfg(not(any( all(target_arch = "x86_64", target_feature = "avx512f"), all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")) )))] - fill_stage_tiles(stage, msg, t0, log_step, msg_rows, n_lanes, row_words, octet); + fill_stage_tiles::(stage, msg, t0, log_step, msg_rows, n_lanes, row_words, lane0); } } -/// [`Stage::fill`], a target-feature function where AVX-512 or AVX2 is enabled, so its tiles move through registers. +/// [`Stage::fill`] of `W` lanes, a target-feature function where AVX-512 or AVX2 is enabled, so its tiles move +/// through registers. #[cfg_attr( all(target_arch = "x86_64", target_feature = "avx512f"), target_feature(enable = "avx512f"), @@ -152,7 +172,7 @@ impl<'a> Stage<'a> { target_feature(enable = "avx2"), inline )] -fn fill_stage_tiles( +fn fill_stage_tiles( stage: &mut [F64], msg: &[F64], t0: usize, @@ -160,28 +180,32 @@ fn fill_stage_tiles( msg_rows: usize, n_lanes: usize, row_words: usize, - octet: usize, + lane0: usize, ) { - let rows = stage.len() / (BATCH * 8); + let rows = stage.len() / (BATCH * W); let pad = row_words - n_lanes; - // Lane block of line `k` of the tile, which is lane `8 * octet + k`. - let block = |k: usize| row_words - 1 - (8 * octet + k); + // Lane block of line `k` of the tile, which is lane `lane0 + k`. + let block = |k: usize| row_words - 1 - (lane0 + k); // The lines of a tile that are padding lanes. - let pad_lines = (0..8).filter(|k| 8 * octet + k < pad).fold(0u8, |m, k| m | 1 << k); + let pad_lines = (0..W).filter(|k| lane0 + k < pad).fold(0u8, |m, k| m | 1 << k); for i in 0..rows { // Line `k` of the tile: 8 residues of one lane block, or zeros for a padding lane. - let src: [usize; 8] = std::array::from_fn(|k| block(k) * msg_rows + t0 + (i << log_step)); - fill_tile(stage, i * BATCH, msg, src, pad_lines); + let src: [usize; W] = std::array::from_fn(|k| block(k) * msg_rows + t0 + (i << log_step)); + fill_tile::(stage, i * BATCH, msg, src, pad_lines); } } -/// Write `stage[tile + s][k] = msg[src[k] + s]` for the 8 x 8 words of eight lines, a line in `zeros` being zeros. +/// Write `stage[(tile + s) * W + k] = msg[src[k] + s]` for the 8 x `W` words of `W` lines, a line in `zeros` being +/// zeros. #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] #[inline] #[target_feature(enable = "avx512f")] -fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; 8], zeros: u8) { +fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; W], zeros: u8) { use core::arch::x86_64::*; use primitives::simd::{load, store, transpose8}; + if W != 8 { + return fill_tile_ymm::(stage, tile, msg, src, zeros); + } let r: [__m512i; 8] = std::array::from_fn(|k| { if zeros >> k & 1 == 1 { _mm512_setzero_si512() @@ -195,24 +219,27 @@ fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; 8], zeros } } -/// [`fill_tile`] as two 4x4 transposes of YMM halves per half of the lines. #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] +use fill_tile_ymm as fill_tile; + +/// [`fill_tile`] as one 4x4 transpose of YMM halves per half of each quad of lines. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] #[inline] #[target_feature(enable = "avx2")] -fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; 8], zeros: u8) { +fn fill_tile_ymm(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; W], zeros: u8) { use core::arch::x86_64::*; use primitives::{load_at, store_at}; // Line `k`, half `h`: words `4h..4h + 4` of the line's 8 residues. - let r: [[__m256i; 2]; 8] = std::array::from_fn(|k| { + let r: [[__m256i; 2]; W] = std::array::from_fn(|k| { if zeros >> k & 1 == 1 { [_mm256_setzero_si256(); 2] } else { [load_at(msg, src[k]), load_at(msg, src[k] + 4)] } }); - // Residue `s` is word `s` of every line: lines 0..4 in its first half, 4..8 in its second. + // Residue `s` is word `s` of every line: lines `4q..4q + 4` at words `4q..4q + 4` of its row. for h in 0..2 { - for q in 0..2 { + for q in 0..W / 4 { let t = |k: usize| r[4 * q + k][h]; let (a0, a1) = (_mm256_unpacklo_epi64(t(0), t(1)), _mm256_unpackhi_epi64(t(0), t(1))); let (a2, a3) = (_mm256_unpacklo_epi64(t(2), t(3)), _mm256_unpackhi_epi64(t(2), t(3))); @@ -223,7 +250,7 @@ fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; 8], zeros _mm256_permute2x128_si256::<0x31>(a1, a3), ]; for (i, col) in cols.into_iter().enumerate() { - store_at(stage, (tile + 4 * h + i) * 8 + 4 * q, col); + store_at(stage, (tile + 4 * h + i) * W + 4 * q, col); } } } @@ -235,10 +262,10 @@ fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; 8], zeros all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")) )))] #[inline(always)] -fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; 8], zeros: u8) { +fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; W], zeros: u8) { for s in 0..8 { - for k in 0..8 { - stage[(tile + s) * 8 + k] = if zeros >> k & 1 == 1 { + for k in 0..W { + stage[(tile + s) * W + k] = if zeros >> k & 1 == 1 { F64::ZERO } else { msg[src[k] + s] diff --git a/crates/experimental_prover/pcs/src/ntt/tests.rs b/crates/experimental_prover/pcs/src/ntt/tests.rs index 78a6652151..4a2dc50826 100644 --- a/crates/experimental_prover/pcs/src/ntt/tests.rs +++ b/crates/experimental_prover/pcs/src/ntt/tests.rs @@ -201,7 +201,8 @@ fn fused_encode_matches_replicate_then_transform() { /// that lane's contiguous message block, which is what makes a commitment over /// `n_lanes` lanes equal to the `2^log_batch_size`-lane one with a zero tail. /// The shapes cover the transposing fused first pass, its fallback, and lane -/// counts that are not powers of two (the padding-free commit's whole point). +/// counts that are not powers of two (the padding-free commit's whole point), +/// in rows padded to four lanes. #[test] fn lane_major_msg_encode_matches_per_lane_reference() { let mut rng = Rng::new(0x1A2E); @@ -211,6 +212,7 @@ fn lane_major_msg_encode_matches_per_lane_reference() { (5, 2, 7), (9, 1, 5), (12, 2, 37), + (12, 2, 43), (14, 1, 64), ] { let log_d = log_rows + log_inv_rate; @@ -218,7 +220,7 @@ fn lane_major_msg_encode_matches_per_lane_reference() { let rows = 1usize << log_rows; let msg: Vec = (0..rows * n_lanes).map(|_| F64(rng.next_u64())).collect(); - let row_words = n_lanes.next_multiple_of(8); + let row_words = n_lanes.next_multiple_of(4); let mut got = vec![F64::ZERO; (row_words << log_rows) << log_inv_rate]; transpose_lane_major(&mut got[..row_words << log_rows], &msg, n_lanes, row_words, log_rows); ntt.encode_interleaved_in_place(&mut got, row_words, log_inv_rate); @@ -246,22 +248,23 @@ fn lane_major_msg_encode_matches_per_lane_reference() { /// The lane-major encode reads the message in its first gathered pass, so it must equal the /// transpose into block 0 followed by the in-place encode: rows handed over and rows opened. -/// The shapes cover a fused first pass (with and without padding lanes, at both rates), -/// a deep-only plan, and a domain too small for any gathered pass. +/// The shapes cover a fused first pass (with and without padding lanes, at both rates, in rows of whole octets +/// and in rows ending in a quad), a deep-only plan, and a domain too small for any gathered pass. #[test] fn lane_major_encode_matches_transpose_then_in_place() { let mut rng = Rng::new(0x7A5E); - for (log_rows, log_inv_rate, n_lanes) in [ - (13usize, 2usize, 47usize), - (14, 1, 40), - (13, 1, 17), - (12, 2, 64), - (5, 2, 7), + for (log_rows, log_inv_rate, n_lanes, row_words) in [ + (13usize, 2usize, 47usize, 48usize), + (14, 1, 40, 40), + (13, 1, 17, 24), + (13, 1, 17, 20), + (15, 1, 17, 20), + (12, 2, 64, 64), + (5, 2, 7, 8), ] { let log_d = log_rows + log_inv_rate; let ntt = AdditiveNttF64::standard(log_d); let msg: Vec = (0..n_lanes << log_rows).map(|_| F64(rng.next_u64())).collect(); - let row_words = n_lanes.next_multiple_of(8); let mut want = vec![F64::ZERO; (row_words << log_rows) << log_inv_rate]; transpose_lane_major(&mut want[..row_words << log_rows], &msg, n_lanes, row_words, log_rows); ntt.encode_interleaved_in_place(&mut want, row_words, log_inv_rate); @@ -272,8 +275,9 @@ fn lane_major_encode_matches_transpose_then_in_place() { let codeword = ntt.encode_lane_major(&msg, n_lanes, row_words, log_inv_rate, keep, &on_rows); drop(on_rows); let keep_nothing = keep == Keep::Nothing; - let context = - format!("log_rows={log_rows}, rate={log_inv_rate}, n_lanes={n_lanes}, keep_nothing={keep_nothing}"); + let context = format!( + "log_rows={log_rows}, rate={log_inv_rate}, n_lanes={n_lanes}, row_words={row_words}, keep_nothing={keep_nothing}" + ); assert_eq!(handed, want, "handed-over rows, {context}"); let mut positions: Vec = (0..40).map(|_| rng.next_u64() as usize % (1 << log_d)).collect(); positions.sort_unstable(); diff --git a/crates/experimental_prover/pcs/src/whir.rs b/crates/experimental_prover/pcs/src/whir.rs index b209663e63..5f6217f4bb 100644 --- a/crates/experimental_prover/pcs/src/whir.rs +++ b/crates/experimental_prover/pcs/src/whir.rs @@ -23,7 +23,7 @@ //! `E`; the committed alphabet remains `K = GF(2^64)`. use crate::merkle; -use crate::ntt::{AdditiveNttF64, Codeword, Keep}; +use crate::ntt::{AdditiveNttF64, Codeword, Keep, ROW_LANES}; use crate::stack_open::Terms; use fiat_shamir::merkle::{Hash, MerklePath}; use primitives::field::{Acc8, F64, F128, F128Unreduced, F128x8, fold_pairs8_ext, msg_terms8_ext, powers}; @@ -88,7 +88,7 @@ pub struct Commitment { pub struct ProverData { ntt: AdditiveNttF64, codeword: Codeword, - /// Words per codeword row: the committed lanes, with zero lanes in front up to the vector width. + /// Words per codeword row: the committed lanes, with zero lanes in front up to a multiple of `ROW_LANES`. row_words: usize, pub(crate) merkle_tree: merkle::CutTree, } @@ -116,9 +116,8 @@ const KEEP_CODEWORD_MAX_BYTES: usize = 1 << 30; /// state every leaf shares. At most the image's tail, the committed lanes, /// rides the proof, the opening saying its width and the image's. /// -/// The encoded row is padded to a whole AVX-512 vector of lanes with zero lanes -/// in front, so the NTT has no masked tail. The padding is the image's zeros and -/// never rides the proof. +/// The encoded row is padded to a whole number of the NTT's vectors (`ROW_LANES`) with zero lanes in front, so the +/// NTT has no vector of padding lanes. The padding is the image's zeros and never rides the proof. pub fn commit(message: &[F64], log_n: usize, log_batch_size: usize, log_inv_rate: usize) -> (Commitment, ProverData) { assert!(log_inv_rate >= 1, "log_inv_rate must be >= 1 for a non-trivial RS code"); assert!(log_n > log_batch_size, "witness must be wider than the interleaving"); @@ -131,23 +130,29 @@ pub fn commit(message: &[F64], log_n: usize, log_batch_size: usize, log_inv_rate ); let k_code = log_rows + log_inv_rate; let n_positions = 1usize << k_code; - let row_words = n_lanes.next_multiple_of(8).min(1usize << log_batch_size); + let row_words = n_lanes.next_multiple_of(ROW_LANES).min(1usize << log_batch_size); - // Leaves are hashed as the encode finishes each block of rows. - let tree = merkle::MerkleBuilder::new(n_positions, row_words, 1usize << log_batch_size); + // Leaves are hashed as the encode finishes each block of rows, from their data lanes on. + let tree = merkle::MerkleBuilder::new(n_positions, row_words, n_lanes, 1usize << log_batch_size); let ntt = AdditiveNttF64::standard(k_code); - let codeword = - tracing::info_span!("NTT", kind = "base encode", log_domain = k_code, lanes = row_words).in_scope(|| { - let on_rows = |row, rows: &[F64]| tree.absorb(row, rows); - let keep = if !primitives::REBUILD_OVER_STORE { - Keep::Whole - } else if n_positions * row_words * size_of::() > KEEP_CODEWORD_MAX_BYTES { - Keep::Nothing - } else { - Keep::BeforeDeep - }; - ntt.encode_lane_major(message, n_lanes, row_words, log_inv_rate, keep, &on_rows) - }); + let span = tracing::info_span!( + "NTT", + kind = "base encode", + log_domain = k_code, + lanes = row_words, + data = n_lanes + ); + let codeword = span.in_scope(|| { + let on_rows = |row, rows: &[F64]| tree.absorb(row, rows); + let keep = if !primitives::REBUILD_OVER_STORE { + Keep::Whole + } else if n_positions * row_words * size_of::() > KEEP_CODEWORD_MAX_BYTES { + Keep::Nothing + } else { + Keep::BeforeDeep + }; + ntt.encode_lane_major(message, n_lanes, row_words, log_inv_rate, keep, &on_rows) + }); let merkle_tree = tracing::info_span!("Merkle").in_scope(|| tree.finish()); let root = merkle_tree.root(); @@ -192,7 +197,9 @@ impl LigeroWitness { .iter() .flat_map(|&q| row(q).as_chunks::<2>().0.iter().map(|w| F128::new(w[0].0, w[1].0))), ); - let paths = self.tree.open(queries, row_words, row_words, &group_rows, row); + let paths = self + .tree + .open(queries, row_words, row_words, row_words, &group_rows, row); (rows, paths) } @@ -221,7 +228,7 @@ pub(crate) fn ligero_commit_ext( // One leaf per row, its F128s as K words: hashed as the encode finishes each block. let row_words = 2 * num_interleaved; - let builder = merkle::MerkleBuilder::new(block_len, row_words, row_words); + let builder = merkle::MerkleBuilder::new(block_len, row_words, row_words, row_words); let codeword = tracing::info_span!( "NTT", kind = "extension encode", @@ -887,7 +894,8 @@ pub(crate) fn recursive_prover_with_prepared_basis( let zeros = std::iter::repeat_n(F128::ZERO, num_interleaved_0 - n_lanes); rows.extend(image(q).iter().rev().map(|&k| F128::from(k)).chain(zeros)); } - ps.hint_merkle(l0_tree.open(&queries, row_words, num_interleaved_0, &group_rows, image)); + let paths = l0_tree.open(&queries, row_words, n_lanes, num_interleaved_0, &group_rows, image); + ps.hint_merkle(paths); // Nothing reads the L0 codeword, its stack or its tree again. drop((prover_data, std::mem::take(&mut stack))); rows diff --git a/crates/experimental_prover/pcs/src/whir_lanes.rs b/crates/experimental_prover/pcs/src/whir_lanes.rs index 535eb88e54..d2ade143ef 100644 --- a/crates/experimental_prover/pcs/src/whir_lanes.rs +++ b/crates/experimental_prover/pcs/src/whir_lanes.rs @@ -23,13 +23,13 @@ //! Where the stack's blocks under a lane `X F` repeat, its blocks repeat past their first few words (one more per //! round, from multiplying by `X`), so a block of `X F` is a repeat when those words are too. -use primitives::field::{F64, F64x8, F128, F128x8, mul_base_splat8}; +use primitives::field::{F64, F128}; use primitives::multilinear::interp; use primitives::stream::Stream; use crate::stack_open::{SWEEP, Terms}; use crate::whir::{FoldChallenges, FoldTable, InitialBasis}; -use crate::xmul::{exponents8, mul_x_block, multiples, sums8, x_at}; +use crate::xmul::{mul_x_block, multiples, x_at}; /// The words of a block of a round-`r` lane `X F` that can differ from the block before when the stack's blocks under /// it repeat: its first `r`. Later rounds skip no blocks. @@ -112,7 +112,16 @@ impl Folded<'_> { /// /// `X` multiplies column `c` by `A_t` after summing two coefficients, so a multiple of `X` enters as the sums, the /// `A_t` folded into its challenge. + #[cfg(not(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + not(target_feature = "avx512f") + )))] fn block(&self, i: usize, first: usize, out: &mut [F128]) { + use crate::xmul::{exponents8, sums8}; + use primitives::field::{F64x8, F128x8, mul_base_splat8}; + let n = out.len(); let alpha = self.ch.alpha; let out = out.as_chunks_mut::<8>().0; @@ -146,6 +155,69 @@ impl Folded<'_> { } } } + + /// [`Self::block`] two coefficients a register, in the order they lie in memory. + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + not(target_feature = "avx512f") + ))] + fn block(&self, i: usize, first: usize, out: &mut [F128]) { + use crate::xmul::head8; + use crate::xmul::lanes::{load8, load8_base, store8, sums}; + use primitives::field::x4::{F64x2, F128x2}; + + let n = out.len(); + let cx = &self.cx; + // `beta A_t` of the columns of an aligned eight, two to a register, all but the first column's. + let (c23, c45) = (F128x2::new(cx[1], cx[0]), F128x2::new(cx[2], cx[0])); + let out = out.as_chunks_mut::<8>().0; + match self.pairs { + Pairs::Stack => { + let f = self.stack_lane(2 * i).expect("a folded lane has its first"); + let g = self + .stack_lane(2 * i + 1) + .map(|g| g[first..first + n].as_chunks::<8>().0); + let (fb, alpha) = (&f[first..first + n], F128x2::splat(self.ch.alpha)); + for (q, (b, o)) in fb.as_chunks::<8>().0.iter().zip(out).enumerate() { + let (s0, t0) = head8(fb, q, first, outside(f)); + let fv = load8_base(b); + let s = sums(fv, F64x2::new(s0, F64::ZERO)); + let c = [F128x2::new(cx[t0], cx[0]), c23, c45, c23]; + let mut v: [F128x2; 4] = std::array::from_fn(|j| { + let mut p = alpha.mul_base_unreduced(fv[j]); + p ^= c[j].mul_base_unreduced(s[j]); + p.reduce() + }); + if let Some(g) = g { + let gv = load8_base(&g[q]); + v = std::array::from_fn(|j| v[j] + F128x2::from(gv[j])); + } + store8(v, o); + } + } + Pairs::Stored { f, g, len } => { + let (f, g) = (&f[i * len..(i + 1) * len], &g[i * len..(i + 1) * len]); + let (fb, gb) = (&f[first..first + n], g[first..first + n].as_chunks::<8>().0); + let alpha = F128x2::splat(self.ch.alpha).factor(); + let (c23, c45) = (c23.factor(), c45.factor()); + for (q, (b, o)) in fb.as_chunks::<8>().0.iter().zip(out).enumerate() { + let (s0, t0) = head8(fb, q, first, outside(f)); + let fv = load8(b); + let s = sums(fv, F128x2::new(s0, F128::ZERO)); + let c = [F128x2::new(cx[t0], cx[0]).factor(), c23, c45, c23]; + let gv = load8(&gb[q]); + let v = std::array::from_fn(|j| { + let mut p = fv[j].mul_unreduced(alpha); + p ^= s[j].mul_unreduced(c[j]); + gv[j] + p.reduce() + }); + store8(v, o); + } + } + } + } } /// What a round's table leaves for its fold: the projections of `X F` per pair, and the pairs. diff --git a/crates/experimental_prover/pcs/src/xmul.rs b/crates/experimental_prover/pcs/src/xmul.rs index ac14a4929a..cba16f82b4 100644 --- a/crates/experimental_prover/pcs/src/xmul.rs +++ b/crates/experimental_prover/pcs/src/xmul.rs @@ -20,6 +20,15 @@ pub(crate) trait Coef: Copy + Send + Sync + std::ops::Add { fn scale(self, k: F64) -> Self; /// `v[j] * k[j]`, where `k[j]` is one at every odd `j`. fn scale8(v: [Self; 8], k: [F64; 8]) -> [Self; 8]; + + /// [`mul_x_block`]. + #[inline(always)] + fn x_block(block: &[Self], first: usize, outside: impl Fn(usize) -> Self, out: &mut [Self]) { + for (q, out) in out.as_chunks_mut::<8>().0.iter_mut().enumerate() { + let (s, t0) = sums8(block, q, first, &outside); + *out = Self::scale8(s, multipliers8(exponents8(t0))); + } + } } impl Coef for F64 { @@ -32,6 +41,16 @@ impl Coef for F64 { fn scale8(v: [Self; 8], k: [F64; 8]) -> [Self; 8] { std::array::from_fn(|j| if j % 2 == 0 { v[j] * k[j] } else { v[j] }) } + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + not(target_feature = "avx512f") + ))] + #[inline(always)] + fn x_block(block: &[Self], first: usize, outside: impl Fn(usize) -> Self, out: &mut [Self]) { + lanes::x_block_k(block, first, outside, out); + } } impl Coef for F128 { @@ -47,6 +66,16 @@ impl Coef for F128 { .reduce_lanes() .to_array() } + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + not(target_feature = "avx512f") + ))] + #[inline(always)] + fn x_block(block: &[Self], first: usize, outside: impl Fn(usize) -> Self, out: &mut [Self]) { + lanes::x_block_e(block, first, outside, out); + } } /// `A_t` for every `t < 64`. @@ -110,13 +139,10 @@ pub(crate) fn multiples(c: F128) -> [F128; 64] { constants().map(|a| c.mul_base(a)) } -/// The sums `s` of columns `8q..8q + 8` of a block of a lane `h`, `(X h)[c] = A_t s[c]` with `t` from [`exponents8`], -/// and `t0`. `block[i] = h[first + i]`, and `outside(c)` gives the coefficients outside the block that the first column -/// reads; the block's length is a power of two that divides `first`. +/// `s[0]` of [`sums8`], and `t0`. #[inline(always)] -pub(crate) fn sums8(block: &[T], q: usize, first: usize, outside: impl Fn(usize) -> T) -> ([T; 8], usize) { - let b = &block[8 * q..8 * q + 8]; - let (s0, t0) = match (q, first) { +pub(crate) fn head8(block: &[T], q: usize, first: usize, outside: impl Fn(usize) -> T) -> (T, usize) { + match (q, first) { (0, 0) => (T::ZERO, 0), (0, _) => { let t = first.trailing_zeros() as usize; @@ -130,7 +156,16 @@ pub(crate) fn sums8(block: &[T], q: usize, first: usize, outside: impl let t = 3 + q.trailing_zeros() as usize; (block[8 * q - 1] + block[8 * q + (1 << t) - 1], t) } - }; + } +} + +/// The sums `s` of columns `8q..8q + 8` of a block of a lane `h`, `(X h)[c] = A_t s[c]` with `t` from [`exponents8`], +/// and `t0`. `block[i] = h[first + i]`, and `outside(c)` gives the coefficients outside the block that the first column +/// reads; the block's length is a power of two that divides `first`. +#[inline(always)] +pub(crate) fn sums8(block: &[T], q: usize, first: usize, outside: impl Fn(usize) -> T) -> ([T; 8], usize) { + let b = &block[8 * q..8 * q + 8]; + let (s0, t0) = head8(block, q, first, outside); let s = [ s0, b[0] + b[1], @@ -149,9 +184,139 @@ pub(crate) fn sums8(block: &[T], q: usize, first: usize, outside: impl /// 8, that divides `first`. pub(crate) fn mul_x_block(block: &[T], first: usize, outside: impl Fn(usize) -> T, out: &mut [T]) { debug_assert!(block.len() >= 8 && block.len().is_power_of_two() && first.is_multiple_of(block.len())); - for (q, out) in out.as_chunks_mut::<8>().0.iter_mut().enumerate() { - let (s, t0) = sums8(block, q, first, &outside); - *out = T::scale8(s, multipliers8(exponents8(t0))); + debug_assert_eq!(block.len(), out.len()); + T::x_block(block, first, outside, out); +} + +/// [`sums8`] and [`mul_x_block`] on two coefficients a register, in the order they lie in memory, where AVX2 has no +/// AVX-512 beside it. +#[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + not(target_feature = "avx512f") +))] +pub(crate) mod lanes { + use core::arch::x86_64::*; + + use primitives::field::gf2_64x2::x86_64::reduce_halves256; + use primitives::field::x4::{F64x2, F64x4, F128x2, clmul}; + use primitives::field::{F64, F128}; + + use super::{constants, head8}; + + /// A register of two coefficients. + pub(crate) trait Lane2: Copy + std::ops::Add { + /// The 128-bit lanes `vperm2i128` picks with immediate `I`. + fn select(a: Self, b: Self) -> Self; + } + + impl Lane2 for F128x2 { + #[inline(always)] + fn select(a: Self, b: Self) -> Self { + F128x2::select::(a, b) + } + } + + impl Lane2 for F64x2 { + #[inline(always)] + fn select(a: Self, b: Self) -> Self { + F64x2::select::(a, b) + } + } + + /// [`super::sums8`] on the registers `[b0 b1, b2 b3, b4 b5, b6 b7]`, with `s0` the low lane of its register. + #[inline(always)] + pub(crate) fn sums([f01, f23, f45, f67]: [L; 4], s0: L) -> [L; 4] { + let pair = f01 + L::select::<0x01>(f01, f01); + [ + L::select::<0x30>(s0, pair), + L::select::<0x21>(f01, f23) + L::select::<0x11>(f23, f23), + L::select::<0x21>(f23, f45) + L::select::<0x31>(f67, f45), + L::select::<0x21>(f45, f67) + L::select::<0x11>(f67, f67), + ] + } + + /// Eight `E` coefficients as four registers. + #[inline(always)] + pub(crate) fn load8(e: &[F128; 8]) -> [F128x2; 4] { + let p = e.as_chunks::<2>().0; + [ + F128x2::load(&p[0]), + F128x2::load(&p[1]), + F128x2::load(&p[2]), + F128x2::load(&p[3]), + ] + } + + #[inline(always)] + pub(crate) fn store8([a, b, c, d]: [F128x2; 4], out: &mut [F128; 8]) { + let p = out.as_chunks_mut::<2>().0; + a.store(&mut p[0]); + b.store(&mut p[1]); + c.store(&mut p[2]); + d.store(&mut p[3]); + } + + /// Eight `K` coefficients as four registers. + #[inline(always)] + pub(crate) fn load8_base(k: &[F64; 8]) -> [F64x2; 4] { + let q = k.as_chunks::<4>().0; + let ([a, b], [c, d]) = (F64x4::load(&q[0]).spread(), F64x4::load(&q[1]).spread()); + [a, b, c, d] + } + + /// [`super::mul_x_block`] on `E` coefficients: the odd columns' constant is one, so only the even ones multiply. + #[inline(always)] + pub(super) fn x_block_e(block: &[F128], first: usize, outside: impl Fn(usize) -> F128, out: &mut [F128]) { + let a = constants(); + let k46 = F64x2::new(a[2], a[1]); + let chunks = block.as_chunks::<8>().0.iter().zip(out.as_chunks_mut::<8>().0); + for (q, (b, o)) in chunks.enumerate() { + let (s0, t0) = head8(block, q, first, &outside); + let s = sums(load8(b), F128x2::new(s0, F128::ZERO)); + let r02 = F128x2::select::<0x20>(s[0], s[1]) + .mul_base_unreduced(F64x2::new(a[t0], a[1])) + .reduce(); + let r46 = F128x2::select::<0x20>(s[2], s[3]).mul_base_unreduced(k46).reduce(); + let out = [ + F128x2::select::<0x30>(r02, s[0]), + F128x2::select::<0x31>(r02, s[1]), + F128x2::select::<0x30>(r46, s[2]), + F128x2::select::<0x31>(r46, s[3]), + ]; + store8(out, o); + } + } + + /// [`super::mul_x_block`] on `K` coefficients: the even columns' sums in one register, multiplied at once. + #[inline(always)] + pub(super) fn x_block_k(block: &[F64], first: usize, outside: impl Fn(usize) -> F64, out: &mut [F64]) { + let a = constants(); + let chunks = block.as_chunks::<8>().0.iter().zip(out.as_chunks_mut::<8>().0); + for (q, (b, o)) in chunks.enumerate() { + let (s0, t0) = head8(block, q, first, &outside); + // SAFETY: the module's cfg enables avx2; each access is four whole qwords of a chunk of eight. + unsafe { + let (lo, hi) = ( + _mm256_loadu_si256(b.as_ptr().cast()), + _mm256_loadu_si256(b[4..].as_ptr().cast()), + ); + // `[b0 b4 b2 b6]` and `[b1 b5 b3 b7]`. + let (u, h) = (_mm256_unpacklo_epi64(lo, hi), _mm256_unpackhi_epi64(lo, hi)); + let odd = _mm256_xor_si256(u, h); + // `[s0 s4 s2 s6]`: the shuffles cancel in the first qword, where `s0` goes. + let even = _mm256_xor_si256( + _mm256_permute4x64_epi64::<0b01_00_10_00>(h), + _mm256_permute4x64_epi64::<0b11_10_11_00>(h), + ); + let even = _mm256_xor_si256(even, _mm256_zextsi128_si256(_mm_cvtsi64_si128(s0.0 as i64))); + let m = _mm256_set_epi64x(a[1].0 as i64, a[1].0 as i64, a[2].0 as i64, a[t0].0 as i64); + let r = reduce_halves256(clmul::<0x00>(even, m), clmul::<0x11>(even, m)); + _mm256_storeu_si256(o.as_mut_ptr().cast(), _mm256_unpacklo_epi64(r, odd)); + _mm256_storeu_si256(o[4..].as_mut_ptr().cast(), _mm256_unpackhi_epi64(r, odd)); + } + } } } @@ -261,6 +426,19 @@ mod tests { let mut part = vec![F128::ONE; (n + 1 - first) * stride]; mul_x(&h, stride, 1, first, &mut part); assert_eq!(part, got[first * stride..], "E range, mu={mu}, high={high}"); + + // Blocks of the first E lane. + let e: Vec = h.iter().step_by(stride).copied().collect(); + let want: Vec = got.iter().step_by(stride).copied().collect(); + for log_block in 3..=mu { + let n_block = 1usize << log_block; + for first in (0..1usize << mu).step_by(n_block) { + let mut part = vec![F128::ONE; n_block]; + let outside = |c: usize| e.get(c).copied().unwrap_or(F128::ZERO); + mul_x_block(&e[first..first + n_block], first, outside, &mut part); + assert_eq!(part, want[first..first + n_block], "E block {first}+{n_block}, mu={mu}"); + } + } } } } diff --git a/crates/experimental_prover/primitives/src/bit_fold.rs b/crates/experimental_prover/primitives/src/bit_fold.rs index 5191aa6270..f726de14f5 100644 --- a/crates/experimental_prover/primitives/src/bit_fold.rs +++ b/crates/experimental_prover/primitives/src/bit_fold.rs @@ -12,21 +12,15 @@ //! - GFNI: an 8x8 bit matrix per (input byte, output byte), applied to 64 rows by one instruction. //! - AVX2 without GFNI, and NEON: each of those matrices as two 16-entry nibble tables, one byte shuffle each per //! 32 or 16 rows. +//! +//! The 256-bit kernels read byte columns, so there the row-major fold transposes its rows first. use crate::field::F128; /// Rows folded per call. pub const BLOCK: usize = 64; -#[cfg(any( - test, - not(all( - target_arch = "x86_64", - target_feature = "gfni", - target_feature = "avx512bw", - target_feature = "avx512vbmi" - )) -))] +#[cfg(any(test, not(all(target_arch = "x86_64", target_feature = "avx2"))))] /// A byte's subset-sum table: 256 entries, then a cache line of padding. type Table = [F128; 256 + 4]; @@ -76,15 +70,7 @@ impl BitFold { } } -#[cfg(any( - test, - not(all( - target_arch = "x86_64", - target_feature = "gfni", - target_feature = "avx512bw", - target_feature = "avx512vbmi" - )) -))] +#[cfg(any(test, not(all(target_arch = "x86_64", target_feature = "avx2"))))] /// One 256-entry subset-sum table per byte of a row: entry `[j][v]` sums the weights of the set bits of `v` at byte `j`. /// /// A table is a cache line longer than its entries, so one entry of every table does not fall in one cache set: rows of @@ -106,15 +92,7 @@ fn lookup_tables(weights: &[F128]) -> Vec { .collect() } -#[cfg(any( - test, - not(all( - target_arch = "x86_64", - target_feature = "gfni", - target_feature = "avx512bw", - target_feature = "avx512vbmi" - )) -))] +#[cfg(any(test, not(all(target_arch = "x86_64", target_feature = "avx2"))))] /// The fold of one row through the byte tables: one lookup and one XOR per byte. #[inline(always)] fn fold_row_lookup(tables: &[Table], row: &[u8; CHUNKS]) -> F128 { @@ -122,9 +100,12 @@ fn fold_row_lookup(tables: &[Table], row: &[u8; CHUNKS]) -> row.iter().zip(tables).map(|(&v, sums)| sums[usize::from(v)]).sum() } -#[cfg(not(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni")))] +#[cfg(not(all(target_arch = "x86_64", target_feature = "avx2")))] use portable::Imp; +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")))] +use avx2::Imp; + #[cfg(all( target_arch = "x86_64", target_feature = "avx2", @@ -141,12 +122,7 @@ use gfni256::Imp; ))] use gfni::Imp; -#[cfg(not(all( - target_arch = "x86_64", - target_feature = "gfni", - target_feature = "avx512bw", - target_feature = "avx512vbmi" -)))] +#[cfg(not(all(target_arch = "x86_64", target_feature = "avx2")))] mod portable { use super::{BLOCK, F128, Table, fold_row_lookup, lookup_tables}; @@ -155,9 +131,6 @@ mod portable { pub(super) struct Imp { /// One subset-sum table per byte of a row. tables: Vec
, - /// The nibble tables of `avx2::fold_columns`. - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")))] - nibbles: Vec, /// The nibble tables of `neon::fold_columns`. #[cfg(target_arch = "aarch64")] nibbles: Vec, @@ -167,8 +140,6 @@ mod portable { pub(super) fn new(weights: &[F128]) -> Self { Self { tables: lookup_tables(weights), - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")))] - nibbles: super::nibble_tables(weights), #[cfg(target_arch = "aarch64")] nibbles: super::nibble_tables(weights), } @@ -182,23 +153,12 @@ mod portable { out[rows.len()..].fill(F128::ZERO); } - // With AVX2 and GFNI the columns go through `gfni256`, which keeps these tables for rows only. - #[cfg_attr( - all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"), - allow(dead_code) - )] #[inline] pub(super) fn fold_columns(&self, cols: &[[u8; BLOCK]; CHUNKS], out: &mut [F128; BLOCK]) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")))] - // SAFETY: AVX2 is enabled at compile time. - return unsafe { super::avx2::fold_columns(&self.nibbles, cols, out) }; #[cfg(target_arch = "aarch64")] // SAFETY: NEON is part of aarch64. return unsafe { super::neon::fold_columns(&self.nibbles, cols, out) }; - #[cfg(not(any( - all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")), - target_arch = "aarch64" - )))] + #[cfg(not(target_arch = "aarch64"))] for (p, o) in out.iter_mut().enumerate() { *o = cols .iter() @@ -210,13 +170,13 @@ mod portable { } } -/// Entry `16 j + o`: the images of the low and the high nibble of input byte `j` in output byte `o`, each 16-entry -/// table repeated to fill `W` bytes. +/// Entry `16 j + o`: the images of the low and the high nibble of input byte `j` in output byte `o`, each a +/// 16-entry table. #[cfg(any( all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")), target_arch = "aarch64" ))] -fn nibble_tables(weights: &[F128]) -> Vec<[[u8; W]; 2]> { +fn nibble_tables(weights: &[F128]) -> Vec<[[u8; 16]; 2]> { let byte = |w: &F128, o: usize| (w.to_limbs()[o / 8] >> (8 * (o % 8))) as u8; let mut out = Vec::with_capacity(2 * weights.len()); for w in weights.as_chunks::<8>().0 { @@ -224,7 +184,7 @@ fn nibble_tables(weights: &[F128]) -> Vec<[[u8; W]; 2]> { out.push(std::array::from_fn(|half| { std::array::from_fn(|i| { (0..4) - .filter(|t| (i % 16) >> t & 1 == 1) + .filter(|t| i >> t & 1 == 1) .fold(0, |acc, t| acc ^ byte(&w[4 * half + t], o)) }) })); @@ -233,6 +193,74 @@ fn nibble_tables(weights: &[F128]) -> Vec<[[u8; W]; 2]> { out } +/// Up to 64 rows as byte columns, `cols[j][p]` byte `j` of row `p`, rows past the input zero. +#[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + not(all(target_feature = "gfni", target_feature = "avx512bw", target_feature = "avx512vbmi")) +))] +#[inline] +#[target_feature(enable = "avx2")] +fn columns_avx2(rows: &[[u8; CHUNKS]], cols: &mut [[u8; BLOCK]; CHUNKS]) { + match rows.try_into() { + Ok(rows) => columns_full_avx2(rows, cols), + Err(_) => { + // A short block transposes a zero-padded copy. + let mut padded = [[0u8; CHUNKS]; BLOCK]; + padded[..rows.len()].copy_from_slice(rows); + columns_full_avx2(&padded, cols); + } + } +} + +/// [`columns_avx2`] of a full block. +/// +/// A 16-byte slice of a row is a lane, rows `p` and `p + 16` sharing a register, and sixteen registers take one in-lane +/// byte transpose. Rows of 8 bytes put rows `p` and `p + 32` in a lane, so the transpose's registers `k` and `k + 8` +/// are the two halves of column `k`. +#[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + not(all(target_feature = "gfni", target_feature = "avx512bw", target_feature = "avx512vbmi")) +))] +#[inline] +#[target_feature(enable = "avx2")] +fn columns_full_avx2(rows: &[[u8; CHUNKS]; BLOCK], cols: &mut [[u8; BLOCK]; CHUNKS]) { + use core::arch::x86_64::*; + + use crate::bits::transpose_bytes16x16_per_lane256 as transpose; + use crate::{load_at, store_at}; + + let pair = |lo: &[u8], hi: &[u8]| -> __m256i { _mm256_set_m128i(load_at(hi, 0), load_at(lo, 0)) }; + if CHUNKS == 8 { + let bytes = rows.as_flattened(); + let mut regs = [_mm256_setzero_si256(); 16]; + for z in (0..16).step_by(2) { + // Lane `l` of `x`: rows `z + 16 l` and `z + 16 l + 1`; of `y`, the same rows plus 32. + let x = pair(&bytes[8 * z..], &bytes[8 * (z + 16)..]); + let y = pair(&bytes[8 * (z + 32)..], &bytes[8 * (z + 48)..]); + regs[z] = _mm256_unpacklo_epi64(x, y); + regs[z + 1] = _mm256_unpackhi_epi64(x, y); + } + transpose(&mut regs); + for (k, col) in cols.iter_mut().enumerate() { + store_at(col, 0, regs[k]); + store_at(col, 32, regs[k + 8]); + } + } else { + for h in 0..2 { + for g in 0..CHUNKS / 16 { + let mut regs: [__m256i; 16] = + std::array::from_fn(|z| pair(&rows[32 * h + z][16 * g..], &rows[32 * h + 16 + z][16 * g..])); + transpose(&mut regs); + for (k, r) in regs.into_iter().enumerate() { + store_at(&mut cols[16 * g + k], 32 * h, r); + } + } + } + } +} + /// The GFNI fold's byte-matrix products as nibble lookups, for AVX2 without GFNI. /// /// Output byte `o` of a row is `XOR_j A[j][o] * byte j`. A byte matrix applied to a byte is the sum of its @@ -244,64 +272,103 @@ mod avx2 { use super::{BLOCK, F128}; use crate::{load_at, store_at}; - /// [`super::nibble_tables`], each table in both 128-bit lanes. - pub(super) type Nibbles = [[u8; 32]; 2]; + /// An entry of [`super::nibble_tables`], broadcast to both 128-bit lanes when used. + type Nibbles = [[u8; 16]; 2]; - /// [`super::BitFold::fold_columns`]: per 32 rows and coefficient, eight output bytes summed over the - /// input bytes, then transposed into words. + /// The nibble tables. + #[derive(Clone, Debug)] + pub(super) struct Imp { + tables: Vec, + } + + impl Imp { + pub(super) fn new(weights: &[F128]) -> Self { + Self { + tables: super::nibble_tables(weights), + } + } + + #[inline] + pub(super) fn fold_block(&self, rows: &[[u8; CHUNKS]], out: &mut [F128; BLOCK]) { + let mut cols = [[0u8; BLOCK]; CHUNKS]; + // SAFETY: AVX2 is enabled at compile time (the module's cfg). + unsafe { super::columns_avx2(rows, &mut cols) }; + self.fold_columns(&cols, out); + } + + #[inline] + pub(super) fn fold_columns(&self, cols: &[[u8; BLOCK]; CHUNKS], out: &mut [F128; BLOCK]) { + // SAFETY: AVX2 is enabled at compile time (the module's cfg). + unsafe { fold_columns(&self.tables, cols, out) } + } + } + + /// [`super::BitFold::fold_columns`]: output bytes summed over the input bytes, 32 rows to a register, then per half a + /// byte transpose of the 16 output-byte registers into 32 F128 values. + /// + /// Up to 64-byte rows, eight output bytes of one half at a time. Wider rows' tables outgrow L1, so four output bytes + /// of both halves at a time read each table once. #[inline] #[target_feature(enable = "avx2")] - pub(super) fn fold_columns( - tables: &[Nibbles], - cols: &[[u8; BLOCK]; CHUNKS], - out: &mut [F128; BLOCK], - ) { - assert_eq!(tables.len(), 16 * CHUNKS); + fn fold_columns(tables: &[Nibbles], cols: &[[u8; BLOCK]; CHUNKS], out: &mut [F128; BLOCK]) { + let tables: &[[Nibbles; 16]; CHUNKS] = tables.as_chunks().0.try_into().expect("16 tables per byte"); let mask = _mm256_set1_epi8(0x0f); - let mut words = [[0u64; BLOCK]; 2]; - for (k, words) in words.iter_mut().enumerate() { + let nibbles = |col: &[u8; BLOCK], h: usize| { + let v: __m256i = load_at(col, 32 * h); + [ + _mm256_and_si256(v, mask), + _mm256_and_si256(_mm256_srli_epi16::<4>(v), mask), + ] + }; + let image = |t: &Nibbles, [lo, hi]: [__m256i; 2]| { + let (tl, th) = ( + _mm256_broadcastsi128_si256(load_at(&t[0], 0)), + _mm256_broadcastsi128_si256(load_at(&t[1], 0)), + ); + _mm256_xor_si256(_mm256_shuffle_epi8(tl, lo), _mm256_shuffle_epi8(th, hi)) + }; + // Byte `16 l + p` of `acc[o]` is byte `o` of row `32 h + 16 l + p`. + let mut store = |h: usize, mut acc: [__m256i; 16]| { + crate::bits::transpose_bytes16x16_per_lane256(&mut acc); + for (k, a) in acc.into_iter().enumerate() { + store_at(out, 32 * h + k, _mm256_castsi256_si128(a)); + store_at(out, 32 * h + 16 + k, _mm256_extracti128_si256::<1>(a)); + } + }; + if CHUNKS <= 64 { for h in 0..2 { - let mut acc = [_mm256_setzero_si256(); 8]; - for (j, col) in cols.iter().enumerate() { - let v: __m256i = load_at(col, 32 * h); - let (lo, hi) = ( - _mm256_and_si256(v, mask), - _mm256_and_si256(_mm256_srli_epi16::<4>(v), mask), - ); - for (i, acc) in acc.iter_mut().enumerate() { - let t = &tables[16 * j + 8 * k + i]; - let (tl, th): (__m256i, __m256i) = (load_at(&t[0], 0), load_at(&t[1], 0)); - let image = _mm256_xor_si256(_mm256_shuffle_epi8(tl, lo), _mm256_shuffle_epi8(th, hi)); - *acc = _mm256_xor_si256(*acc, image); + let mut acc = [_mm256_setzero_si256(); 16]; + for o0 in (0..16).step_by(8) { + let mut a = [_mm256_setzero_si256(); 8]; + for (col, tables) in cols.iter().zip(tables) { + let x = nibbles(col, h); + for (a, t) in a.iter_mut().zip(&tables[o0..o0 + 8]) { + *a = _mm256_xor_si256(*a, image(t, x)); + } } + acc[o0..o0 + 8].copy_from_slice(&a); } - // Byte `p` of `acc[i]` is byte `i` of row `32 h + p`: three unpack stages make words. - let pairs = |a: __m256i, b: __m256i| (_mm256_unpacklo_epi8(a, b), _mm256_unpackhi_epi8(a, b)); - let (l01, h01) = pairs(acc[0], acc[1]); - let (l23, h23) = pairs(acc[2], acc[3]); - let (l45, h45) = pairs(acc[4], acc[5]); - let (l67, h67) = pairs(acc[6], acc[7]); - // Lane 0 of a stage-one register holds rows `base..base + 8`, lane 1 the same rows plus 16. - for (base, (a, b, c, d)) in [(0, (l01, l23, l45, l67)), (8, (h01, h23, h45, h67))] { - let quads = [ - (base, _mm256_unpacklo_epi16(a, b), _mm256_unpacklo_epi16(c, d)), - (base + 4, _mm256_unpackhi_epi16(a, b), _mm256_unpackhi_epi16(c, d)), - ]; - for (base, x, y) in quads { - for (base, w) in [ - (base, _mm256_unpacklo_epi32(x, y)), - (base + 2, _mm256_unpackhi_epi32(x, y)), - ] { - let row = 32 * h + base; - store_at(words, row, _mm256_castsi256_si128(w)); - store_at(words, row + 16, _mm256_extracti128_si256::<1>(w)); + store(h, acc); + } + } else { + let mut acc = [[_mm256_setzero_si256(); 16]; 2]; + for o0 in (0..16).step_by(4) { + let mut a = [[_mm256_setzero_si256(); 4]; 2]; + for (col, tables) in cols.iter().zip(tables) { + let x = [nibbles(col, 0), nibbles(col, 1)]; + for (i, t) in tables[o0..o0 + 4].iter().enumerate() { + for (a, x) in a.iter_mut().zip(x) { + a[i] = _mm256_xor_si256(a[i], image(t, x)); } } } + for (acc, a) in acc.iter_mut().zip(a) { + acc[o0..o0 + 4].copy_from_slice(&a); + } + } + for (h, acc) in acc.into_iter().enumerate() { + store(h, acc); } - } - for (p, o) in out.iter_mut().enumerate() { - *o = F128::new(words[0][p], words[1][p]); } } } @@ -685,8 +752,8 @@ mod gfni { not(all(target_feature = "avx512bw", target_feature = "avx512vbmi")) ))] mod gfni256 { - //! The GFNI fold on 256-bit registers (AVX2 and GFNI, without the AVX-512 kernel's byte permutes), for - //! [`super::BitFold::fold_columns`], whose rows already come as byte columns: + //! The GFNI fold on 256-bit registers (AVX2 and GFNI, without the AVX-512 kernel's byte permutes), on rows as byte + //! columns: //! //! ```text //! per 32 rows: out byte o += A[j][o] * (column j), one affine instruction per column and output byte @@ -694,7 +761,7 @@ mod gfni256 { //! ``` //! //! Sixteen registers hold eight accumulators at a time, so the columns are read twice. The row-major - //! [`super::BitFold::fold_block`] keeps the byte tables. + //! [`super::BitFold::fold_block`] transposes its rows first. use core::arch::x86_64::*; @@ -703,13 +770,11 @@ mod gfni256 { /// Bytes of an F128. const OUT_BYTES: usize = 16; - /// The GFNI matrices, and the byte tables for the row-major fold. + /// The GFNI matrices. #[derive(Clone, Debug)] pub(super) struct Imp { /// `matrices[j][o]` maps byte `j` of a row to byte `o` of its weights' sum. matrices: Vec<[u64; OUT_BYTES]>, - /// The byte tables of the row-major fold. - lookup: super::portable::Imp, } /// Byte `o` of a weight. @@ -743,15 +808,15 @@ mod gfni256 { }) }) .collect(); - Self { - matrices, - lookup: super::portable::Imp::new(weights), - } + Self { matrices } } #[inline] pub(super) fn fold_block(&self, rows: &[[u8; CHUNKS]], out: &mut [F128; BLOCK]) { - self.lookup.fold_block(rows, out); + let mut cols = [[0u8; BLOCK]; CHUNKS]; + // SAFETY: AVX2 is enabled at compile time (the module's cfg). + unsafe { super::columns_avx2(rows, &mut cols) }; + self.fold_columns(&cols, out); } #[inline] @@ -876,23 +941,34 @@ mod tests { check::<128>(&mut rng); } - /// Rows of 16 bytes against the byte-table fold, full and short blocks. + /// Every row width against the byte-table fold, full and short blocks. #[test] fn fold_block_matches_lookup() { - let mut rng = Rng::new(0x0009_A12B); - let weights: Vec = (0..128).map(|_| rng.ext()).collect(); - let (fold, tables) = (BitFold::new(&weights), lookup_tables(&weights)); - for len in [BLOCK, 5] { - let rows: Vec<[u8; 16]> = (0..len) - .map(|_| std::array::from_fn(|_| rng.next_u64() as u8)) - .collect(); - let mut out = [F128::ZERO; BLOCK]; - fold.fold_block(&rows, &mut out); - for p in 0..len { - assert_eq!(out[p], fold_row_lookup(&tables, &rows[p]), "len={len}, row {p}"); + fn check(rng: &mut Rng) { + let weights: Vec = (0..8 * CHUNKS).map(|_| rng.ext()).collect(); + let (fold, tables) = (BitFold::new(&weights), lookup_tables(&weights)); + for len in [BLOCK, 37, 5] { + let rows: Vec<[u8; CHUNKS]> = (0..len) + .map(|_| std::array::from_fn(|_| rng.next_u64() as u8)) + .collect(); + let mut out = [F128::ZERO; BLOCK]; + fold.fold_block(&rows, &mut out); + for p in 0..len { + assert_eq!( + out[p], + fold_row_lookup(&tables, &rows[p]), + "CHUNKS={CHUNKS}, len={len}, row {p}" + ); + } + assert!(out[len..].iter().all(|o| *o == F128::ZERO)); } - assert!(out[len..].iter().all(|o| *o == F128::ZERO)); } + let mut rng = Rng::new(0x0009_A12B); + check::<8>(&mut rng); + check::<16>(&mut rng); + check::<32>(&mut rng); + check::<64>(&mut rng); + check::<128>(&mut rng); } /// The byte tables against the definition: the sum of the weights of the set bits. diff --git a/crates/experimental_prover/primitives/src/bits.rs b/crates/experimental_prover/primitives/src/bits.rs index 1a3932bd03..2a287cffee 100644 --- a/crates/experimental_prover/primitives/src/bits.rs +++ b/crates/experimental_prover/primitives/src/bits.rs @@ -10,6 +10,8 @@ /// So byte column `b` of the eight input rows becomes output row `b`, bit-transposed. /// /// - AVX-512 VBMI with GFNI: one byte permute and one affine map. +/// - AVX2 with GFNI: a byte gather in unpacks and shuffles, then one affine map. +/// - AVX2: the same gather, then three masked-swap rounds. /// - aarch64: a table lookup, then three masked-swap rounds. /// - Elsewhere: the same rounds on a gathered word per column. #[inline] @@ -19,6 +21,21 @@ pub fn bit_transpose_64bytes(input: &[u8; 64], output: &mut [u8; 64]) { unsafe { bit_transpose_64bytes_gfni(input, output) } + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "gfni", + not(target_feature = "avx512vbmi") + ))] + // SAFETY: the features are enabled at compile time. + unsafe { + bit_transpose_64bytes_gfni_avx2(input, output) + } + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")))] + // SAFETY: the feature is enabled at compile time. + unsafe { + bit_transpose_64bytes_avx2(input, output) + } #[cfg(target_arch = "aarch64")] // SAFETY: aarch64 always has NEON. unsafe { @@ -26,6 +43,7 @@ pub fn bit_transpose_64bytes(input: &[u8; 64], output: &mut [u8; 64]) { } #[cfg(not(any( all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"), + all(target_arch = "x86_64", target_feature = "avx2"), target_arch = "aarch64" )))] bit_transpose_64bytes_portable(input, output); @@ -54,6 +72,7 @@ fn transpose_8x8_bits(mut x: u64) -> u64 { #[cfg_attr( any( all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"), + all(target_arch = "x86_64", target_feature = "avx2"), target_arch = "aarch64" ), allow(dead_code) @@ -111,6 +130,80 @@ pub fn bit_transpose_64bytes_m512(input: core::arch::x86_64::__m512i) -> core::a } } +/// Gather each byte column of the eight 8-byte rows into a word, in the row order `order` picks. +/// +/// After two unpack rounds, lane `l` of register `w` holds columns `4w + 2l` and `4w + 2l + 1`, their bytes at: +/// +/// ```text +/// rows 0, 1, 4, 5 bytes 0..4 (first column), 4..8 (second) +/// rows 2, 3, 6, 7 bytes 8..12 (first column), 12..16 (second) +/// ``` +/// +/// So `order[8c + k]` is the byte that becomes byte `k` of the lane's column `c`. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +#[inline] +#[target_feature(enable = "avx2")] +fn gather_columns_avx2(input: &[u8; 64], order: &[u8; 16]) -> [core::arch::x86_64::__m256i; 2] { + use core::arch::x86_64::*; + + // SAFETY: the loads read the two halves of the 64-byte input and the 16-byte order. + let (v0, v1, order) = unsafe { + ( + _mm256_loadu_si256(input.as_ptr().cast()), + _mm256_loadu_si256(input.as_ptr().add(32).cast()), + _mm256_broadcastsi128_si256(_mm_loadu_si128(order.as_ptr().cast())), + ) + }; + // Lane 0: rows 0 and 4, then rows 1 and 5, byte-interleaved; lane 1: rows 2 and 6, then 3 and 7. + let (t0, t1) = (_mm256_unpacklo_epi8(v0, v1), _mm256_unpackhi_epi8(v0, v1)); + // Dword `c` of lane 0 is column `c` of rows 0, 1, 4, 5; of lane 1, of rows 2, 3, 6, 7. + let (u0, u1) = (_mm256_unpacklo_epi8(t0, t1), _mm256_unpackhi_epi8(t0, t1)); + // Bring both halves of two columns into one lane, then order their bytes. + [u0, u1].map(|u| _mm256_shuffle_epi8(_mm256_permute4x64_epi64::<0b11_01_10_00>(u), order)) +} + +/// [`gather_columns_avx2`] with the rows reversed, then the affine map of [`bit_transpose_64bytes_m512`]. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] +#[cfg_attr(target_feature = "avx512vbmi", allow(dead_code))] +#[inline] +#[target_feature(enable = "avx2,gfni")] +fn bit_transpose_64bytes_gfni_avx2(input: &[u8; 64], output: &mut [u8; 64]) { + use core::arch::x86_64::*; + + const REVERSED: [u8; 16] = [11, 10, 3, 2, 9, 8, 1, 0, 15, 14, 7, 6, 13, 12, 5, 4]; + let unit = _mm256_set1_epi64x(0x8040_2010_0804_0201_u64 as i64); + for (i, columns) in gather_columns_avx2(input, &REVERSED).into_iter().enumerate() { + let rows = _mm256_gf2p8affine_epi64_epi8::<0>(unit, columns); + // SAFETY: the two stores fill the 64-byte output. + unsafe { _mm256_storeu_si256(output.as_mut_ptr().add(32 * i).cast(), rows) }; + } +} + +/// [`gather_columns_avx2`] in row order, then the three masked-swap rounds of [`transpose_8x8_bits`] on four words. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +#[cfg_attr(target_feature = "gfni", allow(dead_code))] +#[inline] +#[target_feature(enable = "avx2")] +fn bit_transpose_64bytes_avx2(input: &[u8; 64], output: &mut [u8; 64]) { + use core::arch::x86_64::*; + + const NATURAL: [u8; 16] = [0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15]; + #[inline] + #[target_feature(enable = "avx2")] + fn swap(x: __m256i, mask: u64) -> __m256i { + let mask = _mm256_set1_epi64x(mask as i64); + let t = _mm256_and_si256(_mm256_xor_si256(x, _mm256_srli_epi64::(x)), mask); + _mm256_xor_si256(x, _mm256_xor_si256(t, _mm256_slli_epi64::(t))) + } + for (i, x) in gather_columns_avx2(input, &NATURAL).into_iter().enumerate() { + let x = swap::<7>(x, 0x00AA_00AA_00AA_00AA); + let x = swap::<14>(x, 0x0000_CCCC_0000_CCCC); + let x = swap::<28>(x, 0x0000_0000_F0F0_F0F0); + // SAFETY: the two stores fill the 64-byte output. + unsafe { _mm256_storeu_si256(output.as_mut_ptr().add(32 * i).cast(), x) }; + } +} + /// `vqtbl4q_u8` gathers each byte column into a word, then three masked-swap rounds transpose its bits. #[cfg(target_arch = "aarch64")] #[inline(always)] @@ -291,6 +384,23 @@ mod tests { unsafe { bit_transpose_64bytes_gfni(&input, &mut got) }; assert_eq!(got, want, "gfni"); } + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "gfni", + not(target_feature = "avx512vbmi") + ))] + { + // SAFETY: compiled only with the features enabled. + unsafe { bit_transpose_64bytes_gfni_avx2(&input, &mut got) }; + assert_eq!(got, want, "gfni_avx2"); + } + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] + { + // SAFETY: compiled only with the feature enabled. + unsafe { bit_transpose_64bytes_avx2(&input, &mut got) }; + assert_eq!(got, want, "avx2"); + } #[cfg(target_arch = "aarch64")] { // SAFETY: aarch64 always has NEON. diff --git a/crates/experimental_prover/primitives/src/field/f128x8.rs b/crates/experimental_prover/primitives/src/field/f128x8.rs index f94bf02909..597b3c6fcd 100644 --- a/crates/experimental_prover/primitives/src/field/f128x8.rs +++ b/crates/experimental_prover/primitives/src/field/f128x8.rs @@ -19,14 +19,22 @@ use super::gf2_64x2::{F128, F128Unreduced}; #[cfg(all( target_arch = "x86_64", - target_feature = "vpclmulqdq", target_feature = "avx2", + target_feature = "pclmulqdq", not(target_feature = "avx512f") ))] pub use avx2::{F64x8, F128x8, Prod8}; #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] pub use avx512::{Acc8, F64x8, F128x8}; -#[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2")))] +#[cfg(not(any( + all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + not(target_feature = "avx512f") + ), + all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f") +)))] pub use portable::{F64x8, F128x8, Prod8}; #[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f")))] pub use sum::Acc8; @@ -36,8 +44,16 @@ pub use sum::Acc8; #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] pub type Prod8 = Acc8; -/// The eight-lane types on a target without the wide multiplier. -#[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2")))] +/// The eight-lane types on a target without the `x4` or `x8` multipliers. +#[cfg(not(any( + all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + not(target_feature = "avx512f") + ), + all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f") +)))] mod portable { use super::sum::halves; use super::{F64, F128, F128Unreduced}; @@ -318,12 +334,12 @@ pub fn xor_into8(out: &mut [F128; 8], add: &[F128; 8]) { } } -/// The eight-lane types with VPCLMULQDQ but no AVX-512: two halves of four lanes, each the `x4` form, a -/// YMM plane per coefficient, so a product is lane-wise, two lanes to an instruction. +/// The eight-lane types with AVX2 but no AVX-512: two halves of four lanes, each the `x4` form, a YMM plane per +/// coefficient, so a product is lane-wise, two lanes to an instruction with VPCLMULQDQ and one without. #[cfg(all( target_arch = "x86_64", - target_feature = "vpclmulqdq", target_feature = "avx2", + target_feature = "pclmulqdq", not(target_feature = "avx512f") ))] mod avx2 { @@ -409,6 +425,47 @@ mod avx2 { let ((ae, ao), (be, bo)) = (split(a), split(b)); (Self([ae, be]), Self([ao, bo])) } + + /// Eight runs of eight consecutive elements, transposed: `out[j]` lane `p` is `src[8p + j]`. + #[inline(always)] + pub fn load_runs(src: &[F128; 64]) -> [Self; 8] { + let at = |p: usize, j: usize| src[8 * p + j..].as_ptr().cast::<__m128i>(); + std::array::from_fn(|j| { + Self(std::array::from_fn(|h| { + // Runs `4h + i` and `4h + i + 2` in one register; the unpacks then put lane `i` in qword `i`. + // SAFETY: the module's cfg enables avx2; each address is one element, 16 readable bytes. + let pair = |i: usize| unsafe { _mm256_loadu2_m128i(at(4 * h + i + 2, j), at(4 * h + i, j)) }; + let (a, b) = (pair(0), pair(1)); + // SAFETY: the module's cfg enables avx2. + F128x4(unsafe { [_mm256_unpacklo_epi64(a, b), _mm256_unpackhi_epi64(a, b)] }) + })) + }) + } + + /// Stores the lanes `a0 b0 a1 b1 … a7 b7`. + #[inline(always)] + pub fn store_interleaved(a: Self, b: Self, dst: &mut [F128; 16]) { + let q = dst.as_mut_ptr().cast::<__m256i>(); + for h in 0..2 { + let ([a0, a1], [b0, b1]) = (a.0[h].0, b.0[h].0); + // SAFETY: the module's cfg enables avx2; `dst` is eight writable 32-byte pairs of elements. + unsafe { + // Elements `[x0, x2]` and `[x1, x3]` of each. + let (al, ah) = (_mm256_unpacklo_epi64(a0, a1), _mm256_unpackhi_epi64(a0, a1)); + let (bl, bh) = (_mm256_unpacklo_epi64(b0, b1), _mm256_unpackhi_epi64(b0, b1)); + _mm256_storeu_si256(q.add(4 * h), _mm256_permute2x128_si256::<0x20>(al, bl)); + _mm256_storeu_si256(q.add(4 * h + 1), _mm256_permute2x128_si256::<0x20>(ah, bh)); + _mm256_storeu_si256(q.add(4 * h + 2), _mm256_permute2x128_si256::<0x31>(al, bl)); + _mm256_storeu_si256(q.add(4 * h + 3), _mm256_permute2x128_si256::<0x31>(ah, bh)); + } + } + } + + /// Lanes `0..4` and `4..8`. + #[inline(always)] + pub fn halves(self) -> [F128x4; 2] { + self.0 + } } impl core::ops::Add for F128x8 { diff --git a/crates/experimental_prover/primitives/src/field/gf2_64x2.rs b/crates/experimental_prover/primitives/src/field/gf2_64x2.rs index 83443b02ef..cc1d06d5a7 100644 --- a/crates/experimental_prover/primitives/src/field/gf2_64x2.rs +++ b/crates/experimental_prover/primitives/src/field/gf2_64x2.rs @@ -410,7 +410,7 @@ impl F128Unreduced { } /// Build from two 128-bit coefficients given as `[low, high]` words. - #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2"))] + #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq", target_feature = "avx2"))] #[inline] pub(crate) const fn from_coeffs(coeffs: [[u64; 2]; 2]) -> Self { Self { coeffs } diff --git a/crates/experimental_prover/primitives/src/field/mod.rs b/crates/experimental_prover/primitives/src/field/mod.rs index 5dafc4d88d..6fd0afcf20 100644 --- a/crates/experimental_prover/primitives/src/field/mod.rs +++ b/crates/experimental_prover/primitives/src/field/mod.rs @@ -18,7 +18,7 @@ pub mod gf2_64x2; #[cfg(all( target_arch = "x86_64", target_feature = "avx2", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", not(target_feature = "avx512f") ))] pub mod x4; diff --git a/crates/experimental_prover/primitives/src/field/x4.rs b/crates/experimental_prover/primitives/src/field/x4.rs index 7a63f4b7ff..8e29126f65 100644 --- a/crates/experimental_prover/primitives/src/field/x4.rs +++ b/crates/experimental_prover/primitives/src/field/x4.rs @@ -1,9 +1,10 @@ -//! Four lanes of products in YMM registers, for a machine with VPCLMULQDQ but no AVX-512: the -//! 256-bit counterpart of the AVX-512 `x8`'s `F64x8`, `F128x8` and `Mixed8` for the leaf builder, -//! and the halves of the eight-lane `f128x8` types. +//! Four lanes of products in YMM registers, for a machine with AVX2 but no AVX-512: the 256-bit +//! counterpart of the AVX-512 `x8`'s `F64x8`, `F128x8` and `Mixed8` for the leaf builder, and the +//! halves of the eight-lane `f128x8` types. Without VPCLMULQDQ a 256-bit carry-less product is two +//! 128-bit ones. use core::arch::x86_64::*; -use core::ops::{Add, BitXorAssign, Mul}; +use core::ops::{Add, BitXor, BitXorAssign, Mul}; use super::gf2_64::F64; use super::gf2_64x2::x86_64::{fold, reduce_halves256 as reduce_halves, times_x256}; @@ -26,17 +27,32 @@ pub struct Mixed4 { odd: [__m256i; 2], } +/// The sum over lanes of unreduced products, y-folded: per coefficient, two 128-bit partials. +#[derive(Clone, Copy, Debug)] +pub struct Sum4([__m256i; 2]); + #[inline(always)] fn xor(a: __m256i, b: __m256i) -> __m256i { // SAFETY: the module's cfg enables avx2. unsafe { _mm256_xor_si256(a, b) } } -/// The products of the even lanes (`I = 0x00`) or the odd ones (`I = 0x11`) of two registers. +/// `vpclmulqdq` with immediate `I`: in each 128-bit lane, the product of the qwords `I` picks, bit 0 in `a` and bit 4 in +/// `b`. #[inline(always)] -fn clmul(a: __m256i, b: __m256i) -> __m256i { - // SAFETY: the module's cfg enables vpclmulqdq. - unsafe { _mm256_clmulepi64_epi128::(a, b) } +pub fn clmul(a: __m256i, b: __m256i) -> __m256i { + #[cfg(target_feature = "vpclmulqdq")] + // SAFETY: the cfg enables vpclmulqdq. + return unsafe { _mm256_clmulepi64_epi128::(a, b) }; + #[cfg(not(target_feature = "vpclmulqdq"))] + // SAFETY: the module's cfg enables avx2 and pclmulqdq. + unsafe { + let lane = |a: __m128i, b: __m128i| _mm_clmulepi64_si128::(a, b); + _mm256_set_m128i( + lane(_mm256_extracti128_si256::<1>(a), _mm256_extracti128_si256::<1>(b)), + lane(_mm256_castsi256_si128(a), _mm256_castsi256_si128(b)), + ) + } } /// The y-folded Karatsuba product of the even or the odd lanes. @@ -84,13 +100,8 @@ impl Mul for F64x4 { type Output = Self; #[inline(always)] fn mul(self, rhs: Self) -> Self { - // SAFETY: the module's cfg enables avx2 and vpclmulqdq. - Self(unsafe { - reduce_halves( - _mm256_clmulepi64_epi128::<0x00>(self.0, rhs.0), - _mm256_clmulepi64_epi128::<0x11>(self.0, rhs.0), - ) - }) + // SAFETY: the module's cfg enables avx2. + Self(unsafe { reduce_halves(clmul::<0x00>(self.0, rhs.0), clmul::<0x11>(self.0, rhs.0)) }) } } @@ -133,6 +144,57 @@ impl F128x4 { odd: karatsuba::<0x11>(self.0, rhs.0), } } + + /// The lane-wise products, summed over the lanes and left unreduced. Karatsuba's products are summed + /// before their one fold. + #[inline(always)] + pub fn mul_sum(self, rhs: Self) -> Sum4 { + let both = |x: __m256i, y: __m256i| xor(clmul::<0x00>(x, y), clmul::<0x11>(x, y)); + let ([a0, a1], [b0, b1]) = (self.0, rhs.0); + let (m0, m1, m01) = (both(a0, b0), both(a1, b1), both(xor(a0, a1), xor(b0, b1))); + // SAFETY: the module's cfg enables avx2. + Sum4(fold(m0, m1, m01, xor, |p| unsafe { times_x256(p) })) + } +} + +impl Mul for F128x4 { + type Output = Self; + #[inline(always)] + fn mul(self, rhs: Self) -> Self { + self.mul_unreduced(rhs).reduce() + } +} + +impl Sum4 { + // SAFETY: an all-zero bit pattern is a valid `__m256i`. + pub const ZERO: Self = Self(unsafe { std::mem::zeroed() }); + + /// The lane sum, reduced. + #[inline] + pub fn reduce(self) -> F128 { + let coeffs = self.0.map(|v| { + // SAFETY: both are 32 bytes. + let w: [u64; 4] = unsafe { std::mem::transmute(v) }; + [w[0] ^ w[2], w[1] ^ w[3]] + }); + F128Unreduced::from_coeffs(coeffs).reduce() + } +} + +impl BitXorAssign for Sum4 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + self.0 = std::array::from_fn(|k| xor(self.0[k], rhs.0[k])); + } +} + +impl BitXor for Sum4 { + type Output = Self; + #[inline(always)] + fn bitxor(mut self, rhs: Self) -> Self { + self ^= rhs; + self + } } /// The embedding of the base field. @@ -203,6 +265,236 @@ impl BitXorAssign for Mixed4 { } } +/// Two `E` elements in one register as they lie in memory: 128-bit lane `i` is `[c0, c1]` of element `i`. +/// +/// Products keep this layout from load to store, where the plane form of [`F128x4`] transposes on both sides. +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +pub struct F128x2(__m256i); + +/// Two `K` elements in one register, each repeated across its 128-bit lane, so that it meets either coefficient of +/// the [`F128x2`] lane it multiplies. +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +pub struct F64x2(__m256i); + +/// A [`F128x2`] multiplier with `c0 + c1` of each lane in its low qword, the operand Karatsuba's middle product reads. +#[derive(Clone, Copy, Debug)] +pub struct Mul2 { + v: __m256i, + s: __m256i, +} + +/// Unreduced sums of [`F128x2`] products: the Karatsuba partials `a0 b0`, `a1 b1` and `(a0 + a1)(b0 + b1)` of each +/// lane, y-folded once at the reduction. +#[derive(Clone, Copy, Debug)] +pub struct Prod2 { + p0: __m256i, + p1: __m256i, + p01: __m256i, +} + +/// Unreduced sums of mixed products `e * k`: the coefficients of `y^0` and `y^1` of each lane. +#[derive(Clone, Copy, Debug)] +pub struct Mixed2 { + d0: __m256i, + d1: __m256i, +} + +/// `[c1, c0]` of each lane: the two qwords swapped. +#[inline(always)] +fn swap_qwords(v: __m256i) -> __m256i { + // SAFETY: the module's cfg enables avx2. + unsafe { _mm256_shuffle_epi32::<0x4E>(v) } +} + +/// The 128-bit lanes `vperm2i128` picks with immediate `I`: per half of the result, `0..=3` names `a.lo`, `a.hi`, +/// `b.lo`, `b.hi`, and bit 3 zeroes it. +#[inline(always)] +fn select(a: __m256i, b: __m256i) -> __m256i { + // SAFETY: the module's cfg enables avx2. + unsafe { _mm256_permute2x128_si256::(a, b) } +} + +impl F128x2 { + pub const ZERO: Self = Self( + // SAFETY: an all-zero bit pattern is a valid `__m256i`. + unsafe { std::mem::zeroed() }, + ); + + #[inline(always)] + pub fn load(e: &[F128; 2]) -> Self { + // SAFETY: the module's cfg enables avx2; two elements are 32 bytes. + Self(unsafe { _mm256_loadu_si256(e.as_ptr().cast()) }) + } + + #[inline(always)] + pub fn store(self, dst: &mut [F128; 2]) { + // SAFETY: the module's cfg enables avx2; two elements are 32 bytes. + unsafe { _mm256_storeu_si256(dst.as_mut_ptr().cast(), self.0) } + } + + #[inline(always)] + pub fn new(lo: F128, hi: F128) -> Self { + // SAFETY: the module's cfg enables avx2. + Self(unsafe { _mm256_set_epi64x(hi.c1 as i64, hi.c0 as i64, lo.c1 as i64, lo.c0 as i64) }) + } + + #[inline(always)] + pub fn splat(e: F128) -> Self { + Self::new(e, e) + } + + /// The lanes [`select`] picks from `a` and `b` with immediate `I`. + #[inline(always)] + pub fn select(a: Self, b: Self) -> Self { + Self(select::(a.0, b.0)) + } + + /// The coefficients `c0` of two pairs of lanes, then their coefficients `c1`, in lane order. + #[inline(always)] + pub fn limbs(a: Self, b: Self) -> (F64x4, F64x4) { + // SAFETY: the module's cfg enables avx2. + unsafe { + ( + F64x4(_mm256_permute4x64_epi64::<0xD8>(_mm256_unpacklo_epi64(a.0, b.0))), + F64x4(_mm256_permute4x64_epi64::<0xD8>(_mm256_unpackhi_epi64(a.0, b.0))), + ) + } + } + + /// This pair as a multiplier. + #[inline(always)] + pub fn factor(self) -> Mul2 { + Mul2 { + v: self.0, + s: xor(self.0, swap_qwords(self.0)), + } + } + + /// The lane-wise products by `m`, unreduced. + #[inline(always)] + pub fn mul_unreduced(self, m: Mul2) -> Prod2 { + let s = xor(self.0, swap_qwords(self.0)); + Prod2 { + p0: clmul::<0x00>(self.0, m.v), + p1: clmul::<0x11>(self.0, m.v), + p01: clmul::<0x00>(s, m.s), + } + } + + /// The lane-wise mixed products by `k`, unreduced. + #[inline(always)] + pub fn mul_base_unreduced(self, k: F64x2) -> Mixed2 { + // Immediate 0x01 multiplies the high qword of the first operand by the low qword of the second. + Mixed2 { + d0: clmul::<0x00>(self.0, k.0), + d1: clmul::<0x01>(self.0, k.0), + } + } +} + +impl Add for F128x2 { + type Output = Self; + #[inline(always)] + fn add(self, rhs: Self) -> Self { + Self(xor(self.0, rhs.0)) + } +} + +/// The embedding of the base field. +impl From for F128x2 { + #[inline(always)] + fn from(k: F64x2) -> Self { + // SAFETY: the module's cfg enables avx2. + Self(unsafe { _mm256_blend_epi32::<0xCC>(k.0, _mm256_setzero_si256()) }) + } +} + +impl F64x2 { + #[inline(always)] + pub fn new(lo: F64, hi: F64) -> Self { + // SAFETY: the module's cfg enables avx2. + Self(unsafe { _mm256_set_epi64x(hi.0 as i64, hi.0 as i64, lo.0 as i64, lo.0 as i64) }) + } + + #[inline(always)] + pub fn splat(k: F64) -> Self { + Self::new(k, k) + } + + /// The lanes [`select`] picks from `a` and `b` with immediate `I`. + #[inline(always)] + pub fn select(a: Self, b: Self) -> Self { + Self(select::(a.0, b.0)) + } +} + +impl Add for F64x2 { + type Output = Self; + #[inline(always)] + fn add(self, rhs: Self) -> Self { + Self(xor(self.0, rhs.0)) + } +} + +impl F64x4 { + /// Lanes `0, 1` and lanes `2, 3`, each as a [`F64x2`]. + #[inline(always)] + pub fn spread(self) -> [F64x2; 2] { + // SAFETY: the module's cfg enables avx2. + unsafe { + [ + F64x2(_mm256_permute4x64_epi64::<0x50>(self.0)), + F64x2(_mm256_permute4x64_epi64::<0xFA>(self.0)), + ] + } + } + + #[inline(always)] + pub fn store(self, dst: &mut [F64; 4]) { + // SAFETY: the module's cfg enables avx2; four qwords are 32 bytes. + unsafe { _mm256_storeu_si256(dst.as_mut_ptr().cast(), self.0) } + } +} + +impl Prod2 { + /// Both lanes reduced. + #[inline(always)] + pub fn reduce(self) -> F128x2 { + // SAFETY: the module's cfg enables avx2. + let [d0, d1] = fold(self.p0, self.p1, self.p01, xor, |p| unsafe { times_x256(p) }); + // SAFETY: the module's cfg enables avx2. + F128x2(unsafe { reduce_halves(d0, d1) }) + } +} + +impl BitXorAssign for Prod2 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + self.p0 = xor(self.p0, rhs.p0); + self.p1 = xor(self.p1, rhs.p1); + self.p01 = xor(self.p01, rhs.p01); + } +} + +impl Mixed2 { + /// Both lanes reduced. + #[inline(always)] + pub fn reduce(self) -> F128x2 { + // SAFETY: the module's cfg enables avx2. + F128x2(unsafe { reduce_halves(self.d0, self.d1) }) + } +} + +impl BitXorAssign for Mixed2 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + self.d0 = xor(self.d0, rhs.d0); + self.d1 = xor(self.d1, rhs.d1); + } +} + #[cfg(test)] mod tests { use super::*; @@ -248,4 +540,45 @@ mod tests { assert_eq!(F128x4::from(F64x4::new(k)).to_array(), k.map(F128::from)); } } + + #[test] + fn pairs_match_scalar() { + let arr = |v: F128x2| { + let mut out = [F128::ZERO; 2]; + v.store(&mut out); + out + }; + let mut rng = Rng::new(31); + for _ in 0..200 { + let (a, b, c, d): ([F128; 2], [F128; 2], [F128; 2], [F128; 2]) = ( + [rng.ext(), F128::new(u64::MAX, 1 << 63)], + [rng.ext(), rng.ext()], + [rng.ext(), rng.ext()], + [rng.ext(), rng.ext()], + ); + let k: [F64; 4] = std::array::from_fn(|_| F64(rng.next_u64())); + let (va, vb, vc, vd) = (F128x2::load(&a), F128x2::load(&b), F128x2::load(&c), F128x2::load(&d)); + let mut acc = va.mul_unreduced(vb.factor()); + acc ^= vc.mul_unreduced(vd.factor()); + assert_eq!(arr(acc.reduce()), std::array::from_fn(|i| a[i] * b[i] + c[i] * d[i])); + let [k01, k23] = F64x4::load(&k).spread(); + let mut mixed = va.mul_base_unreduced(k01); + mixed ^= vb.mul_base_unreduced(k23); + let want = std::array::from_fn(|i| a[i].mul_base(k[i]) + b[i].mul_base(k[2 + i]) + F128::from(k[i])); + assert_eq!(arr(mixed.reduce() + F128x2::from(k01)), want); + assert_eq!(arr(F128x2::select::<0x21>(va, vb)), [a[1], b[0]]); + let (lo, hi) = F128x2::limbs(va, vb); + assert_eq!(lo.to_array(), [a[0].c0, a[1].c0, b[0].c0, b[1].c0].map(F64)); + assert_eq!(hi.to_array(), [a[0].c1, a[1].c1, b[0].c1, b[1].c1].map(F64)); + let mut stored = [F64::ZERO; 4]; + F64x4::load(&k).store(&mut stored); + assert_eq!(stored, k); + assert_eq!( + arr(F128x2::new(a[0], a[1]) + F128x2::splat(c[0])), + [a[0] + c[0], a[1] + c[0]] + ); + let sum = F64x2::new(k[0], k[1]) + F64x2::select::<0x01>(F64x2::splat(k[2]), F64x2::splat(k[2])); + assert_eq!(arr(F128x2::from(sum)), [k[0] + k[2], k[1] + k[2]].map(F128::from)); + } + } } diff --git a/crates/fiat_shamir_prover/src/lib.rs b/crates/fiat_shamir_prover/src/lib.rs index d1d67edcaf..748e9a52ad 100644 --- a/crates/fiat_shamir_prover/src/lib.rs +++ b/crates/fiat_shamir_prover/src/lib.rs @@ -170,19 +170,17 @@ impl FiatShamirState { // first lane. let mut base = self.clone(); base.absorb(F::from_limbs([0, 0]), DS_POW_NONCE); - let (base, lane) = (base.state, 4 * (base.n_pending - 1)); + let search = match 4 * (base.n_pending - 1) { + 0 => keccak256::first_nonce::<0>, + 4 => keccak256::first_nonce::<4>, + 8 => keccak256::first_nonce::<8>, + 12 => keccak256::first_nonce::<12>, + lane => unreachable!("a scalar's first lane in the rate, not {lane}"), + }; + let base = base.state; // The smallest passing nonce of batch `b`, the nonces `b * BATCH..(b + 1) * BATCH`. const BATCH: u64 = 64; - let batch = |b: u64| { - let mut states = [base; BATCH as usize]; - for (nonce, state) in (b * BATCH..).zip(&mut states) { - state[lane] = nonce; - } - keccak256::keccak_f_many(&mut states); - let pass = - |&(nonce, state): &(u64, &[u64; STATE_LANES])| nonce >= from && state[2] & ((1 << bits) - 1) == 0; - (b * BATCH..).zip(&states).find(pass).map(|(nonce, _)| nonce) - }; + let batch = |b: u64| search(&base, (b * BATCH).max(from)..(b + 1) * BATCH, 2, (1 << bits) - 1); // A short search, a few batches, is not worth a dispatch to the pool. let nonce = if bits < 10 { (from / BATCH..).find_map(batch) diff --git a/crates/keccak256/src/lanes.rs b/crates/keccak256/src/lanes.rs index 50e5ccee2f..b0db27a44a 100644 --- a/crates/keccak256/src/lanes.rs +++ b/crates/keccak256/src/lanes.rs @@ -2,7 +2,8 @@ //! vector register per state lane: AVX-512 (eight states), AVX2 (four), NEON (two, with the SHA3 instructions where //! the target has them), or one at a time elsewhere. Every batched function is byte-identical to its one-input form. -use crate::{END_BIT, OUT_LEN, PAD_FIRST, RATE, RATE_LANES, RC, STATE_LANES, hash_from_state, keccak_f}; +use crate::{END_BIT, OUT_LEN, PAD_FIRST, RATE, RATE_LANES, RC, ROUNDS, STATE_LANES, hash_from_state, keccak_f}; +use core::ops::Range; /// One state lane across [`Lane::WIDTH`] states. The permutation is written once over this trait. pub(crate) trait Lane: Copy { @@ -37,38 +38,76 @@ pub(crate) trait Lane: Copy { /// the lanes χ writes, which schedules better than forming every lane before χ and every parity after it. #[inline(always)] pub(crate) fn permute(a: &mut [S; STATE_LANES]) { - let mut c: [S; 5] = core::array::from_fn(|x| S::xor5(a[x], a[x + 5], a[x + 10], a[x + 15], a[x + 20])); + let mut c = parities(a); for &rc in &RC { - let d: [S; 5] = core::array::from_fn(|x| S::theta_d(c[(x + 4) % 5], c[(x + 1) % 5])); - let t = |i: usize| a[i].xor(d[i % 5]); - // Row `y` holds the lanes π sends there, each rotated by its ρ. - #[rustfmt::skip] - let rows: [[S; 5]; 5] = [ - [t(0), t(6).rotl::<44>(), t(12).rotl::<43>(), t(18).rotl::<21>(), t(24).rotl::<14>()], - [t(3).rotl::<28>(), t(9).rotl::<20>(), t(10).rotl::<3>(), t(16).rotl::<45>(), t(22).rotl::<61>()], - [t(1).rotl::<1>(), t(7).rotl::<6>(), t(13).rotl::<25>(), t(19).rotl::<8>(), t(20).rotl::<18>()], - [t(4).rotl::<27>(), t(5).rotl::<36>(), t(11).rotl::<10>(), t(17).rotl::<15>(), t(23).rotl::<56>()], - [t(2).rotl::<62>(), t(8).rotl::<55>(), t(14).rotl::<39>(), t(15).rotl::<41>(), t(21).rotl::<2>()], + round(a, &mut c, rc); + } +} + +/// The column parities θ starts from. +#[inline(always)] +fn parities(a: &[S; STATE_LANES]) -> [S; 5] { + core::array::from_fn(|x| S::xor5(a[x], a[x + 5], a[x + 10], a[x + 15], a[x + 20])) +} + +/// Row `y` before χ, from `t`, the lanes after θ: the lanes π sends there, each rotated by its ρ. +#[inline(always)] +fn row(y: usize, t: impl Fn(usize) -> S) -> [S; 5] { + #[rustfmt::skip] + let row = match y { + 0 => [t(0), t(6).rotl::<44>(), t(12).rotl::<43>(), t(18).rotl::<21>(), t(24).rotl::<14>()], + 1 => [t(3).rotl::<28>(), t(9).rotl::<20>(), t(10).rotl::<3>(), t(16).rotl::<45>(), t(22).rotl::<61>()], + 2 => [t(1).rotl::<1>(), t(7).rotl::<6>(), t(13).rotl::<25>(), t(19).rotl::<8>(), t(20).rotl::<18>()], + 3 => [t(4).rotl::<27>(), t(5).rotl::<36>(), t(11).rotl::<10>(), t(17).rotl::<15>(), t(23).rotl::<56>()], + _ => [t(2).rotl::<62>(), t(8).rotl::<55>(), t(14).rotl::<39>(), t(15).rotl::<41>(), t(21).rotl::<2>()], + }; + row +} + +/// One round of θ, ρ, π, χ and ι, `c` holding the column parities of `a` before and after. +#[inline(always)] +fn round(a: &mut [S; STATE_LANES], c: &mut [S; 5], rc: u64) { + let d: [S; 5] = core::array::from_fn(|x| S::theta_d(c[(x + 4) % 5], c[(x + 1) % 5])); + let t = |i: usize| a[i].xor(d[i % 5]); + let rows: [[S; 5]; 5] = core::array::from_fn(|y| row(y, t)); + for (y, &[b0, b1, b2, b3, b4]) in rows.iter().enumerate() { + let mut row = [ + S::chi(b0, b1, b2), + S::chi(b1, b2, b3), + S::chi(b2, b3, b4), + S::chi(b3, b4, b0), + S::chi(b4, b0, b1), ]; - for (y, &[b0, b1, b2, b3, b4]) in rows.iter().enumerate() { - let mut row = [ - S::chi(b0, b1, b2), - S::chi(b1, b2, b3), - S::chi(b2, b3, b4), - S::chi(b3, b4, b0), - S::chi(b4, b0, b1), - ]; - if y == 0 { - row[0] = row[0].xor(S::splat(rc)); - } - for x in 0..5 { - a[5 * y + x] = row[x]; - c[x] = if y == 0 { row[x] } else { c[x].xor(row[x]) }; - } + if y == 0 { + row[0] = row[0].xor(S::splat(rc)); + } + for x in 0..5 { + a[5 * y + x] = row[x]; + c[x] = if y == 0 { row[x] } else { c[x].xor(row[x]) }; } } } +/// Lane `out` of the permutation of `a`, whose column parities are `c`, with the last round computing that lane alone. +/// The first round stands alone, so a caller permuting many states that differ in a few lanes can hoist the work on +/// the others out of its loop. +#[inline(always)] +fn permute_to_lane(a: &mut [S; STATE_LANES], mut c: [S; 5], out: usize) -> S { + round(a, &mut c, RC[0]); + for &rc in &RC[1..ROUNDS - 1] { + round(a, &mut c, rc); + } + let d: [S; 5] = core::array::from_fn(|x| S::theta_d(c[(x + 4) % 5], c[(x + 1) % 5])); + let b = row(out / 5, |i: usize| a[i].xor(d[i % 5])); + let x = out % 5; + let lane = S::chi(b[x], b[(x + 1) % 5], b[(x + 2) % 5]); + if out == 0 { + lane.xor(S::splat(RC[ROUNDS - 1])) + } else { + lane + } +} + /// One state: the scalar permutation, and the batched one where no vector backend applies. impl Lane for u64 { const WIDTH: usize = 1; @@ -246,6 +285,34 @@ pub fn digest_halves_from(template: &[u64; STATE_LANES], at: usize, values: &[u1 } } +/// The first of `nonces` that, written into lane `AT` of `template`, leaves lane `out` of the permuted state zero under +/// `mask`, [`LANES`] nonces at a time. `AT` is a constant so that the work on the lanes no nonce touches, the same in +/// every state, leaves the loop; the last round computes lane `out` alone. +pub fn first_nonce( + template: &[u64; STATE_LANES], + nonces: Range, + out: usize, + mask: u64, +) -> Option { + const { assert!(AT < STATE_LANES) }; + assert!(out < STATE_LANES); + let template: [Backend; STATE_LANES] = core::array::from_fn(|i| Backend::splat(template[i])); + let mut first = nonces.start; + while first < nonces.end { + let mut st = template; + st[AT] = Backend::load(&core::array::from_fn::(|l| first.wrapping_add(l as u64))); + let c = parities(&st); + let mut words = [0; LANES]; + permute_to_lane(&mut st, c, out).store(&mut words); + let n = (nonces.end - first).min(LANES as u64) as usize; + if let Some(l) = words[..n].iter().position(|w| w & mask == 0) { + return Some(first + l as u64); + } + first = first.saturating_add(LANES as u64); + } + None +} + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] mod x86 { use super::Lane; diff --git a/crates/keccak256/src/lib.rs b/crates/keccak256/src/lib.rs index a590cf5654..115129d654 100644 --- a/crates/keccak256/src/lib.rs +++ b/crates/keccak256/src/lib.rs @@ -3,7 +3,7 @@ //! tree. //! //! - [`keccak_f`], the Keccak-f\[1600\] permutation, which the VM's `KECCAK` opcode computes and `keccak_gkr` -//! proves, and [`keccak_f_many`], its batched form. +//! proves, and [`keccak_f_many`], its batched form; [`first_nonce`], the provers' proof-of-work search over it. //! - [`hash`] / [`Hasher`], Keccak-256 over bytes, and [`hash_many`], its batched form over equal-length inputs. //! - [`positions`], the permutation one bit position at a time, as `keccak_gkr` proves it. //! @@ -91,7 +91,7 @@ pub const END_BIT: u64 = (PAD_LAST as u64) << 56; mod lanes; pub mod positions; -pub use lanes::{LANES, digest_halves_from, hash_many, hash_many_from_state, keccak_f_many}; +pub use lanes::{LANES, digest_halves_from, first_nonce, hash_many, hash_many_from_state, keccak_f_many}; #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] mod single_avx512; @@ -367,4 +367,30 @@ mod tests { assert_eq!(got, want); } } + + /// The batched nonce search, from unaligned starts and to unaligned ends, against permuting each candidate. + #[test] + fn first_nonce_matches_keccak_f() { + fn check(out: usize, mask: u64) { + let template: [u64; STATE_LANES] = std::array::from_fn(|i| (i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15)); + let pass = |nonce: u64| { + let mut state = template; + state[AT] = nonce; + keccak_f(&mut state); + state[out] & mask == 0 + }; + for (start, end) in [(0, 200), (3, 61), (17, 18), (5, 5), (u64::MAX - 70, u64::MAX)] { + let want = (start..end).find(|&nonce| pass(nonce)); + assert_eq!( + first_nonce::(&template, start..end, out, mask), + want, + "{AT} {out} {start}..{end}" + ); + } + } + check::<0>(2, 0xf); + check::<4>(0, 0x7); + check::<12>(2, 0x1f); + check::<24>(24, 0x3); + } }