From e5fe852cad4fd12689daeec6a510382dc1a27857 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:02:58 +0000 Subject: [PATCH 01/18] experimental primitives: AVX2 arms for bit_transpose_64bytes Without AVX-512 VBMI, gather each byte column with two unpack rounds, a qword permute and a byte shuffle, then transpose the bits of four columns per YMM: with GFNI one affine map (rows reversed by the gather), without it the three masked-swap rounds. The consumers are the PCS level-0 projection and the Keccak openings. Throwaway microbench, one 64-byte block, single thread on an EPYC 9454P (Zen 4) core: portable 7.43 ns, AVX2 (haswell) 4.48 ns, AVX2+GFNI 1.33 ns. --- .../primitives/src/bits.rs | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/crates/experimental_prover/primitives/src/bits.rs b/crates/experimental_prover/primitives/src/bits.rs index 1a3932bd03..2a287cffee 100644 --- a/crates/experimental_prover/primitives/src/bits.rs +++ b/crates/experimental_prover/primitives/src/bits.rs @@ -10,6 +10,8 @@ /// So byte column `b` of the eight input rows becomes output row `b`, bit-transposed. /// /// - AVX-512 VBMI with GFNI: one byte permute and one affine map. +/// - AVX2 with GFNI: a byte gather in unpacks and shuffles, then one affine map. +/// - AVX2: the same gather, then three masked-swap rounds. /// - aarch64: a table lookup, then three masked-swap rounds. /// - Elsewhere: the same rounds on a gathered word per column. #[inline] @@ -19,6 +21,21 @@ pub fn bit_transpose_64bytes(input: &[u8; 64], output: &mut [u8; 64]) { unsafe { bit_transpose_64bytes_gfni(input, output) } + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "gfni", + not(target_feature = "avx512vbmi") + ))] + // SAFETY: the features are enabled at compile time. + unsafe { + bit_transpose_64bytes_gfni_avx2(input, output) + } + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")))] + // SAFETY: the feature is enabled at compile time. + unsafe { + bit_transpose_64bytes_avx2(input, output) + } #[cfg(target_arch = "aarch64")] // SAFETY: aarch64 always has NEON. unsafe { @@ -26,6 +43,7 @@ pub fn bit_transpose_64bytes(input: &[u8; 64], output: &mut [u8; 64]) { } #[cfg(not(any( all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"), + all(target_arch = "x86_64", target_feature = "avx2"), target_arch = "aarch64" )))] bit_transpose_64bytes_portable(input, output); @@ -54,6 +72,7 @@ fn transpose_8x8_bits(mut x: u64) -> u64 { #[cfg_attr( any( all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"), + all(target_arch = "x86_64", target_feature = "avx2"), target_arch = "aarch64" ), allow(dead_code) @@ -111,6 +130,80 @@ pub fn bit_transpose_64bytes_m512(input: core::arch::x86_64::__m512i) -> core::a } } +/// Gather each byte column of the eight 8-byte rows into a word, in the row order `order` picks. +/// +/// After two unpack rounds, lane `l` of register `w` holds columns `4w + 2l` and `4w + 2l + 1`, their bytes at: +/// +/// ```text +/// rows 0, 1, 4, 5 bytes 0..4 (first column), 4..8 (second) +/// rows 2, 3, 6, 7 bytes 8..12 (first column), 12..16 (second) +/// ``` +/// +/// So `order[8c + k]` is the byte that becomes byte `k` of the lane's column `c`. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +#[inline] +#[target_feature(enable = "avx2")] +fn gather_columns_avx2(input: &[u8; 64], order: &[u8; 16]) -> [core::arch::x86_64::__m256i; 2] { + use core::arch::x86_64::*; + + // SAFETY: the loads read the two halves of the 64-byte input and the 16-byte order. + let (v0, v1, order) = unsafe { + ( + _mm256_loadu_si256(input.as_ptr().cast()), + _mm256_loadu_si256(input.as_ptr().add(32).cast()), + _mm256_broadcastsi128_si256(_mm_loadu_si128(order.as_ptr().cast())), + ) + }; + // Lane 0: rows 0 and 4, then rows 1 and 5, byte-interleaved; lane 1: rows 2 and 6, then 3 and 7. + let (t0, t1) = (_mm256_unpacklo_epi8(v0, v1), _mm256_unpackhi_epi8(v0, v1)); + // Dword `c` of lane 0 is column `c` of rows 0, 1, 4, 5; of lane 1, of rows 2, 3, 6, 7. + let (u0, u1) = (_mm256_unpacklo_epi8(t0, t1), _mm256_unpackhi_epi8(t0, t1)); + // Bring both halves of two columns into one lane, then order their bytes. + [u0, u1].map(|u| _mm256_shuffle_epi8(_mm256_permute4x64_epi64::<0b11_01_10_00>(u), order)) +} + +/// [`gather_columns_avx2`] with the rows reversed, then the affine map of [`bit_transpose_64bytes_m512`]. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] +#[cfg_attr(target_feature = "avx512vbmi", allow(dead_code))] +#[inline] +#[target_feature(enable = "avx2,gfni")] +fn bit_transpose_64bytes_gfni_avx2(input: &[u8; 64], output: &mut [u8; 64]) { + use core::arch::x86_64::*; + + const REVERSED: [u8; 16] = [11, 10, 3, 2, 9, 8, 1, 0, 15, 14, 7, 6, 13, 12, 5, 4]; + let unit = _mm256_set1_epi64x(0x8040_2010_0804_0201_u64 as i64); + for (i, columns) in gather_columns_avx2(input, &REVERSED).into_iter().enumerate() { + let rows = _mm256_gf2p8affine_epi64_epi8::<0>(unit, columns); + // SAFETY: the two stores fill the 64-byte output. + unsafe { _mm256_storeu_si256(output.as_mut_ptr().add(32 * i).cast(), rows) }; + } +} + +/// [`gather_columns_avx2`] in row order, then the three masked-swap rounds of [`transpose_8x8_bits`] on four words. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +#[cfg_attr(target_feature = "gfni", allow(dead_code))] +#[inline] +#[target_feature(enable = "avx2")] +fn bit_transpose_64bytes_avx2(input: &[u8; 64], output: &mut [u8; 64]) { + use core::arch::x86_64::*; + + const NATURAL: [u8; 16] = [0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15]; + #[inline] + #[target_feature(enable = "avx2")] + fn swap(x: __m256i, mask: u64) -> __m256i { + let mask = _mm256_set1_epi64x(mask as i64); + let t = _mm256_and_si256(_mm256_xor_si256(x, _mm256_srli_epi64::(x)), mask); + _mm256_xor_si256(x, _mm256_xor_si256(t, _mm256_slli_epi64::(t))) + } + for (i, x) in gather_columns_avx2(input, &NATURAL).into_iter().enumerate() { + let x = swap::<7>(x, 0x00AA_00AA_00AA_00AA); + let x = swap::<14>(x, 0x0000_CCCC_0000_CCCC); + let x = swap::<28>(x, 0x0000_0000_F0F0_F0F0); + // SAFETY: the two stores fill the 64-byte output. + unsafe { _mm256_storeu_si256(output.as_mut_ptr().add(32 * i).cast(), x) }; + } +} + /// `vqtbl4q_u8` gathers each byte column into a word, then three masked-swap rounds transpose its bits. #[cfg(target_arch = "aarch64")] #[inline(always)] @@ -291,6 +384,23 @@ mod tests { unsafe { bit_transpose_64bytes_gfni(&input, &mut got) }; assert_eq!(got, want, "gfni"); } + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "gfni", + not(target_feature = "avx512vbmi") + ))] + { + // SAFETY: compiled only with the features enabled. + unsafe { bit_transpose_64bytes_gfni_avx2(&input, &mut got) }; + assert_eq!(got, want, "gfni_avx2"); + } + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] + { + // SAFETY: compiled only with the feature enabled. + unsafe { bit_transpose_64bytes_avx2(&input, &mut got) }; + assert_eq!(got, want, "avx2"); + } #[cfg(target_arch = "aarch64")] { // SAFETY: aarch64 always has NEON. From 8f7705d648066baad36dbaee4cb3723370949673 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:02:58 +0000 Subject: [PATCH 02/18] experimental primitives: AVX2 row transpose in BitFold::fold_block On AVX2 without the AVX-512 GFNI kernel, the row-major fold transposes its rows into byte columns (16x16 in-lane byte transposes, rows of 8 bytes paired across the two halves) and runs the column kernel, instead of one 256-entry subset-sum lookup per byte, so the byte tables are no longer built there. The nibble kernel (AVX2 without GFNI) stores each 16-entry table once and broadcasts it to both lanes, which halves its table footprint, and writes its output through the same 16x16 byte transpose as the GFNI-256 kernel instead of a word buffer. Rows of 128 bytes, whose tables outgrow L1, take four output bytes of both 32-row halves at a time so each table is read once per block. Throwaway microbench, ns per 64-row block, single thread, EPYC 9454P core 40, min of runs: fold_columns, nibble (haswell) C=16 256 -> 205, C=32 634 -> 386, C=64 1502 -> 899, C=128 2775 -> 2024 fold_block, nibble (haswell) C=16 349 -> 228, C=32 1229 -> 437 fold_block, GFNI-256 C=16 327 -> 130, C=32 2619 -> 245 aggregate --xmss 100, LEANVM_NUM_THREADS=1, core 40, both commits against upstream main, min over six alternating runs, ms (base -> new): vp: Prove 979 -> 971, Project 59.4 -> 54.4, Openings 7.6 -> 5.4, Lift 1.5 -> 1.2 gfni: Prove 931 -> 923, Project 52.1 -> 49.0, Openings 5.6 -> 3.9, Lift 1.5 -> 1.0 hsw: Prove 1330 -> 1330, Project 82.1 -> 77.0, Openings 7.5 -> 5.5, Lift 2.0 -> 1.7 --- .../primitives/src/bit_fold.rs | 328 +++++++++++------- 1 file changed, 202 insertions(+), 126 deletions(-) diff --git a/crates/experimental_prover/primitives/src/bit_fold.rs b/crates/experimental_prover/primitives/src/bit_fold.rs index 5191aa6270..f726de14f5 100644 --- a/crates/experimental_prover/primitives/src/bit_fold.rs +++ b/crates/experimental_prover/primitives/src/bit_fold.rs @@ -12,21 +12,15 @@ //! - GFNI: an 8x8 bit matrix per (input byte, output byte), applied to 64 rows by one instruction. //! - AVX2 without GFNI, and NEON: each of those matrices as two 16-entry nibble tables, one byte shuffle each per //! 32 or 16 rows. +//! +//! The 256-bit kernels read byte columns, so there the row-major fold transposes its rows first. use crate::field::F128; /// Rows folded per call. pub const BLOCK: usize = 64; -#[cfg(any( - test, - not(all( - target_arch = "x86_64", - target_feature = "gfni", - target_feature = "avx512bw", - target_feature = "avx512vbmi" - )) -))] +#[cfg(any(test, not(all(target_arch = "x86_64", target_feature = "avx2"))))] /// A byte's subset-sum table: 256 entries, then a cache line of padding. type Table = [F128; 256 + 4]; @@ -76,15 +70,7 @@ impl BitFold { } } -#[cfg(any( - test, - not(all( - target_arch = "x86_64", - target_feature = "gfni", - target_feature = "avx512bw", - target_feature = "avx512vbmi" - )) -))] +#[cfg(any(test, not(all(target_arch = "x86_64", target_feature = "avx2"))))] /// One 256-entry subset-sum table per byte of a row: entry `[j][v]` sums the weights of the set bits of `v` at byte `j`. /// /// A table is a cache line longer than its entries, so one entry of every table does not fall in one cache set: rows of @@ -106,15 +92,7 @@ fn lookup_tables(weights: &[F128]) -> Vec { .collect() } -#[cfg(any( - test, - not(all( - target_arch = "x86_64", - target_feature = "gfni", - target_feature = "avx512bw", - target_feature = "avx512vbmi" - )) -))] +#[cfg(any(test, not(all(target_arch = "x86_64", target_feature = "avx2"))))] /// The fold of one row through the byte tables: one lookup and one XOR per byte. #[inline(always)] fn fold_row_lookup(tables: &[Table], row: &[u8; CHUNKS]) -> F128 { @@ -122,9 +100,12 @@ fn fold_row_lookup(tables: &[Table], row: &[u8; CHUNKS]) -> row.iter().zip(tables).map(|(&v, sums)| sums[usize::from(v)]).sum() } -#[cfg(not(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni")))] +#[cfg(not(all(target_arch = "x86_64", target_feature = "avx2")))] use portable::Imp; +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")))] +use avx2::Imp; + #[cfg(all( target_arch = "x86_64", target_feature = "avx2", @@ -141,12 +122,7 @@ use gfni256::Imp; ))] use gfni::Imp; -#[cfg(not(all( - target_arch = "x86_64", - target_feature = "gfni", - target_feature = "avx512bw", - target_feature = "avx512vbmi" -)))] +#[cfg(not(all(target_arch = "x86_64", target_feature = "avx2")))] mod portable { use super::{BLOCK, F128, Table, fold_row_lookup, lookup_tables}; @@ -155,9 +131,6 @@ mod portable { pub(super) struct Imp { /// One subset-sum table per byte of a row. tables: Vec
, - /// The nibble tables of `avx2::fold_columns`. - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")))] - nibbles: Vec, /// The nibble tables of `neon::fold_columns`. #[cfg(target_arch = "aarch64")] nibbles: Vec, @@ -167,8 +140,6 @@ mod portable { pub(super) fn new(weights: &[F128]) -> Self { Self { tables: lookup_tables(weights), - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")))] - nibbles: super::nibble_tables(weights), #[cfg(target_arch = "aarch64")] nibbles: super::nibble_tables(weights), } @@ -182,23 +153,12 @@ mod portable { out[rows.len()..].fill(F128::ZERO); } - // With AVX2 and GFNI the columns go through `gfni256`, which keeps these tables for rows only. - #[cfg_attr( - all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"), - allow(dead_code) - )] #[inline] pub(super) fn fold_columns(&self, cols: &[[u8; BLOCK]; CHUNKS], out: &mut [F128; BLOCK]) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")))] - // SAFETY: AVX2 is enabled at compile time. - return unsafe { super::avx2::fold_columns(&self.nibbles, cols, out) }; #[cfg(target_arch = "aarch64")] // SAFETY: NEON is part of aarch64. return unsafe { super::neon::fold_columns(&self.nibbles, cols, out) }; - #[cfg(not(any( - all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")), - target_arch = "aarch64" - )))] + #[cfg(not(target_arch = "aarch64"))] for (p, o) in out.iter_mut().enumerate() { *o = cols .iter() @@ -210,13 +170,13 @@ mod portable { } } -/// Entry `16 j + o`: the images of the low and the high nibble of input byte `j` in output byte `o`, each 16-entry -/// table repeated to fill `W` bytes. +/// Entry `16 j + o`: the images of the low and the high nibble of input byte `j` in output byte `o`, each a +/// 16-entry table. #[cfg(any( all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")), target_arch = "aarch64" ))] -fn nibble_tables(weights: &[F128]) -> Vec<[[u8; W]; 2]> { +fn nibble_tables(weights: &[F128]) -> Vec<[[u8; 16]; 2]> { let byte = |w: &F128, o: usize| (w.to_limbs()[o / 8] >> (8 * (o % 8))) as u8; let mut out = Vec::with_capacity(2 * weights.len()); for w in weights.as_chunks::<8>().0 { @@ -224,7 +184,7 @@ fn nibble_tables(weights: &[F128]) -> Vec<[[u8; W]; 2]> { out.push(std::array::from_fn(|half| { std::array::from_fn(|i| { (0..4) - .filter(|t| (i % 16) >> t & 1 == 1) + .filter(|t| i >> t & 1 == 1) .fold(0, |acc, t| acc ^ byte(&w[4 * half + t], o)) }) })); @@ -233,6 +193,74 @@ fn nibble_tables(weights: &[F128]) -> Vec<[[u8; W]; 2]> { out } +/// Up to 64 rows as byte columns, `cols[j][p]` byte `j` of row `p`, rows past the input zero. +#[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + not(all(target_feature = "gfni", target_feature = "avx512bw", target_feature = "avx512vbmi")) +))] +#[inline] +#[target_feature(enable = "avx2")] +fn columns_avx2(rows: &[[u8; CHUNKS]], cols: &mut [[u8; BLOCK]; CHUNKS]) { + match rows.try_into() { + Ok(rows) => columns_full_avx2(rows, cols), + Err(_) => { + // A short block transposes a zero-padded copy. + let mut padded = [[0u8; CHUNKS]; BLOCK]; + padded[..rows.len()].copy_from_slice(rows); + columns_full_avx2(&padded, cols); + } + } +} + +/// [`columns_avx2`] of a full block. +/// +/// A 16-byte slice of a row is a lane, rows `p` and `p + 16` sharing a register, and sixteen registers take one in-lane +/// byte transpose. Rows of 8 bytes put rows `p` and `p + 32` in a lane, so the transpose's registers `k` and `k + 8` +/// are the two halves of column `k`. +#[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + not(all(target_feature = "gfni", target_feature = "avx512bw", target_feature = "avx512vbmi")) +))] +#[inline] +#[target_feature(enable = "avx2")] +fn columns_full_avx2(rows: &[[u8; CHUNKS]; BLOCK], cols: &mut [[u8; BLOCK]; CHUNKS]) { + use core::arch::x86_64::*; + + use crate::bits::transpose_bytes16x16_per_lane256 as transpose; + use crate::{load_at, store_at}; + + let pair = |lo: &[u8], hi: &[u8]| -> __m256i { _mm256_set_m128i(load_at(hi, 0), load_at(lo, 0)) }; + if CHUNKS == 8 { + let bytes = rows.as_flattened(); + let mut regs = [_mm256_setzero_si256(); 16]; + for z in (0..16).step_by(2) { + // Lane `l` of `x`: rows `z + 16 l` and `z + 16 l + 1`; of `y`, the same rows plus 32. + let x = pair(&bytes[8 * z..], &bytes[8 * (z + 16)..]); + let y = pair(&bytes[8 * (z + 32)..], &bytes[8 * (z + 48)..]); + regs[z] = _mm256_unpacklo_epi64(x, y); + regs[z + 1] = _mm256_unpackhi_epi64(x, y); + } + transpose(&mut regs); + for (k, col) in cols.iter_mut().enumerate() { + store_at(col, 0, regs[k]); + store_at(col, 32, regs[k + 8]); + } + } else { + for h in 0..2 { + for g in 0..CHUNKS / 16 { + let mut regs: [__m256i; 16] = + std::array::from_fn(|z| pair(&rows[32 * h + z][16 * g..], &rows[32 * h + 16 + z][16 * g..])); + transpose(&mut regs); + for (k, r) in regs.into_iter().enumerate() { + store_at(&mut cols[16 * g + k], 32 * h, r); + } + } + } + } +} + /// The GFNI fold's byte-matrix products as nibble lookups, for AVX2 without GFNI. /// /// Output byte `o` of a row is `XOR_j A[j][o] * byte j`. A byte matrix applied to a byte is the sum of its @@ -244,64 +272,103 @@ mod avx2 { use super::{BLOCK, F128}; use crate::{load_at, store_at}; - /// [`super::nibble_tables`], each table in both 128-bit lanes. - pub(super) type Nibbles = [[u8; 32]; 2]; + /// An entry of [`super::nibble_tables`], broadcast to both 128-bit lanes when used. + type Nibbles = [[u8; 16]; 2]; - /// [`super::BitFold::fold_columns`]: per 32 rows and coefficient, eight output bytes summed over the - /// input bytes, then transposed into words. + /// The nibble tables. + #[derive(Clone, Debug)] + pub(super) struct Imp { + tables: Vec, + } + + impl Imp { + pub(super) fn new(weights: &[F128]) -> Self { + Self { + tables: super::nibble_tables(weights), + } + } + + #[inline] + pub(super) fn fold_block(&self, rows: &[[u8; CHUNKS]], out: &mut [F128; BLOCK]) { + let mut cols = [[0u8; BLOCK]; CHUNKS]; + // SAFETY: AVX2 is enabled at compile time (the module's cfg). + unsafe { super::columns_avx2(rows, &mut cols) }; + self.fold_columns(&cols, out); + } + + #[inline] + pub(super) fn fold_columns(&self, cols: &[[u8; BLOCK]; CHUNKS], out: &mut [F128; BLOCK]) { + // SAFETY: AVX2 is enabled at compile time (the module's cfg). + unsafe { fold_columns(&self.tables, cols, out) } + } + } + + /// [`super::BitFold::fold_columns`]: output bytes summed over the input bytes, 32 rows to a register, then per half a + /// byte transpose of the 16 output-byte registers into 32 F128 values. + /// + /// Up to 64-byte rows, eight output bytes of one half at a time. Wider rows' tables outgrow L1, so four output bytes + /// of both halves at a time read each table once. #[inline] #[target_feature(enable = "avx2")] - pub(super) fn fold_columns( - tables: &[Nibbles], - cols: &[[u8; BLOCK]; CHUNKS], - out: &mut [F128; BLOCK], - ) { - assert_eq!(tables.len(), 16 * CHUNKS); + fn fold_columns(tables: &[Nibbles], cols: &[[u8; BLOCK]; CHUNKS], out: &mut [F128; BLOCK]) { + let tables: &[[Nibbles; 16]; CHUNKS] = tables.as_chunks().0.try_into().expect("16 tables per byte"); let mask = _mm256_set1_epi8(0x0f); - let mut words = [[0u64; BLOCK]; 2]; - for (k, words) in words.iter_mut().enumerate() { + let nibbles = |col: &[u8; BLOCK], h: usize| { + let v: __m256i = load_at(col, 32 * h); + [ + _mm256_and_si256(v, mask), + _mm256_and_si256(_mm256_srli_epi16::<4>(v), mask), + ] + }; + let image = |t: &Nibbles, [lo, hi]: [__m256i; 2]| { + let (tl, th) = ( + _mm256_broadcastsi128_si256(load_at(&t[0], 0)), + _mm256_broadcastsi128_si256(load_at(&t[1], 0)), + ); + _mm256_xor_si256(_mm256_shuffle_epi8(tl, lo), _mm256_shuffle_epi8(th, hi)) + }; + // Byte `16 l + p` of `acc[o]` is byte `o` of row `32 h + 16 l + p`. + let mut store = |h: usize, mut acc: [__m256i; 16]| { + crate::bits::transpose_bytes16x16_per_lane256(&mut acc); + for (k, a) in acc.into_iter().enumerate() { + store_at(out, 32 * h + k, _mm256_castsi256_si128(a)); + store_at(out, 32 * h + 16 + k, _mm256_extracti128_si256::<1>(a)); + } + }; + if CHUNKS <= 64 { for h in 0..2 { - let mut acc = [_mm256_setzero_si256(); 8]; - for (j, col) in cols.iter().enumerate() { - let v: __m256i = load_at(col, 32 * h); - let (lo, hi) = ( - _mm256_and_si256(v, mask), - _mm256_and_si256(_mm256_srli_epi16::<4>(v), mask), - ); - for (i, acc) in acc.iter_mut().enumerate() { - let t = &tables[16 * j + 8 * k + i]; - let (tl, th): (__m256i, __m256i) = (load_at(&t[0], 0), load_at(&t[1], 0)); - let image = _mm256_xor_si256(_mm256_shuffle_epi8(tl, lo), _mm256_shuffle_epi8(th, hi)); - *acc = _mm256_xor_si256(*acc, image); + let mut acc = [_mm256_setzero_si256(); 16]; + for o0 in (0..16).step_by(8) { + let mut a = [_mm256_setzero_si256(); 8]; + for (col, tables) in cols.iter().zip(tables) { + let x = nibbles(col, h); + for (a, t) in a.iter_mut().zip(&tables[o0..o0 + 8]) { + *a = _mm256_xor_si256(*a, image(t, x)); + } } + acc[o0..o0 + 8].copy_from_slice(&a); } - // Byte `p` of `acc[i]` is byte `i` of row `32 h + p`: three unpack stages make words. - let pairs = |a: __m256i, b: __m256i| (_mm256_unpacklo_epi8(a, b), _mm256_unpackhi_epi8(a, b)); - let (l01, h01) = pairs(acc[0], acc[1]); - let (l23, h23) = pairs(acc[2], acc[3]); - let (l45, h45) = pairs(acc[4], acc[5]); - let (l67, h67) = pairs(acc[6], acc[7]); - // Lane 0 of a stage-one register holds rows `base..base + 8`, lane 1 the same rows plus 16. - for (base, (a, b, c, d)) in [(0, (l01, l23, l45, l67)), (8, (h01, h23, h45, h67))] { - let quads = [ - (base, _mm256_unpacklo_epi16(a, b), _mm256_unpacklo_epi16(c, d)), - (base + 4, _mm256_unpackhi_epi16(a, b), _mm256_unpackhi_epi16(c, d)), - ]; - for (base, x, y) in quads { - for (base, w) in [ - (base, _mm256_unpacklo_epi32(x, y)), - (base + 2, _mm256_unpackhi_epi32(x, y)), - ] { - let row = 32 * h + base; - store_at(words, row, _mm256_castsi256_si128(w)); - store_at(words, row + 16, _mm256_extracti128_si256::<1>(w)); + store(h, acc); + } + } else { + let mut acc = [[_mm256_setzero_si256(); 16]; 2]; + for o0 in (0..16).step_by(4) { + let mut a = [[_mm256_setzero_si256(); 4]; 2]; + for (col, tables) in cols.iter().zip(tables) { + let x = [nibbles(col, 0), nibbles(col, 1)]; + for (i, t) in tables[o0..o0 + 4].iter().enumerate() { + for (a, x) in a.iter_mut().zip(x) { + a[i] = _mm256_xor_si256(a[i], image(t, x)); } } } + for (acc, a) in acc.iter_mut().zip(a) { + acc[o0..o0 + 4].copy_from_slice(&a); + } + } + for (h, acc) in acc.into_iter().enumerate() { + store(h, acc); } - } - for (p, o) in out.iter_mut().enumerate() { - *o = F128::new(words[0][p], words[1][p]); } } } @@ -685,8 +752,8 @@ mod gfni { not(all(target_feature = "avx512bw", target_feature = "avx512vbmi")) ))] mod gfni256 { - //! The GFNI fold on 256-bit registers (AVX2 and GFNI, without the AVX-512 kernel's byte permutes), for - //! [`super::BitFold::fold_columns`], whose rows already come as byte columns: + //! The GFNI fold on 256-bit registers (AVX2 and GFNI, without the AVX-512 kernel's byte permutes), on rows as byte + //! columns: //! //! ```text //! per 32 rows: out byte o += A[j][o] * (column j), one affine instruction per column and output byte @@ -694,7 +761,7 @@ mod gfni256 { //! ``` //! //! Sixteen registers hold eight accumulators at a time, so the columns are read twice. The row-major - //! [`super::BitFold::fold_block`] keeps the byte tables. + //! [`super::BitFold::fold_block`] transposes its rows first. use core::arch::x86_64::*; @@ -703,13 +770,11 @@ mod gfni256 { /// Bytes of an F128. const OUT_BYTES: usize = 16; - /// The GFNI matrices, and the byte tables for the row-major fold. + /// The GFNI matrices. #[derive(Clone, Debug)] pub(super) struct Imp { /// `matrices[j][o]` maps byte `j` of a row to byte `o` of its weights' sum. matrices: Vec<[u64; OUT_BYTES]>, - /// The byte tables of the row-major fold. - lookup: super::portable::Imp, } /// Byte `o` of a weight. @@ -743,15 +808,15 @@ mod gfni256 { }) }) .collect(); - Self { - matrices, - lookup: super::portable::Imp::new(weights), - } + Self { matrices } } #[inline] pub(super) fn fold_block(&self, rows: &[[u8; CHUNKS]], out: &mut [F128; BLOCK]) { - self.lookup.fold_block(rows, out); + let mut cols = [[0u8; BLOCK]; CHUNKS]; + // SAFETY: AVX2 is enabled at compile time (the module's cfg). + unsafe { super::columns_avx2(rows, &mut cols) }; + self.fold_columns(&cols, out); } #[inline] @@ -876,23 +941,34 @@ mod tests { check::<128>(&mut rng); } - /// Rows of 16 bytes against the byte-table fold, full and short blocks. + /// Every row width against the byte-table fold, full and short blocks. #[test] fn fold_block_matches_lookup() { - let mut rng = Rng::new(0x0009_A12B); - let weights: Vec = (0..128).map(|_| rng.ext()).collect(); - let (fold, tables) = (BitFold::new(&weights), lookup_tables(&weights)); - for len in [BLOCK, 5] { - let rows: Vec<[u8; 16]> = (0..len) - .map(|_| std::array::from_fn(|_| rng.next_u64() as u8)) - .collect(); - let mut out = [F128::ZERO; BLOCK]; - fold.fold_block(&rows, &mut out); - for p in 0..len { - assert_eq!(out[p], fold_row_lookup(&tables, &rows[p]), "len={len}, row {p}"); + fn check(rng: &mut Rng) { + let weights: Vec = (0..8 * CHUNKS).map(|_| rng.ext()).collect(); + let (fold, tables) = (BitFold::new(&weights), lookup_tables(&weights)); + for len in [BLOCK, 37, 5] { + let rows: Vec<[u8; CHUNKS]> = (0..len) + .map(|_| std::array::from_fn(|_| rng.next_u64() as u8)) + .collect(); + let mut out = [F128::ZERO; BLOCK]; + fold.fold_block(&rows, &mut out); + for p in 0..len { + assert_eq!( + out[p], + fold_row_lookup(&tables, &rows[p]), + "CHUNKS={CHUNKS}, len={len}, row {p}" + ); + } + assert!(out[len..].iter().all(|o| *o == F128::ZERO)); } - assert!(out[len..].iter().all(|o| *o == F128::ZERO)); } + let mut rng = Rng::new(0x0009_A12B); + check::<8>(&mut rng); + check::<16>(&mut rng); + check::<32>(&mut rng); + check::<64>(&mut rng); + check::<128>(&mut rng); } /// The byte tables against the definition: the sum of the weights of the set bits. From e7d324025a6e251d419b60c61d292db97a51b9b1 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 19:58:19 +0000 Subject: [PATCH 03/18] experimental keccak_gkr: table rows by byte maps on AVX2 The table pass's rows come from 32 x 16 byte maps per register of positions on every AVX2 target, not only AVX-512 with GFNI: one 256-bit affine product per map with GFNI, two vpshufb nibble lookups without. The 6 * BITS scratch and its reduction pass remain only off x86. The module is renamed table_gfni -> table_maps; the AVX-512 arm is unchanged but for sharing the unpack rounds. Single thread, aggregate --xmss 100, last pass, Tables span, same core (16), base then candidate: - gfni (x86-64-v3 + gfni): 49.1 -> 22.6, and 49.1 -> 23.3 on a repeat - vp (x86-64-v3 + vpclmulqdq): 49.1 -> 33.7 - hsw (haswell): 81.9 -> 43.9 - native (Zen 4, AVX-512 arm): 19.5 -> 19.6 Rows alone, microbenchmark per 1600-position block: 11.3 us with GFNI, 22.3 us with nibble lookups. --- .../experimental_prover/keccak_gkr/src/lib.rs | 9 +- .../keccak_gkr/src/prover.rs | 24 +- .../keccak_gkr/src/table_gfni.rs | 243 ---------- .../keccak_gkr/src/table_maps.rs | 422 ++++++++++++++++++ 4 files changed, 431 insertions(+), 267 deletions(-) delete mode 100644 crates/experimental_prover/keccak_gkr/src/table_gfni.rs create mode 100644 crates/experimental_prover/keccak_gkr/src/table_maps.rs diff --git a/crates/experimental_prover/keccak_gkr/src/lib.rs b/crates/experimental_prover/keccak_gkr/src/lib.rs index bc74084840..a0970c1bcc 100644 --- a/crates/experimental_prover/keccak_gkr/src/lib.rs +++ b/crates/experimental_prover/keccak_gkr/src/lib.rs @@ -116,13 +116,8 @@ mod prover; pub mod rmfe; #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] mod skip_gfni; -#[cfg(all( - target_arch = "x86_64", - target_feature = "avx512f", - target_feature = "avx512bw", - target_feature = "gfni" -))] -mod table_gfni; +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +mod table_maps; pub mod witness; #[cfg(test)] diff --git a/crates/experimental_prover/keccak_gkr/src/prover.rs b/crates/experimental_prover/keccak_gkr/src/prover.rs index db472c174b..680b8d741f 100644 --- a/crates/experimental_prover/keccak_gkr/src/prover.rs +++ b/crates/experimental_prover/keccak_gkr/src/prover.rs @@ -411,30 +411,20 @@ impl Layer<'_> { let field = round.field; let fold: Vec = eq_table(challenges).iter().map(|&e| field.to_poly(e)).collect(); let fold: [[u64; GROUP]; 2] = std::array::from_fn(|k| std::array::from_fn(|q| fold[q][k])); - #[cfg(all( - target_arch = "x86_64", - target_feature = "avx512f", - target_feature = "avx512bw", - target_feature = "gfni" - ))] - let row_maps = crate::table_gfni::RowMaps::new(field, &fold); + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] + let row_maps = crate::table_maps::RowMaps::new(field, &fold); // Rows of segments that do not exist are zero, and so is their product. table[self.rounds.len() * layout.live * ROW..layout.len() * ROW].fill(0); pads[self.rounds.len() * ROW..].fill(0); - // The GFNI rows need no wide scratch. - let gfni = cfg!(all( - target_arch = "x86_64", - target_feature = "avx512f", - target_feature = "avx512bw", - target_feature = "gfni" - )); - let scratch = || vec![0u64; if gfni { 0 } else { 6 * BITS }]; + // The byte-map rows need no wide scratch. + let maps = cfg!(all(target_arch = "x86_64", target_feature = "avx2")); + let scratch = || vec![0u64; if maps { 0 } else { 6 * BITS }]; // A block's rows: both groups', or the padding block's one. let build = |wide: &mut Vec, block: &[u64], rows: &mut [u64]| { let halves = rows.len() / ROW; cfg_select! { - all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "avx512bw", target_feature = "gfni") => { - // Each group's value by affine byte maps (`table_gfni`). + all(target_arch = "x86_64", target_feature = "avx2") => { + // Each group's value by byte maps (`table_maps`). let _ = wide; row_maps.rows(block, rows, halves); } diff --git a/crates/experimental_prover/keccak_gkr/src/table_gfni.rs b/crates/experimental_prover/keccak_gkr/src/table_gfni.rs deleted file mode 100644 index f07553ba2d..0000000000 --- a/crates/experimental_prover/keccak_gkr/src/table_gfni.rs +++ /dev/null @@ -1,243 +0,0 @@ -//! The table pass's rows by GFNI (x86-64 with AVX-512BW and GFNI): the same rows as embedding each -//! position's packs, weighting them by the folded challenges with carry-less products and reducing. -//! -//! A position's value in a group, `reduce(sum over packs q of embed(pack q) * fold[q])`, is -//! GF(2)-linear in the group's 256 instance bits, so it is a sum of one 8x8 bit matrix per (input -//! byte, output byte): byte `t` of the group's 32 bytes holds bits `8 (t % 2) ..` of pack `t / 2`, -//! and each of its bits maps to the reduced product of one unit embedding by that pack's weight. -//! A block is byte-transposed 64 positions at a time (register `b` = byte `b` of the 64 positions), -//! each group then costs `32 * 16` affine products per 64 positions, and the 16 output bytes are -//! transposed back to the row's words. -//! -//! The scheme is the table fill of hashmancer-keccak's GFNI kernels. - -use core::arch::x86_64::*; - -use primitives::field::gf2_64::mul_wide; -use verifier::keccak::BITS; - -use crate::kernels::GROUP; -use crate::power::PowerField; -use crate::rmfe::RMFE; -use crate::witness::BLOCK_WORDS; - -/// Bytes of a group of a position (16 packs of 16 bits), and of a reduced value. -const GROUP_BYTES: usize = 32; -const VALUE_BYTES: usize = 16; -/// Positions per register. -const REGISTER_POSITIONS: usize = 64; - -/// The affine maps of one layer's fold weights: `matrices[t][o]` sends byte `t` of a group to byte `o` -/// of its value. -pub struct RowMaps { - matrices: [[u64; VALUE_BYTES]; GROUP_BYTES], -} - -impl RowMaps { - /// From the fold weights `fold[k][q]` (word `k` of pack `q`'s weight, in the power basis). - pub fn new(field: &PowerField, fold: &[[u64; GROUP]; 2]) -> Self { - // The value of bit `i` of pack `q` alone, image `16 q + i`: its unit embedding times the pack's - // weight, reduced, a row of three-word products laid out by word as the carry-less path's. - const IMAGES: usize = 16 * GROUP; - let mut wide = vec![0u64; 3 * IMAGES]; - for q in 0..GROUP { - for (i, &e) in RMFE.emb.iter().enumerate() { - for (k, w) in fold.iter().enumerate() { - let x = mul_wide(u64::from(e), w[q]); - wide[k * IMAGES + 16 * q + i] ^= x as u64; - wide[(k + 1) * IMAGES + 16 * q + i] ^= (x >> 64) as u64; - } - } - } - let mut images = vec![0u64; 2 * IMAGES]; - field.reduce_row(&mut images, &wide, IMAGES); - let bit = |image: usize, n: usize| (images[(n / 64) * IMAGES + image] >> (n % 64)) as u8 & 1; - // An affine byte map reads output bit `k` off matrix byte `7 - k`: its bit `c` is bit `8 o + k` - // of the image of input bit `c`. - let matrices = std::array::from_fn(|t| { - std::array::from_fn(|o| { - (0..8).fold(0u64, |m, k| { - let row = (0..8).fold(0u8, |row, c| row | bit(16 * (t / 2) + 8 * (t % 2) + c, 8 * o + k) << c); - m | u64::from(row) << (8 * (7 - k)) - }) - }) - }); - Self { matrices } - } - - /// The rows of a block's first `halves` groups: row `h`, word `k` of position `j` at - /// `rows[h * 2 BITS + k * BITS + j]`. - pub fn rows(&self, block: &[u64], rows: &mut [u64], halves: usize) { - assert!(block.len() >= BITS * BLOCK_WORDS && halves <= 2 && rows.len() >= halves * 2 * BITS); - // SAFETY: AVX-512F/BW and GFNI are enabled at compile time (module cfg); every load reads - // positions inside `block` and every store writes words inside `rows` (asserted). - unsafe { - for g in 0..BITS / REGISTER_POSITIONS { - let y = transpose_position_bytes(block.as_ptr().add(REGISTER_POSITIONS * g * BLOCK_WORDS)); - for h in 0..halves { - let acc = self.accumulate(&y[GROUP_BYTES * h..][..GROUP_BYTES]); - for k in 0..2 { - let words = byte_planes_to_words(acc[8 * k..][..8].try_into().unwrap()); - let dst = rows.as_mut_ptr().add(h * 2 * BITS + k * BITS + REGISTER_POSITIONS * g); - for (r, w) in words.iter().enumerate() { - _mm512_storeu_si512(dst.add(8 * r).cast(), *w); - } - } - } - } - } - } - - /// Output byte `o` of the 64 positions: the sum over input bytes `t` of `matrices[t][o]` applied to `y[t]`. - #[inline] - #[target_feature(enable = "avx512f", enable = "avx512bw", enable = "gfni")] - fn accumulate(&self, y: &[__m512i]) -> [__m512i; VALUE_BYTES] { - let mut acc = [_mm512_setzero_si512(); VALUE_BYTES]; - for t in (0..GROUP_BYTES).step_by(2) { - let (x0, x1, m0, m1) = (y[t], y[t + 1], &self.matrices[t], &self.matrices[t + 1]); - for (o, a) in acc.iter_mut().enumerate() { - let g0 = _mm512_gf2p8affine_epi64_epi8::<0>(x0, _mm512_set1_epi64(m0[o] as i64)); - let g1 = _mm512_gf2p8affine_epi64_epi8::<0>(x1, _mm512_set1_epi64(m1[o] as i64)); - *a = _mm512_ternarylogic_epi64::<0x96>(*a, g0, g1); - } - } - acc - } -} - -/// 4x4 transpose of 128-bit lanes: `out[l]` lane `g` = `x[g]` lane `l`. -#[inline] -#[target_feature(enable = "avx512f")] -fn transpose_lanes(x: [__m512i; 4]) -> [__m512i; 4] { - let t0 = _mm512_shuffle_i64x2::<0b01_00_01_00>(x[0], x[1]); - let t1 = _mm512_shuffle_i64x2::<0b11_10_11_10>(x[0], x[1]); - let t2 = _mm512_shuffle_i64x2::<0b01_00_01_00>(x[2], x[3]); - let t3 = _mm512_shuffle_i64x2::<0b11_10_11_10>(x[2], x[3]); - [ - _mm512_shuffle_i64x2::<0b10_00_10_00>(t0, t2), - _mm512_shuffle_i64x2::<0b11_01_11_01>(t0, t2), - _mm512_shuffle_i64x2::<0b10_00_10_00>(t1, t3), - _mm512_shuffle_i64x2::<0b11_01_11_01>(t1, t3), - ] -} - -/// The 64 bytes of 64 consecutive positions (eight words each, at `p`), transposed: register `b` -/// byte `n` is byte `b` of position `n`. -/// -/// # Safety -/// `p` is valid for reading 64 positions; AVX-512F/BW. -#[inline] -#[target_feature(enable = "avx512f", enable = "avx512bw")] -unsafe fn transpose_position_bytes(p: *const u64) -> [__m512i; 64] { - let mut y = [[_mm512_setzero_si512(); 16]; 4]; - for (q, yq) in y.iter_mut().enumerate() { - for (n, v) in yq.iter_mut().enumerate() { - // SAFETY: position `16 q + n` lies inside the caller's 64. - *v = unsafe { _mm512_loadu_si512(p.add(BLOCK_WORDS * (16 * q + n)).cast()) }; - } - // yq[k] lane l byte n = byte 16 l + k of position 16 q + n. - primitives::bits::transpose_bytes16x16_per_lane512(yq); - } - let mut out = [_mm512_setzero_si512(); 64]; - for k in 0..16 { - // out[16 l + k] lane q = y[q][k] lane l. - for (l, v) in transpose_lanes([y[0][k], y[1][k], y[2][k], y[3][k]]) - .into_iter() - .enumerate() - { - out[16 * l + k] = v; - } - } - out -} - -/// Eight byte planes of 64 positions (plane `b` byte `n` = byte `b` of position `n`) as the -/// positions' words: register `r` holds positions `8 r .. 8 r + 8`. -#[inline] -#[target_feature(enable = "avx512f", enable = "avx512bw")] -fn byte_planes_to_words(a: &[__m512i; 8]) -> [__m512i; 8] { - // Per 128-bit lane `L` (positions 16 L ..): bytes, then 16-bit pairs, then 32-bit quads interleave. - let u: [__m512i; 8] = std::array::from_fn(|i| { - let (x, y) = (a[2 * (i / 2)], a[2 * (i / 2) + 1]); - if i % 2 == 0 { - _mm512_unpacklo_epi8(x, y) - } else { - _mm512_unpackhi_epi8(x, y) - } - }); - // u[2 m + s]: lane L, 16-bit element e = bytes of planes 2m, 2m + 1 for position 16 L + 8 s + e. - let v: [__m512i; 8] = std::array::from_fn(|i| { - let (s, half) = (i / 4, i % 4 / 2); - let (x, y) = (u[s], u[2 + s]); - let (x, y) = if i % 2 == 0 { (x, y) } else { (u[4 + s], u[6 + s]) }; - if half == 0 { - _mm512_unpacklo_epi16(x, y) - } else { - _mm512_unpackhi_epi16(x, y) - } - }); - // v[4 s + 2 h + c]: lane L, 32-bit element e = planes 4c .. 4c + 4 of position 16 L + 8 s + 4 h + e. - let w: [__m512i; 8] = std::array::from_fn(|i| { - let (j, hi) = (i / 2, i % 2); - let (s, h) = (j / 2, j % 2); - let (x, y) = (v[4 * s + 2 * h], v[4 * s + 2 * h + 1]); - if hi == 0 { - _mm512_unpacklo_epi32(x, y) - } else { - _mm512_unpackhi_epi32(x, y) - } - }); - // w[2 j + e2]: lane L, qword d = position 16 L + 4 j + 2 e2 + d, all eight planes: lane L of - // w[m] holds positions 16 L + 2 m, 16 L + 2 m + 1. - let lo = transpose_lanes([w[0], w[1], w[2], w[3]]); - let hi = transpose_lanes([w[4], w[5], w[6], w[7]]); - // lo[L] = positions 16 L .. 16 L + 8, hi[L] = 16 L + 8 .. 16 L + 16. - [lo[0], hi[0], lo[1], hi[1], lo[2], hi[2], lo[3], hi[3]] -} - -#[cfg(test)] -mod tests { - use primitives::test_rng::Rng; - - use super::*; - use crate::basis::carry; - use crate::kernels::{embed, fold_groups}; - use crate::power::Element; - use crate::witness::words; - - /// The rows the per-position path builds: embeddings weighted by carry-less products, reduced. - #[test] - fn rows_match_the_carry_less_path() { - let mut rng = Rng::new(0x7AB1E); - { - let field = PowerField::new(rng.ext()).expect("a generator"); - // The fold weights: field elements in the power basis, by word. - let elems: Vec = (0..GROUP).map(|_| field.to_poly(rng.ext())).collect(); - let fold: [[u64; GROUP]; 2] = std::array::from_fn(|k| std::array::from_fn(|q| elems[q][k])); - let block: Vec = (0..BITS * BLOCK_WORDS).map(|_| rng.next_u64()).collect(); - let mut want = vec![0u64; 2 * 2 * BITS]; - let mut wide = vec![0u64; 2 * 3 * BITS]; - for j in 0..BITS { - let sums = fold_groups(&embed(words(&block, j)), &fold); - for (h, sum) in sums.iter().enumerate() { - let sum: [u64; 3] = carry(sum); - for (k, &w) in sum.iter().enumerate() { - wide[h * 3 * BITS + k * BITS + j] = w; - } - } - } - for h in 0..2 { - field.reduce_row( - &mut want[h * 2 * BITS..][..2 * BITS], - &wide[h * 3 * BITS..][..3 * BITS], - BITS, - ); - } - for halves in [2, 1] { - let mut got = vec![0u64; 2 * 2 * BITS]; - RowMaps::new(&field, &fold).rows(&block, &mut got, halves); - assert_eq!(got[..halves * 2 * BITS], want[..halves * 2 * BITS], "{halves} halves"); - } - } - } -} diff --git a/crates/experimental_prover/keccak_gkr/src/table_maps.rs b/crates/experimental_prover/keccak_gkr/src/table_maps.rs new file mode 100644 index 0000000000..ab97a63000 --- /dev/null +++ b/crates/experimental_prover/keccak_gkr/src/table_maps.rs @@ -0,0 +1,422 @@ +//! The table pass's rows by byte maps (x86-64 with AVX2): the same rows as embedding each +//! position's packs, weighting them by the folded challenges with carry-less products and reducing. +//! +//! A position's value in a group, `reduce(sum over packs q of embed(pack q) * fold[q])`, is +//! GF(2)-linear in the group's 256 instance bits, so it is a sum of one 8x8 bit matrix per (input +//! byte, output byte): byte `t` of the group's 32 bytes holds bits `8 (t % 2) ..` of pack `t / 2`, +//! and each of its bits maps to the reduced product of one unit embedding by that pack's weight. +//! A block is byte-transposed a register of positions at a time (register `b` = byte `b` of the +//! register's positions), each group then costs `32 * 16` byte maps per register, and the 16 +//! output bytes are transposed back to the row's words. A byte map is one GFNI affine product, 512 +//! bits wide with AVX-512BW, or without GFNI two `vpshufb` nibble lookups. +//! +//! The scheme is the table fill of hashmancer-keccak's GFNI kernels. + +use core::arch::x86_64::*; + +use primitives::field::gf2_64::mul_wide; +use verifier::keccak::BITS; + +use crate::kernels::GROUP; +use crate::power::PowerField; +use crate::rmfe::RMFE; +use crate::witness::BLOCK_WORDS; + +/// Bytes of a group of a position (16 packs of 16 bits), and of a reduced value. +const GROUP_BYTES: usize = 32; +const VALUE_BYTES: usize = 16; + +/// The byte maps of one layer's fold weights: `matrices[t][o]` sends byte `t` of a group to byte `o` +/// of its value. +pub struct RowMaps { + matrices: [[Matrix; VALUE_BYTES]; GROUP_BYTES], +} + +impl RowMaps { + /// From the fold weights `fold[k][q]` (word `k` of pack `q`'s weight, in the power basis). + pub fn new(field: &PowerField, fold: &[[u64; GROUP]; 2]) -> Self { + // The value of bit `i` of pack `q` alone, image `16 q + i`: its unit embedding times the pack's + // weight, reduced, a row of three-word products laid out by word as the carry-less path's. + const IMAGES: usize = 16 * GROUP; + let mut wide = vec![0u64; 3 * IMAGES]; + for q in 0..GROUP { + for (i, &e) in RMFE.emb.iter().enumerate() { + for (k, w) in fold.iter().enumerate() { + let x = mul_wide(u64::from(e), w[q]); + wide[k * IMAGES + 16 * q + i] ^= x as u64; + wide[(k + 1) * IMAGES + 16 * q + i] ^= (x >> 64) as u64; + } + } + } + let mut images = vec![0u64; 2 * IMAGES]; + field.reduce_row(&mut images, &wide, IMAGES); + let bit = |image: usize, n: usize| (images[(n / 64) * IMAGES + image] >> (n % 64)) as u8 & 1; + // An affine byte map reads output bit `k` off matrix byte `7 - k`: its bit `c` is bit `8 o + k` + // of the image of input bit `c`. + let matrices = std::array::from_fn(|t| { + std::array::from_fn(|o| { + (0..8).fold(0u64, |m, k| { + let row = (0..8).fold(0u8, |row, c| row | bit(16 * (t / 2) + 8 * (t % 2) + c, 8 * o + k) << c); + m | u64::from(row) << (8 * (7 - k)) + }) + }) + }); + Self { + matrices: matrices.map(|m| m.map(matrix)), + } + } +} + +/// The three in-lane unpack rounds from eight byte planes `$a` (plane `b` byte `n` of a 128-bit lane +/// `L` = byte `b` of position `16 L + n`) to the positions' words: register `m` lane `L` holds positions +/// `16 L + 2 m` and `16 L + 2 m + 1`. +macro_rules! interleave_planes { + ($a:expr, $lo8:ident, $hi8:ident, $lo16:ident, $hi16:ident, $lo32:ident, $hi32:ident) => {{ + let a = $a; + // Per lane: bytes, then 16-bit pairs, then 32-bit quads interleave. + let u: [_; 8] = std::array::from_fn(|i| { + let (x, y) = (a[2 * (i / 2)], a[2 * (i / 2) + 1]); + if i % 2 == 0 { $lo8(x, y) } else { $hi8(x, y) } + }); + // u[2 m + s]: lane L, 16-bit element e = bytes of planes 2m, 2m + 1 for position 16 L + 8 s + e. + let v: [_; 8] = std::array::from_fn(|i| { + let (s, half) = (i / 4, i % 4 / 2); + let (x, y) = if i % 2 == 0 { + (u[s], u[2 + s]) + } else { + (u[4 + s], u[6 + s]) + }; + if half == 0 { $lo16(x, y) } else { $hi16(x, y) } + }); + // v[4 s + 2 h + c]: lane L, 32-bit element e = planes 4c .. 4c + 4 of position 16 L + 8 s + 4 h + e. + let w: [_; 8] = std::array::from_fn(|i| { + let (j, hi) = (i / 2, i % 2); + let (s, h) = (j / 2, j % 2); + let (x, y) = (v[4 * s + 2 * h], v[4 * s + 2 * h + 1]); + if hi == 0 { $lo32(x, y) } else { $hi32(x, y) } + }); + w + }}; +} + +cfg_select! { + all(target_feature = "avx512f", target_feature = "avx512bw", target_feature = "gfni") => { + /// Positions per register. + const REGISTER_POSITIONS: usize = 64; + + /// A matrix, broadcast by the affine product's memory operand. + type Matrix = u64; + + #[inline] + fn matrix(m: u64) -> Matrix { + m + } + + impl RowMaps { + /// The rows of a block's first `halves` groups: row `h`, word `k` of position `j` at + /// `rows[h * 2 BITS + k * BITS + j]`. + pub fn rows(&self, block: &[u64], rows: &mut [u64], halves: usize) { + assert!(block.len() >= BITS * BLOCK_WORDS && halves <= 2 && rows.len() >= halves * 2 * BITS); + // SAFETY: AVX-512F/BW and GFNI are enabled at compile time (module cfg); every load reads + // positions inside `block` and every store writes words inside `rows` (asserted). + unsafe { + for g in 0..BITS / REGISTER_POSITIONS { + let y = transpose_position_bytes(block.as_ptr().add(REGISTER_POSITIONS * g * BLOCK_WORDS)); + for h in 0..halves { + let acc = self.accumulate(&y[GROUP_BYTES * h..][..GROUP_BYTES]); + for k in 0..2 { + let words = byte_planes_to_words(acc[8 * k..][..8].try_into().unwrap()); + let dst = rows.as_mut_ptr().add(h * 2 * BITS + k * BITS + REGISTER_POSITIONS * g); + for (r, w) in words.iter().enumerate() { + _mm512_storeu_si512(dst.add(8 * r).cast(), *w); + } + } + } + } + } + } + + /// Output byte `o` of the 64 positions: the sum over input bytes `t` of `matrices[t][o]` applied to `y[t]`. + #[inline] + #[target_feature(enable = "avx512f", enable = "avx512bw", enable = "gfni")] + fn accumulate(&self, y: &[__m512i]) -> [__m512i; VALUE_BYTES] { + let mut acc = [_mm512_setzero_si512(); VALUE_BYTES]; + for t in (0..GROUP_BYTES).step_by(2) { + let (x0, x1, m0, m1) = (y[t], y[t + 1], &self.matrices[t], &self.matrices[t + 1]); + for (o, a) in acc.iter_mut().enumerate() { + let g0 = _mm512_gf2p8affine_epi64_epi8::<0>(x0, _mm512_set1_epi64(m0[o] as i64)); + let g1 = _mm512_gf2p8affine_epi64_epi8::<0>(x1, _mm512_set1_epi64(m1[o] as i64)); + *a = _mm512_ternarylogic_epi64::<0x96>(*a, g0, g1); + } + } + acc + } + } + + /// 4x4 transpose of 128-bit lanes: `out[l]` lane `g` = `x[g]` lane `l`. + #[inline] + #[target_feature(enable = "avx512f")] + fn transpose_lanes(x: [__m512i; 4]) -> [__m512i; 4] { + let t0 = _mm512_shuffle_i64x2::<0b01_00_01_00>(x[0], x[1]); + let t1 = _mm512_shuffle_i64x2::<0b11_10_11_10>(x[0], x[1]); + let t2 = _mm512_shuffle_i64x2::<0b01_00_01_00>(x[2], x[3]); + let t3 = _mm512_shuffle_i64x2::<0b11_10_11_10>(x[2], x[3]); + [ + _mm512_shuffle_i64x2::<0b10_00_10_00>(t0, t2), + _mm512_shuffle_i64x2::<0b11_01_11_01>(t0, t2), + _mm512_shuffle_i64x2::<0b10_00_10_00>(t1, t3), + _mm512_shuffle_i64x2::<0b11_01_11_01>(t1, t3), + ] + } + + /// The 64 bytes of 64 consecutive positions (eight words each, at `p`), transposed: register `b` + /// byte `n` is byte `b` of position `n`. + /// + /// # Safety + /// `p` is valid for reading 64 positions; AVX-512F/BW. + #[inline] + #[target_feature(enable = "avx512f", enable = "avx512bw")] + unsafe fn transpose_position_bytes(p: *const u64) -> [__m512i; 64] { + let mut y = [[_mm512_setzero_si512(); 16]; 4]; + for (q, yq) in y.iter_mut().enumerate() { + for (n, v) in yq.iter_mut().enumerate() { + // SAFETY: position `16 q + n` lies inside the caller's 64. + *v = unsafe { _mm512_loadu_si512(p.add(BLOCK_WORDS * (16 * q + n)).cast()) }; + } + // yq[k] lane l byte n = byte 16 l + k of position 16 q + n. + primitives::bits::transpose_bytes16x16_per_lane512(yq); + } + let mut out = [_mm512_setzero_si512(); 64]; + for k in 0..16 { + // out[16 l + k] lane q = y[q][k] lane l. + for (l, v) in transpose_lanes([y[0][k], y[1][k], y[2][k], y[3][k]]) + .into_iter() + .enumerate() + { + out[16 * l + k] = v; + } + } + out + } + + /// Eight byte planes of 64 positions (plane `b` byte `n` = byte `b` of position `n`) as the + /// positions' words: register `r` holds positions `8 r .. 8 r + 8`. + #[inline] + #[target_feature(enable = "avx512f", enable = "avx512bw")] + fn byte_planes_to_words(a: &[__m512i; 8]) -> [__m512i; 8] { + let w = interleave_planes!( + a, + _mm512_unpacklo_epi8, + _mm512_unpackhi_epi8, + _mm512_unpacklo_epi16, + _mm512_unpackhi_epi16, + _mm512_unpacklo_epi32, + _mm512_unpackhi_epi32 + ); + let lo = transpose_lanes([w[0], w[1], w[2], w[3]]); + let hi = transpose_lanes([w[4], w[5], w[6], w[7]]); + // lo[L] = positions 16 L .. 16 L + 8, hi[L] = 16 L + 8 .. 16 L + 16. + [lo[0], hi[0], lo[1], hi[1], lo[2], hi[2], lo[3], hi[3]] + } + } + _ => { + /// Positions per register. + const REGISTER_POSITIONS: usize = 32; + + cfg_select! { + target_feature = "gfni" => { + /// A matrix in every lane, so that the affine product reads it from memory. + type Matrix = __m256i; + + fn matrix(m: u64) -> Matrix { + // SAFETY: AVX is enabled at compile time (module cfg). + unsafe { _mm256_set1_epi64x(m as i64) } + } + + /// An input register as [`product`] reads it. + type Input = __m256i; + + #[inline(always)] + fn input(x: __m256i) -> Input { + x + } + + #[inline(always)] + fn product(x: Input, m: &Matrix) -> __m256i { + // SAFETY: GFNI and AVX are enabled at compile time (module cfg); registers only. + unsafe { _mm256_gf2p8affine_epi64_epi8::<0>(x, *m) } + } + } + _ => { + /// A matrix as two `vpshufb` tables: its images of the low nibbles, then of the high ones. + type Matrix = [u8; 32]; + + fn matrix(m: u64) -> Matrix { + // Output bit `k` is the parity of the input against matrix byte `7 - k`. + let image = |x: u8| (0..8).fold(0u8, |r, k| r | (((m >> (8 * (7 - k))) as u8 & x).count_ones() as u8 & 1) << k); + std::array::from_fn(|i| image(((i % 16) as u8) << (4 * (i / 16)))) + } + + /// An input register as [`product`] reads it: its low nibbles, then its high ones. + type Input = [__m256i; 2]; + + #[inline(always)] + fn input(x: __m256i) -> Input { + // SAFETY: AVX2 is enabled at compile time (module cfg); registers only. + unsafe { + let nibble = _mm256_set1_epi8(0x0F); + [_mm256_and_si256(x, nibble), _mm256_and_si256(_mm256_srli_epi16::<4>(x), nibble)] + } + } + + #[inline(always)] + fn product(x: Input, m: &Matrix) -> __m256i { + // SAFETY: AVX2 is enabled at compile time (module cfg), and each load reads 16 of the + // matrix's bytes. + unsafe { + let lo = _mm256_broadcastsi128_si256(_mm_loadu_si128(m.as_ptr().cast())); + let hi = _mm256_broadcastsi128_si256(_mm_loadu_si128(m.as_ptr().add(16).cast())); + _mm256_xor_si256(_mm256_shuffle_epi8(lo, x[0]), _mm256_shuffle_epi8(hi, x[1])) + } + } + } + } + + impl RowMaps { + /// The rows of a block's first `halves` groups: row `h`, word `k` of position `j` at + /// `rows[h * 2 BITS + k * BITS + j]`. + pub fn rows(&self, block: &[u64], rows: &mut [u64], halves: usize) { + assert!(block.len() >= BITS * BLOCK_WORDS && halves <= 2 && rows.len() >= halves * 2 * BITS); + // SAFETY: AVX2 is enabled at compile time (module cfg); every load reads + // positions inside `block` and every store writes words inside `rows` (asserted). + unsafe { + let mut y = [_mm256_setzero_si256(); GROUP_BYTES]; + for g in 0..BITS / REGISTER_POSITIONS { + for h in 0..halves { + let at = BLOCK_WORDS * REGISTER_POSITIONS * g + GROUP_BYTES / 8 * h; + transpose_group_bytes(block.as_ptr().add(at), &mut y); + for k in 0..2 { + let words = byte_planes_to_words(&self.accumulate(&y, k)); + let dst = rows.as_mut_ptr().add(h * 2 * BITS + k * BITS + REGISTER_POSITIONS * g); + for (r, w) in words.iter().enumerate() { + _mm256_storeu_si256(dst.add(4 * r).cast(), *w); + } + } + } + } + } + } + + /// Output bytes `8 k .. 8 k + 8` of the 32 positions, word `k` of their values: the sums over + /// input bytes `t` of `matrices[t][o]` applied to `y[t]`. Eight outputs at a time keep the sums + /// in registers. + #[inline] + #[target_feature(enable = "avx2")] + fn accumulate(&self, y: &[__m256i; GROUP_BYTES], k: usize) -> [__m256i; 8] { + let mut acc = [_mm256_setzero_si256(); 8]; + for (&x, m) in y.iter().zip(&self.matrices) { + let x = input(x); + for (a, m) in acc.iter_mut().zip(&m[8 * k..][..8]) { + *a = _mm256_xor_si256(*a, product(x, m)); + } + } + acc + } + } + + /// The 32 bytes of a group of 32 consecutive positions (eight words apart, from `p`), transposed: + /// `y[t]` byte `n` is byte `t` of the group of position `n`. + /// + /// # Safety + /// `p` is valid for reading the 32 positions' groups; AVX2. + #[inline] + #[target_feature(enable = "avx2")] + unsafe fn transpose_group_bytes(p: *const u64, y: &mut [__m256i; GROUP_BYTES]) { + for (l, y) in y.as_chunks_mut::<16>().0.iter_mut().enumerate() { + // Register `n`: bytes `16 l ..` of positions `n` and `16 + n`. + *y = std::array::from_fn(|n| { + // SAFETY: positions `n` and `16 + n` lie inside the caller's 32. + unsafe { + _mm256_loadu2_m128i( + p.add(BLOCK_WORDS * (16 + n) + 2 * l).cast(), + p.add(BLOCK_WORDS * n + 2 * l).cast(), + ) + } + }); + // y[k] lane i byte n = byte 16 l + k of position 16 i + n. + primitives::bits::transpose_bytes16x16_per_lane256(y); + } + } + + /// Eight byte planes of 32 positions (plane `b` byte `n` = byte `b` of position `n`) as the + /// positions' words: register `r` holds positions `4 r .. 4 r + 4`. + #[inline] + #[target_feature(enable = "avx2")] + fn byte_planes_to_words(a: &[__m256i; 8]) -> [__m256i; 8] { + let w = interleave_planes!( + a, + _mm256_unpacklo_epi8, + _mm256_unpackhi_epi8, + _mm256_unpacklo_epi16, + _mm256_unpackhi_epi16, + _mm256_unpacklo_epi32, + _mm256_unpackhi_epi32 + ); + std::array::from_fn(|r| { + let (x, y) = (w[2 * (r % 4)], w[2 * (r % 4) + 1]); + if r < 4 { + _mm256_permute2x128_si256::<0x20>(x, y) + } else { + _mm256_permute2x128_si256::<0x31>(x, y) + } + }) + } + } +} + +#[cfg(test)] +mod tests { + use primitives::test_rng::Rng; + + use super::*; + use crate::basis::carry; + use crate::kernels::{embed, fold_groups}; + use crate::power::Element; + use crate::witness::words; + + /// The rows the per-position path builds: embeddings weighted by carry-less products, reduced. + #[test] + fn rows_match_the_carry_less_path() { + let mut rng = Rng::new(0x7AB1E); + { + let field = PowerField::new(rng.ext()).expect("a generator"); + // The fold weights: field elements in the power basis, by word. + let elems: Vec = (0..GROUP).map(|_| field.to_poly(rng.ext())).collect(); + let fold: [[u64; GROUP]; 2] = std::array::from_fn(|k| std::array::from_fn(|q| elems[q][k])); + let block: Vec = (0..BITS * BLOCK_WORDS).map(|_| rng.next_u64()).collect(); + let mut want = vec![0u64; 2 * 2 * BITS]; + let mut wide = vec![0u64; 2 * 3 * BITS]; + for j in 0..BITS { + let sums = fold_groups(&embed(words(&block, j)), &fold); + for (h, sum) in sums.iter().enumerate() { + let sum: [u64; 3] = carry(sum); + for (k, &w) in sum.iter().enumerate() { + wide[h * 3 * BITS + k * BITS + j] = w; + } + } + } + for h in 0..2 { + field.reduce_row( + &mut want[h * 2 * BITS..][..2 * BITS], + &wide[h * 3 * BITS..][..3 * BITS], + BITS, + ); + } + for halves in [2, 1] { + let mut got = vec![0u64; 2 * 2 * BITS]; + RowMaps::new(&field, &fold).rows(&block, &mut got, halves); + assert_eq!(got[..halves * 2 * BITS], want[..halves * 2 * BITS], "{halves} halves"); + } + } + } +} From 6e4fea81049b5bccac746432d44ce52a8902a20f Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:16:45 +0000 Subject: [PATCH 04/18] keccak256: search grinding nonces with first_nonce The proof-of-work grind permuted 64 whole states through keccak_f_many, transposing each into lanes and back, and kept 25 output lanes to read one. `first_nonce` builds the states in lanes from the template, takes the nonce lane as a constant so LLVM hoists the first round's work on the untouched lanes out of the loop, and computes only the output lane in the last round. It returns the same smallest passing nonce, so proofs are unchanged; both provers grind through it. Single thread, AMD EPYC 9454P core 12, same core, back to back. Grind kernel, ns per permutation, keccak_f_many on 64 states before, first_nonce after: hsw 97.9 / 98.1 -> 85.6 / 81.4. aggregate --xmss 100, last pass, ms, base -> this, two runs each: - hsw: prove 1344 / 1355 -> 1320 / 1323; Bus grind 34.1 -> 27.5; PCS open 286 -> 272; Final level 12.9 -> 10.8 - vp: prove 983 / 992 -> 958 / 960; Bus grind 34.0 -> 27.5; PCS open 238 -> 223; Final level 13.1 -> 10.8 - gfni: prove 942 / 939 -> 915 / 915 - native: prove 551 / 552 -> 522 / 529; Bus grind 22.5 -> 15.7; PCS open 151 -> 135; Final level 8.6 -> 5.9 The prove totals include the next commit's AVX2 absorb, which moves only Commit. --- crates/fiat_shamir_prover/src/lib.rs | 20 ++--- crates/keccak256/src/lanes.rs | 121 +++++++++++++++++++++------ crates/keccak256/src/lib.rs | 30 ++++++- 3 files changed, 131 insertions(+), 40 deletions(-) diff --git a/crates/fiat_shamir_prover/src/lib.rs b/crates/fiat_shamir_prover/src/lib.rs index d1d67edcaf..748e9a52ad 100644 --- a/crates/fiat_shamir_prover/src/lib.rs +++ b/crates/fiat_shamir_prover/src/lib.rs @@ -170,19 +170,17 @@ impl FiatShamirState { // first lane. let mut base = self.clone(); base.absorb(F::from_limbs([0, 0]), DS_POW_NONCE); - let (base, lane) = (base.state, 4 * (base.n_pending - 1)); + let search = match 4 * (base.n_pending - 1) { + 0 => keccak256::first_nonce::<0>, + 4 => keccak256::first_nonce::<4>, + 8 => keccak256::first_nonce::<8>, + 12 => keccak256::first_nonce::<12>, + lane => unreachable!("a scalar's first lane in the rate, not {lane}"), + }; + let base = base.state; // The smallest passing nonce of batch `b`, the nonces `b * BATCH..(b + 1) * BATCH`. const BATCH: u64 = 64; - let batch = |b: u64| { - let mut states = [base; BATCH as usize]; - for (nonce, state) in (b * BATCH..).zip(&mut states) { - state[lane] = nonce; - } - keccak256::keccak_f_many(&mut states); - let pass = - |&(nonce, state): &(u64, &[u64; STATE_LANES])| nonce >= from && state[2] & ((1 << bits) - 1) == 0; - (b * BATCH..).zip(&states).find(pass).map(|(nonce, _)| nonce) - }; + let batch = |b: u64| search(&base, (b * BATCH).max(from)..(b + 1) * BATCH, 2, (1 << bits) - 1); // A short search, a few batches, is not worth a dispatch to the pool. let nonce = if bits < 10 { (from / BATCH..).find_map(batch) diff --git a/crates/keccak256/src/lanes.rs b/crates/keccak256/src/lanes.rs index 50e5ccee2f..b0db27a44a 100644 --- a/crates/keccak256/src/lanes.rs +++ b/crates/keccak256/src/lanes.rs @@ -2,7 +2,8 @@ //! vector register per state lane: AVX-512 (eight states), AVX2 (four), NEON (two, with the SHA3 instructions where //! the target has them), or one at a time elsewhere. Every batched function is byte-identical to its one-input form. -use crate::{END_BIT, OUT_LEN, PAD_FIRST, RATE, RATE_LANES, RC, STATE_LANES, hash_from_state, keccak_f}; +use crate::{END_BIT, OUT_LEN, PAD_FIRST, RATE, RATE_LANES, RC, ROUNDS, STATE_LANES, hash_from_state, keccak_f}; +use core::ops::Range; /// One state lane across [`Lane::WIDTH`] states. The permutation is written once over this trait. pub(crate) trait Lane: Copy { @@ -37,38 +38,76 @@ pub(crate) trait Lane: Copy { /// the lanes χ writes, which schedules better than forming every lane before χ and every parity after it. #[inline(always)] pub(crate) fn permute(a: &mut [S; STATE_LANES]) { - let mut c: [S; 5] = core::array::from_fn(|x| S::xor5(a[x], a[x + 5], a[x + 10], a[x + 15], a[x + 20])); + let mut c = parities(a); for &rc in &RC { - let d: [S; 5] = core::array::from_fn(|x| S::theta_d(c[(x + 4) % 5], c[(x + 1) % 5])); - let t = |i: usize| a[i].xor(d[i % 5]); - // Row `y` holds the lanes π sends there, each rotated by its ρ. - #[rustfmt::skip] - let rows: [[S; 5]; 5] = [ - [t(0), t(6).rotl::<44>(), t(12).rotl::<43>(), t(18).rotl::<21>(), t(24).rotl::<14>()], - [t(3).rotl::<28>(), t(9).rotl::<20>(), t(10).rotl::<3>(), t(16).rotl::<45>(), t(22).rotl::<61>()], - [t(1).rotl::<1>(), t(7).rotl::<6>(), t(13).rotl::<25>(), t(19).rotl::<8>(), t(20).rotl::<18>()], - [t(4).rotl::<27>(), t(5).rotl::<36>(), t(11).rotl::<10>(), t(17).rotl::<15>(), t(23).rotl::<56>()], - [t(2).rotl::<62>(), t(8).rotl::<55>(), t(14).rotl::<39>(), t(15).rotl::<41>(), t(21).rotl::<2>()], + round(a, &mut c, rc); + } +} + +/// The column parities θ starts from. +#[inline(always)] +fn parities(a: &[S; STATE_LANES]) -> [S; 5] { + core::array::from_fn(|x| S::xor5(a[x], a[x + 5], a[x + 10], a[x + 15], a[x + 20])) +} + +/// Row `y` before χ, from `t`, the lanes after θ: the lanes π sends there, each rotated by its ρ. +#[inline(always)] +fn row(y: usize, t: impl Fn(usize) -> S) -> [S; 5] { + #[rustfmt::skip] + let row = match y { + 0 => [t(0), t(6).rotl::<44>(), t(12).rotl::<43>(), t(18).rotl::<21>(), t(24).rotl::<14>()], + 1 => [t(3).rotl::<28>(), t(9).rotl::<20>(), t(10).rotl::<3>(), t(16).rotl::<45>(), t(22).rotl::<61>()], + 2 => [t(1).rotl::<1>(), t(7).rotl::<6>(), t(13).rotl::<25>(), t(19).rotl::<8>(), t(20).rotl::<18>()], + 3 => [t(4).rotl::<27>(), t(5).rotl::<36>(), t(11).rotl::<10>(), t(17).rotl::<15>(), t(23).rotl::<56>()], + _ => [t(2).rotl::<62>(), t(8).rotl::<55>(), t(14).rotl::<39>(), t(15).rotl::<41>(), t(21).rotl::<2>()], + }; + row +} + +/// One round of θ, ρ, π, χ and ι, `c` holding the column parities of `a` before and after. +#[inline(always)] +fn round(a: &mut [S; STATE_LANES], c: &mut [S; 5], rc: u64) { + let d: [S; 5] = core::array::from_fn(|x| S::theta_d(c[(x + 4) % 5], c[(x + 1) % 5])); + let t = |i: usize| a[i].xor(d[i % 5]); + let rows: [[S; 5]; 5] = core::array::from_fn(|y| row(y, t)); + for (y, &[b0, b1, b2, b3, b4]) in rows.iter().enumerate() { + let mut row = [ + S::chi(b0, b1, b2), + S::chi(b1, b2, b3), + S::chi(b2, b3, b4), + S::chi(b3, b4, b0), + S::chi(b4, b0, b1), ]; - for (y, &[b0, b1, b2, b3, b4]) in rows.iter().enumerate() { - let mut row = [ - S::chi(b0, b1, b2), - S::chi(b1, b2, b3), - S::chi(b2, b3, b4), - S::chi(b3, b4, b0), - S::chi(b4, b0, b1), - ]; - if y == 0 { - row[0] = row[0].xor(S::splat(rc)); - } - for x in 0..5 { - a[5 * y + x] = row[x]; - c[x] = if y == 0 { row[x] } else { c[x].xor(row[x]) }; - } + if y == 0 { + row[0] = row[0].xor(S::splat(rc)); + } + for x in 0..5 { + a[5 * y + x] = row[x]; + c[x] = if y == 0 { row[x] } else { c[x].xor(row[x]) }; } } } +/// Lane `out` of the permutation of `a`, whose column parities are `c`, with the last round computing that lane alone. +/// The first round stands alone, so a caller permuting many states that differ in a few lanes can hoist the work on +/// the others out of its loop. +#[inline(always)] +fn permute_to_lane(a: &mut [S; STATE_LANES], mut c: [S; 5], out: usize) -> S { + round(a, &mut c, RC[0]); + for &rc in &RC[1..ROUNDS - 1] { + round(a, &mut c, rc); + } + let d: [S; 5] = core::array::from_fn(|x| S::theta_d(c[(x + 4) % 5], c[(x + 1) % 5])); + let b = row(out / 5, |i: usize| a[i].xor(d[i % 5])); + let x = out % 5; + let lane = S::chi(b[x], b[(x + 1) % 5], b[(x + 2) % 5]); + if out == 0 { + lane.xor(S::splat(RC[ROUNDS - 1])) + } else { + lane + } +} + /// One state: the scalar permutation, and the batched one where no vector backend applies. impl Lane for u64 { const WIDTH: usize = 1; @@ -246,6 +285,34 @@ pub fn digest_halves_from(template: &[u64; STATE_LANES], at: usize, values: &[u1 } } +/// The first of `nonces` that, written into lane `AT` of `template`, leaves lane `out` of the permuted state zero under +/// `mask`, [`LANES`] nonces at a time. `AT` is a constant so that the work on the lanes no nonce touches, the same in +/// every state, leaves the loop; the last round computes lane `out` alone. +pub fn first_nonce( + template: &[u64; STATE_LANES], + nonces: Range, + out: usize, + mask: u64, +) -> Option { + const { assert!(AT < STATE_LANES) }; + assert!(out < STATE_LANES); + let template: [Backend; STATE_LANES] = core::array::from_fn(|i| Backend::splat(template[i])); + let mut first = nonces.start; + while first < nonces.end { + let mut st = template; + st[AT] = Backend::load(&core::array::from_fn::(|l| first.wrapping_add(l as u64))); + let c = parities(&st); + let mut words = [0; LANES]; + permute_to_lane(&mut st, c, out).store(&mut words); + let n = (nonces.end - first).min(LANES as u64) as usize; + if let Some(l) = words[..n].iter().position(|w| w & mask == 0) { + return Some(first + l as u64); + } + first = first.saturating_add(LANES as u64); + } + None +} + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] mod x86 { use super::Lane; diff --git a/crates/keccak256/src/lib.rs b/crates/keccak256/src/lib.rs index a590cf5654..115129d654 100644 --- a/crates/keccak256/src/lib.rs +++ b/crates/keccak256/src/lib.rs @@ -3,7 +3,7 @@ //! tree. //! //! - [`keccak_f`], the Keccak-f\[1600\] permutation, which the VM's `KECCAK` opcode computes and `keccak_gkr` -//! proves, and [`keccak_f_many`], its batched form. +//! proves, and [`keccak_f_many`], its batched form; [`first_nonce`], the provers' proof-of-work search over it. //! - [`hash`] / [`Hasher`], Keccak-256 over bytes, and [`hash_many`], its batched form over equal-length inputs. //! - [`positions`], the permutation one bit position at a time, as `keccak_gkr` proves it. //! @@ -91,7 +91,7 @@ pub const END_BIT: u64 = (PAD_LAST as u64) << 56; mod lanes; pub mod positions; -pub use lanes::{LANES, digest_halves_from, hash_many, hash_many_from_state, keccak_f_many}; +pub use lanes::{LANES, digest_halves_from, first_nonce, hash_many, hash_many_from_state, keccak_f_many}; #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] mod single_avx512; @@ -367,4 +367,30 @@ mod tests { assert_eq!(got, want); } } + + /// The batched nonce search, from unaligned starts and to unaligned ends, against permuting each candidate. + #[test] + fn first_nonce_matches_keccak_f() { + fn check(out: usize, mask: u64) { + let template: [u64; STATE_LANES] = std::array::from_fn(|i| (i as u64).wrapping_mul(0x9E37_79B9_7F4A_7C15)); + let pass = |nonce: u64| { + let mut state = template; + state[AT] = nonce; + keccak_f(&mut state); + state[out] & mask == 0 + }; + for (start, end) in [(0, 200), (3, 61), (17, 18), (5, 5), (u64::MAX - 70, u64::MAX)] { + let want = (start..end).find(|&nonce| pass(nonce)); + assert_eq!( + first_nonce::(&template, start..end, out, mask), + want, + "{AT} {out} {start}..{end}" + ); + } + } + check::<0>(2, 0xf); + check::<4>(0, 0x7); + check::<12>(2, 0x1f); + check::<24>(24, 0x3); + } } From 9e6527ecdc16a9485d43c54f65d60f54fd05dd44 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:28:20 +0000 Subject: [PATCH 05/18] experimental lean_vm_core: bus GKR vertical kernels on AVX2+VPCLMULQDQ The transposed eight-lane kernels of the bus GKR (`products_into`, the round message and the fused fold) ran only with AVX-512. They now also run with AVX2 and VPCLMULQDQ, on `f128x8`'s AVX2 `F128x8`: two `x4` halves of four lanes. The message runs one half after the other on `x4::F128x4`, so its live set fits sixteen YMM registers; interleaving the halves, or looping over them, measured slower. `x4` gains `Sum4` (a lane sum of unreduced products, Karatsuba's terms summed before one fold), `F128x4::mul_sum` and `Mul`; `f128x8`'s AVX2 arm gains `load_runs`, `store_interleaved` and `halves`. The AVX-512 path is unchanged. Single thread, core 32 of an EPYC 9454P, `-C target-cpu=x86-64-v3 +pclmulqdq,+vpclmulqdq` (vp), upstream main vs this, back to back: - a 2^18-row layer of three trees (min of 8): first message 10.36 -> 6.20 ms, first fused fold 11.03 -> 7.18, second 5.24 -> 4.84, the rest 5.31 -> 3.80; `products_into` over 2^20 elements 1.75 -> 1.10 ms. - `aggregate --xmss 100`, three pairs: Bus GKR 61.3 / 61.7 / 61.1 -> 40.4 / 40.0 / 41.4 ms; Bus leaves 29.9 -> 27.8 ms (the four- and sixteen-tuple products). --- .../lean_vm_core/src/gkr.rs | 113 +++++++++++------- .../primitives/src/field/f128x8.rs | 41 +++++++ .../primitives/src/field/x4.rs | 57 ++++++++- 3 files changed, 170 insertions(+), 41 deletions(-) diff --git a/crates/experimental_prover/lean_vm_core/src/gkr.rs b/crates/experimental_prover/lean_vm_core/src/gkr.rs index a5922c160c..1711e1317a 100644 --- a/crates/experimental_prover/lean_vm_core/src/gkr.rs +++ b/crates/experimental_prover/lean_vm_core/src/gkr.rs @@ -6,7 +6,7 @@ //! Leaves and every layer are `E`-valued (the bus fingerprints mix `K`-columns //! into `E` upstream, [`crate::leaf`]). //! -//! On AVX-512 the large rounds run eight pairs of rows at a time in transposed form +//! With VPCLMULQDQ the large rounds run eight pairs of rows at a time in transposed form //! (`vertical`), and skip the runs of rows known to be the identity. use std::ops::Range; @@ -41,9 +41,9 @@ fn sum_tasks(tasks: usize, parallel: bool, f: impl Fn(usize) -> [F128; 4] + Sync /// `out[i] = ∏ current[4i..4i+4]`, for `out.len() == current.len() / 4`. pub(crate) fn products_into(current: &[F128], out: &mut [F128]) { debug_assert_eq!(out.len(), current.len() / 4); - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] let done = vertical::products_into(current, out); - #[cfg(not(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq")))] + #[cfg(not(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq")))] let done = 0; for (row, slot) in out.iter_mut().enumerate().skip(done) { let [left, right] = mul2( @@ -161,8 +161,8 @@ struct SplitEq { hi: Vec, lo_bits: usize, /// `lo` eight entries at a time, for the vertical kernels. - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] - lo8: Vec, + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + lo8: Vec, } impl SplitEq { @@ -171,13 +171,8 @@ impl SplitEq { let (low, high) = vars.split_at(lo_bits); let lo = eq_table(low); Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] - lo8: lo - .as_chunks::<8>() - .0 - .iter() - .map(primitives::field::x8::F128x8::load) - .collect(), + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + lo8: lo.as_chunks::<8>().0.iter().map(vertical::load_eight).collect(), lo, hi: eq_table(high), lo_bits, @@ -264,7 +259,7 @@ fn runs<'a>(range: Range, ones: &'a [Range]) -> impl Iterator, - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] spare: Vec, /// Runs of rows known to be the identity, sorted: the kernels skip them, and /// the vertical ones leave their groups unwritten. @@ -295,9 +290,9 @@ impl QuaternaryLayerState { Self { values, logical_rows: width, - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] groups: Vec::new(), - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] spare: Vec::new(), ones: Vec::new(), } @@ -305,7 +300,7 @@ impl QuaternaryLayerState { /// `(q(0)+q(1), [X²]q, [X³]q, [X⁴]q)`. fn round_message(&self, eq: &SplitEq) -> [F128; 4] { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] if self.logical_rows >= VERTICAL_ROWS { return vertical::message(&self.values, &self.ones, eq); } @@ -313,13 +308,13 @@ impl QuaternaryLayerState { } fn fold(&mut self, challenge: F128) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] self.leave_groups(); self.fold_scalar(challenge); } /// The vertical kernels' view of the rows, and the buffer the fold writes. - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] fn split(&mut self) -> (vertical::Input<'_>, &mut Vec) { let data = if self.groups.is_empty() { vertical::Data::Rows(&self.values) @@ -330,7 +325,7 @@ impl QuaternaryLayerState { } /// Move the rows back into `values` for the scalar kernels. - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] fn leave_groups(&mut self) { if !self.groups.is_empty() { self.values = vertical::to_rows(&self.groups, &self.ones); @@ -515,7 +510,7 @@ fn fold_and_message( challenge: F128, eq: &SplitEq, ) -> [[F128; 4]; N_BUS_TREES] { - #[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] { if trees[0].logical_rows >= VERTICAL_ROWS { let [a, b, c] = trees.each_mut().map(QuaternaryLayerState::split); @@ -535,18 +530,38 @@ fn fold_and_message( trees.each_mut().map(|tree| tree.fold_and_message_scalar(challenge, eq)) } -/// The AVX-512 kernels: eight pairs of rows at a time, transposed so that every -/// product is lane-wise ([`primitives::field::x8`]). After its first fold a layer -/// keeps its rows transposed, in [`Group`]s, until it is small enough for the scalar -/// kernels. -#[cfg(all(target_arch = "x86_64", target_feature = "avx512f", target_feature = "vpclmulqdq"))] +/// The VPCLMULQDQ kernels: eight pairs of rows at a time, transposed so that every +/// product is lane-wise (`primitives::field::x8` on AVX-512, `f128x8`'s two halves +/// of `x4` lanes on AVX2). After its first fold a layer keeps its rows transposed, +/// in [`Group`]s, until it is small enough for the scalar kernels. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] mod vertical { use super::{N_BUS_TREES, SplitEq, add4, coarsen, sum_tasks}; use primitives::field::F128; - use primitives::field::x8::{F128x8, Sum8}; + // A message runs on `Lane`s: all eight lanes on AVX-512, and on AVX2 a four-lane half at a time, which + // keeps its live set within the sixteen YMM registers. + cfg_select! { + target_feature = "avx512f" => { + use primitives::field::x8::{F128x8 as Lane, Sum8 as Sum}; + pub(super) use primitives::field::x8::F128x8; + } + _ => { + use primitives::field::x4::{F128x4 as Lane, Sum4 as Sum}; + pub(super) use primitives::field::f128x8::F128x8; + } + } use primitives::stream::Stream; use std::ops::Range; + /// Eight consecutive elements. + #[inline(always)] + pub(super) fn load_eight(src: &[F128; 8]) -> F128x8 { + cfg_select! { + target_feature = "avx512f" => F128x8::load(src), + _ => F128x8::from_array(src), + } + } + /// Whether `ones` covers `rows`. fn covered(ones: &[Range], rows: Range) -> bool { let i = ones.partition_point(|r| r.start <= rows.start); @@ -573,10 +588,10 @@ mod vertical { } } - /// Add eight pairs' `eq`-weighted summands. The weight scales child 0's line, and - /// with it every product, so it costs two products rather than one per coefficient. + /// Add the `eq`-weighted summands of a group's pairs, in `Lane`s. The weight scales child 0's line, + /// and with it every product, so it costs two products rather than one per coefficient. #[inline(always)] - fn add_message(g: &Group, eq: F128x8, acc: &mut [Sum8; 4]) { + fn add_message(g: [Lane; 8], eq: Lane, acc: &mut [Sum; 4]) { let a0 = eq * g[0]; let d0 = eq * (g[0] + g[4]); let left0 = a0 * g[1]; @@ -601,7 +616,7 @@ mod vertical { struct Message<'a> { eq: &'a SplitEq, block: usize, - sums: [Sum8; 4], + sums: [Sum; 4], total: [F128; 4], } @@ -610,7 +625,7 @@ mod vertical { Self { eq, block: first >> (eq.lo_bits - 3), - sums: [Sum8::ZERO; 4], + sums: [Sum::ZERO; 4], total: [F128::ZERO; 4], } } @@ -622,11 +637,20 @@ mod vertical { self.flush(); self.block = g >> shift; } - add_message(group, self.eq.lo8[g & ((1 << shift) - 1)], &mut self.sums); + let eq = self.eq.lo8[g & ((1 << shift) - 1)]; + cfg_select! { + target_feature = "avx512f" => add_message(*group, eq, &mut self.sums), + _ => { + // One half after the other, unrolled: a loop over them runs slower. + let (e, h) = (eq.halves(), group.map(F128x8::halves)); + add_message(h.map(|lanes| lanes[0]), e[0], &mut self.sums); + add_message(h.map(|lanes| lanes[1]), e[1], &mut self.sums); + } + } } fn flush(&mut self) { - let sums = std::mem::replace(&mut self.sums, [Sum8::ZERO; 4]).map(Sum8::reduce); + let sums = std::mem::replace(&mut self.sums, [Sum::ZERO; 4]).map(Sum::reduce); let scaled = primitives::field::mul4([self.eq.hi[self.block]; 4], sums); self.total = add4(self.total, scaled); } @@ -658,7 +682,10 @@ mod vertical { let fold = |g: &Group, c: usize| g[c] + r * (g[c] + g[4 + c]); let mut out = [F128x8::splat(F128::ZERO); 8]; for c in 0..4 { - (out[c], out[4 + c]) = F128x8::deinterleave(fold(a, c), fold(b, c)); + (out[c], out[4 + c]) = cfg_select! { + target_feature = "avx512f" => F128x8::deinterleave(fold(a, c), fold(b, c)), + _ => F128x8::even_odd(fold(a, c), fold(b, c)), + }; } out } @@ -775,10 +802,16 @@ mod vertical { let (chunks, _) = out.as_chunks_mut::<16>(); for (slot, run) in chunks.iter_mut().zip(current.as_chunks::<64>().0) { let g = F128x8::load_runs(run); - let (lo, hi) = F128x8::interleave(product(g[0], g[1], g[2], g[3]), product(g[4], g[5], g[6], g[7])); - let (first, second) = slot.split_at_mut(8); - lo.store(first.try_into().unwrap()); - hi.store(second.try_into().unwrap()); + let (a, b) = (product(g[0], g[1], g[2], g[3]), product(g[4], g[5], g[6], g[7])); + cfg_select! { + target_feature = "avx512f" => { + let (lo, hi) = F128x8::interleave(a, b); + let (first, second) = slot.split_at_mut(8); + lo.store(first.try_into().unwrap()); + hi.store(second.try_into().unwrap()); + } + _ => F128x8::store_interleaved(a, b, slot), + } } 16 * chunks.len() } diff --git a/crates/experimental_prover/primitives/src/field/f128x8.rs b/crates/experimental_prover/primitives/src/field/f128x8.rs index f94bf02909..3bceab7abe 100644 --- a/crates/experimental_prover/primitives/src/field/f128x8.rs +++ b/crates/experimental_prover/primitives/src/field/f128x8.rs @@ -409,6 +409,47 @@ mod avx2 { let ((ae, ao), (be, bo)) = (split(a), split(b)); (Self([ae, be]), Self([ao, bo])) } + + /// Eight runs of eight consecutive elements, transposed: `out[j]` lane `p` is `src[8p + j]`. + #[inline(always)] + pub fn load_runs(src: &[F128; 64]) -> [Self; 8] { + let at = |p: usize, j: usize| src[8 * p + j..].as_ptr().cast::<__m128i>(); + std::array::from_fn(|j| { + Self(std::array::from_fn(|h| { + // Runs `4h + i` and `4h + i + 2` in one register; the unpacks then put lane `i` in qword `i`. + // SAFETY: the module's cfg enables avx2; each address is one element, 16 readable bytes. + let pair = |i: usize| unsafe { _mm256_loadu2_m128i(at(4 * h + i + 2, j), at(4 * h + i, j)) }; + let (a, b) = (pair(0), pair(1)); + // SAFETY: the module's cfg enables avx2. + F128x4(unsafe { [_mm256_unpacklo_epi64(a, b), _mm256_unpackhi_epi64(a, b)] }) + })) + }) + } + + /// Stores the lanes `a0 b0 a1 b1 … a7 b7`. + #[inline(always)] + pub fn store_interleaved(a: Self, b: Self, dst: &mut [F128; 16]) { + let q = dst.as_mut_ptr().cast::<__m256i>(); + for h in 0..2 { + let ([a0, a1], [b0, b1]) = (a.0[h].0, b.0[h].0); + // SAFETY: the module's cfg enables avx2; `dst` is eight writable 32-byte pairs of elements. + unsafe { + // Elements `[x0, x2]` and `[x1, x3]` of each. + let (al, ah) = (_mm256_unpacklo_epi64(a0, a1), _mm256_unpackhi_epi64(a0, a1)); + let (bl, bh) = (_mm256_unpacklo_epi64(b0, b1), _mm256_unpackhi_epi64(b0, b1)); + _mm256_storeu_si256(q.add(4 * h), _mm256_permute2x128_si256::<0x20>(al, bl)); + _mm256_storeu_si256(q.add(4 * h + 1), _mm256_permute2x128_si256::<0x20>(ah, bh)); + _mm256_storeu_si256(q.add(4 * h + 2), _mm256_permute2x128_si256::<0x31>(al, bl)); + _mm256_storeu_si256(q.add(4 * h + 3), _mm256_permute2x128_si256::<0x31>(ah, bh)); + } + } + } + + /// Lanes `0..4` and `4..8`. + #[inline(always)] + pub fn halves(self) -> [F128x4; 2] { + self.0 + } } impl core::ops::Add for F128x8 { diff --git a/crates/experimental_prover/primitives/src/field/x4.rs b/crates/experimental_prover/primitives/src/field/x4.rs index 7a63f4b7ff..2cdf76aa95 100644 --- a/crates/experimental_prover/primitives/src/field/x4.rs +++ b/crates/experimental_prover/primitives/src/field/x4.rs @@ -3,7 +3,7 @@ //! and the halves of the eight-lane `f128x8` types. use core::arch::x86_64::*; -use core::ops::{Add, BitXorAssign, Mul}; +use core::ops::{Add, BitXor, BitXorAssign, Mul}; use super::gf2_64::F64; use super::gf2_64x2::x86_64::{fold, reduce_halves256 as reduce_halves, times_x256}; @@ -26,6 +26,10 @@ pub struct Mixed4 { odd: [__m256i; 2], } +/// The sum over lanes of unreduced products, y-folded: per coefficient, two 128-bit partials. +#[derive(Clone, Copy, Debug)] +pub struct Sum4([__m256i; 2]); + #[inline(always)] fn xor(a: __m256i, b: __m256i) -> __m256i { // SAFETY: the module's cfg enables avx2. @@ -133,6 +137,57 @@ impl F128x4 { odd: karatsuba::<0x11>(self.0, rhs.0), } } + + /// The lane-wise products, summed over the lanes and left unreduced. Karatsuba's products are summed + /// before their one fold. + #[inline(always)] + pub fn mul_sum(self, rhs: Self) -> Sum4 { + let both = |x: __m256i, y: __m256i| xor(clmul::<0x00>(x, y), clmul::<0x11>(x, y)); + let ([a0, a1], [b0, b1]) = (self.0, rhs.0); + let (m0, m1, m01) = (both(a0, b0), both(a1, b1), both(xor(a0, a1), xor(b0, b1))); + // SAFETY: the module's cfg enables avx2. + Sum4(fold(m0, m1, m01, xor, |p| unsafe { times_x256(p) })) + } +} + +impl Mul for F128x4 { + type Output = Self; + #[inline(always)] + fn mul(self, rhs: Self) -> Self { + self.mul_unreduced(rhs).reduce() + } +} + +impl Sum4 { + // SAFETY: an all-zero bit pattern is a valid `__m256i`. + pub const ZERO: Self = Self(unsafe { std::mem::zeroed() }); + + /// The lane sum, reduced. + #[inline] + pub fn reduce(self) -> F128 { + let coeffs = self.0.map(|v| { + // SAFETY: both are 32 bytes. + let w: [u64; 4] = unsafe { std::mem::transmute(v) }; + [w[0] ^ w[2], w[1] ^ w[3]] + }); + F128Unreduced::from_coeffs(coeffs).reduce() + } +} + +impl BitXorAssign for Sum4 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + self.0 = std::array::from_fn(|k| xor(self.0[k], rhs.0[k])); + } +} + +impl BitXor for Sum4 { + type Output = Self; + #[inline(always)] + fn bitxor(mut self, rhs: Self) -> Self { + self ^= rhs; + self + } } /// The embedding of the base field. From 918df52d14153d54689a0871e6684f09f0da3b45 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:37:20 +0000 Subject: [PATCH 06/18] experimental pcs: register-tiled NTT sweeps on AVX2 without VPCLMULQDQ The radix-8 and radix-4 register kernels now also build where AVX2 has only 128-bit PCLMULQDQ (the haswell tier, and AVX-512 without VPCLMULQDQ), with one four-lane vector per row in flight instead of two. Before, that tier ran each butterfly through memory, a row pair at a time. The product is shared with the row-pair kernel as `mul_xor_ymm`; the VPCLMULQDQ and AVX-512 arms are unchanged, and the transposed butterfly keeps its old arms. Single thread, `aggregate --xmss 100`, AMD EPYC 9454P core 28, same-core A/B against upstream main, three alternating passes each, ms (base / new): hsw (-C target-cpu=haswell): proving time 1346-1349 / 1318-1319 Commit / NTT 208 / 195 Commit L1 57.3 / 53.8 Commit L2 25.6 / 24.5 Commit L3 11.2 / 10.9 Open L0 18.3 / 14.4-14.8 Open L1 6.5 / 5.3 NTT alone (no hashing, jemalloc), hsw: base encode (2^18 x 48, 44 lanes) 86 -> 73, extension encodes 2^17/2^16/2^15 x 32 words 21.0/7.5/2.15 -> 17.5/6.3/1.87; kernel 2.86-3.0 -> 2.44-2.49 ns per four-lane butterfly, against 2.43 for the same butterflies kept in registers. vp and gfni compile to the same kernels: proving time 982/982-984 and 937-938/937, Commit / NTT 173/173-174 and 174/174. --- .../pcs/src/ntt/kernels.rs | 125 ++++++++++-------- 1 file changed, 71 insertions(+), 54 deletions(-) diff --git a/crates/experimental_prover/pcs/src/ntt/kernels.rs b/crates/experimental_prover/pcs/src/ntt/kernels.rs index ebce4cfcf1..706fa26a03 100644 --- a/crates/experimental_prover/pcs/src/ntt/kernels.rs +++ b/crates/experimental_prover/pcs/src/ntt/kernels.rs @@ -130,13 +130,14 @@ pub(super) fn butterfly_interleaved_fused_3layer( } #[cfg(all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ))] { #[inline] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + #[target_feature(enable = "pclmulqdq", enable = "avx2")] + #[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] fn sweep(block: &mut [F64], src: Option<&[F64]>, t: &[F64; 7], eighth: usize, num_ntts: usize) { avx2::fused_sweep::<8, 7>( block, @@ -153,16 +154,16 @@ pub(super) fn butterfly_interleaved_fused_3layer( }, ); } - // SAFETY: this code is compiled only where VPCLMULQDQ and AVX2 are enabled. + // SAFETY: this code is compiled only where PCLMULQDQ and AVX2 are enabled, with VPCLMULQDQ when it is. unsafe { sweep(block, src, t, eighth, num_ntts) } } #[cfg(not(any( all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ) )))] fused_rows::<8>(block, src, eighth, num_ntts, |src, rows| { @@ -198,9 +199,9 @@ fn fused_sweep_avx512( all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ) )))] #[inline(always)] @@ -214,9 +215,9 @@ fn radix8_butterflies(src: Option<[&[F64]; 8]>, rows: &mut [&mut [F64]; 8], t: & all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ) )))] #[inline(always)] @@ -230,9 +231,9 @@ fn butterfly_rows(rows: &mut [&mut [F64]; N], top: usize, bot: u all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ) )))] #[inline(always)] @@ -278,13 +279,14 @@ pub(super) fn butterfly_interleaved_fused_2layer( } #[cfg(all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ))] { #[inline] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + #[target_feature(enable = "pclmulqdq", enable = "avx2")] + #[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] fn sweep(block: &mut [F64], src: Option<&[F64]>, t: &[F64; 3], quarter: usize, num_ntts: usize) { avx2::fused_sweep::<4, 3>( block, @@ -301,16 +303,16 @@ pub(super) fn butterfly_interleaved_fused_2layer( }, ); } - // SAFETY: this code is compiled only where VPCLMULQDQ and AVX2 are enabled. + // SAFETY: as above. unsafe { sweep(block, src, t, quarter, num_ntts) } } #[cfg(not(any( all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ) )))] fused_rows::<4>(block, src, quarter, num_ntts, |src, rows| { @@ -323,9 +325,9 @@ pub(super) fn butterfly_interleaved_fused_2layer( all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"), all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ) )))] #[inline(always)] @@ -470,8 +472,7 @@ fn butterfly_lanes_avx2(top: &mut [F64], bot: &mut [F64], twiddle: u64) { } } -/// Four butterflies `u' = u + v * t`, `v' = v + u'` in registers: the products one per 128-bit -/// lane, even lanes then odd lanes as in `mul_xor_avx512`, then one shift-and-XOR reduction. +/// Four butterflies `u' = u + v * t`, `v' = v + u'` in registers, by [`mul_xor_ymm`]. #[cfg(all( target_arch = "x86_64", target_feature = "pclmulqdq", @@ -487,20 +488,39 @@ fn butterfly_ymm( twiddle: u64, ) -> (core::arch::x86_64::__m256i, core::arch::x86_64::__m256i) { use core::arch::x86_64::*; + + let new_u = mul_xor_ymm(u, v, _mm256_set1_epi64x(twiddle as i64)); + (new_u, _mm256_xor_si256(v, new_u)) +} + +/// `u ^ v * tw` in each of the four lanes, `tw` broadcast: the products one per 128-bit lane, even lanes then +/// odd lanes as in `mul_xor_avx512`, then one shift-and-XOR reduction. +#[cfg(all( + target_arch = "x86_64", + target_feature = "pclmulqdq", + target_feature = "avx2", + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) +))] +#[inline] +#[target_feature(enable = "pclmulqdq", enable = "avx2")] +#[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] +fn mul_xor_ymm( + u: core::arch::x86_64::__m256i, + v: core::arch::x86_64::__m256i, + tw: core::arch::x86_64::__m256i, +) -> core::arch::x86_64::__m256i { + use core::arch::x86_64::*; use primitives::field::gf2_64x2::x86_64::reduce_halves256; #[cfg(target_feature = "vpclmulqdq")] - let (even, odd) = { - let tw = _mm256_set1_epi64x(twiddle as i64); - ( - _mm256_clmulepi64_epi128::<0x00>(v, tw), - _mm256_clmulepi64_epi128::<0x11>(v, tw), - ) - }; - // Four 128-bit multiplies; immediate bit 0 picks the qword of `v`. + let (even, odd) = ( + _mm256_clmulepi64_epi128::<0x00>(v, tw), + _mm256_clmulepi64_epi128::<0x11>(v, tw), + ); + // Four 128-bit multiplies by the twiddle in the low qword; immediate bit 0 picks the qword of `v`. #[cfg(not(target_feature = "vpclmulqdq"))] let (even, odd) = { - let tw = _mm_cvtsi64_si128(twiddle as i64); + let tw = _mm256_castsi256_si128(tw); let (v01, v23) = (_mm256_castsi256_si128(v), _mm256_extracti128_si256::<1>(v)); ( _mm256_set_m128i( @@ -513,8 +533,7 @@ fn butterfly_ymm( ), ) }; - let new_u = _mm256_xor_si256(u, reduce_halves256(even, odd)); - (new_u, _mm256_xor_si256(v, new_u)) + _mm256_xor_si256(u, reduce_halves256(even, odd)) } /// The butterflies of one row pair that the NEON kernel covers: whole groups of eight lanes. @@ -744,24 +763,27 @@ fn transpose_butterfly_lanes_avx512(top: &mut [F64], bot: &mut [F64], twiddle: u ); } -/// The AVX-512 register kernels at half the width, for a machine with VPCLMULQDQ but no AVX-512. +/// The AVX-512 register kernels at half the width, for a machine with AVX2 but not both VPCLMULQDQ and AVX-512. #[cfg(all( target_arch = "x86_64", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", target_feature = "avx2", - not(target_feature = "avx512f") + not(all(target_feature = "vpclmulqdq", target_feature = "avx512f")) ))] mod avx2 { use core::arch::x86_64::*; use primitives::field::F64; - use primitives::field::gf2_64x2::x86_64::reduce_halves256; use primitives::{load_at, store_at}; - use super::fused_rows; + use super::{fused_rows, mul_xor_ymm}; - /// Four-lane vectors of every row that a register kernel keeps in flight, as on AVX-512. + /// Four-lane vectors of every row that a register kernel keeps in flight: two as on AVX-512, but one with + /// 128-bit multiplies, where a vector's four products already hide each other's latency. + #[cfg(target_feature = "vpclmulqdq")] const LANE_VECTORS: usize = 2; + #[cfg(not(target_feature = "vpclmulqdq"))] + const LANE_VECTORS: usize = 1; /// A broadcast twiddle, and whether it is zero: the butterfly is then `v' = v + u`, with no product. pub(super) type Twiddle = (__m256i, bool); @@ -769,15 +791,6 @@ mod avx2 { /// The vectors of `N` rows held in registers: `v[row][k]` is vector `k` of the row. pub(super) type RowVectors = [[__m256i; LANE_VECTORS]; N]; - /// `u ^ v * tw` in each of the four lanes, as `mul_xor_avx512`. - #[inline] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] - fn mul_xor(u: __m256i, v: __m256i, tw: __m256i) -> __m256i { - let even = _mm256_clmulepi64_epi128::<0x00>(v, tw); - let odd = _mm256_clmulepi64_epi128::<0x11>(v, tw); - _mm256_xor_si256(u, reduce_halves256(even, odd)) - } - /// Words `4i..4i + n` of `src`, then zeros. /// /// A short tail goes through a padded copy rather than `vpmaskmovq`, which AMD lets fault on masked-out words. @@ -814,11 +827,12 @@ mod avx2 { /// Butterfly rows `top` and `bot` of every vector in flight. #[inline] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + #[target_feature(enable = "pclmulqdq", enable = "avx2")] + #[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] pub(super) fn butterfly_regs(v: &mut RowVectors, top: usize, bot: usize, (tw, zero): Twiddle) { for k in 0..LANE_VECTORS { let (u, w) = (v[top][k], v[bot][k]); - let new_u = if zero { u } else { mul_xor(u, w, tw) }; + let new_u = if zero { u } else { mul_xor_ymm(u, w, tw) }; v[top][k] = new_u; v[bot][k] = _mm256_xor_si256(w, new_u); } @@ -826,7 +840,8 @@ mod avx2 { /// The sweep of [`fused_rows`] with `kernel` on registers: [`fused_lanes`] for each row group. #[inline] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + #[target_feature(enable = "pclmulqdq", enable = "avx2")] + #[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] pub(super) fn fused_sweep( block: &mut [F64], src: Option<&[F64]>, @@ -848,7 +863,8 @@ mod avx2 { /// `fused_lanes_avx512` at half the width: [`LANE_VECTORS`] four-lane vectors of every row at a time. #[inline] - #[target_feature(enable = "vpclmulqdq", enable = "avx2")] + #[target_feature(enable = "pclmulqdq", enable = "avx2")] + #[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] fn fused_lanes( src: Option<[&[F64]; N]>, rows: &mut [&mut [F64]; N], @@ -900,6 +916,7 @@ mod avx2 { } /// `transpose_butterfly_lanes_avx512` at half the width: `s = u + v; u' = s; v' = s * t + v`. + #[cfg(target_feature = "vpclmulqdq")] #[inline] #[target_feature(enable = "vpclmulqdq", enable = "avx2")] pub(super) fn transpose_butterfly_lanes(top: &mut [F64], bot: &mut [F64], twiddle: u64) { @@ -909,14 +926,14 @@ mod avx2 { let (u, v): (__m256i, __m256i) = (load_at(top, 4 * i), load_at(bot, 4 * i)); let s = _mm256_xor_si256(u, v); store_at(top, 4 * i, s); - store_at(bot, 4 * i, mul_xor(v, s, tw)); + store_at(bot, 4 * i, mul_xor_ymm(v, s, tw)); } let tail = top.len() - 4 * vectors; if tail > 0 { let (u, v) = (load_prefix(top, vectors, tail), load_prefix(bot, vectors, tail)); let s = _mm256_xor_si256(u, v); store_prefix(top, vectors, tail, s); - store_prefix(bot, vectors, tail, mul_xor(v, s, tw)); + store_prefix(bot, vectors, tail, mul_xor_ymm(v, s, tw)); } } } From c1fc8efc384c10421b8314bd52379bff6bf0938d Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:36:08 +0000 Subject: [PATCH 07/18] experimental keccak_gkr: cofactor skip pass on x86 without GFNI Without GFNI the skip pass embedded and extended every position, then took per slot and position a product of two embeddings and two products of it with the weight's words: fifteen carry-less products a slot. Those products bound it on these tiers. `skip_clmul` takes a row's sum as `e[2] A + e[4] B + e[0] C`, the cofactors `A = w[0] e[1] + w[1] e[3]`, `B = w[2] e[3] + w[3] e[0]` and `C = w[4] e[1]` multiplied out on the 32 packs (ten products a pack) and extended like them. A cofactor has three words, so a slot takes nine products, each of an embedding and a word: 1490 a row instead of 1950. The extension's binary points of new vectors are stored, and its low mixed points summed as the products read them. With VPCLMULQDQ a register's two lanes are the two groups at the same output; with PCLMULQDQ alone the groups take a pass each. NEON and the portable form keep the row kernel. Measured on one core of an EPYC 9454P (Zen 4), single thread, `aggregate --xmss 100`, last of three passes, ms, base and new run back to back twice on the same core: vp tier (x86-64-v3 + VPCLMULQDQ): span base new base2 new2 Prove 990 941 977 940 Keccak reduction 397 348 384 348 Keccak / Layer / Skip 174.4 139.1 174.2 138.8 hsw tier (haswell): span base new base2 new2 Prove 1330 1280 1330 1270 Keccak reduction 611 553 612 551 Keccak / Layer / Skip 318.5 258.1 318.9 258.0 `SkipBlock::run` on one random block (microbenchmark, best of 4 runs): vp 194 us before, 152 us after; hsw 356 us before, 292 us after. Tried and rejected on the vp tier (us per block, same core): - The extension stored to a slot array, field by field, as 128-bit lanes (the products then reading two slots a register): 262 against 194. - The extension stored to an output array a register at a time: 216 against 194. Storing only the new vectors and summing the low points as the products read them is the kept 152. - Residues without GFNI (nibble lookups for the affine maps, shift and add for the byte products): not built; a byte product costs about sixty AVX2 operations against one, and the products dominate. --- .../keccak_gkr/src/kernels.rs | 12 +- .../experimental_prover/keccak_gkr/src/lib.rs | 15 +- .../keccak_gkr/src/skip_clmul.rs | 212 ++++++++++++++++++ 3 files changed, 234 insertions(+), 5 deletions(-) create mode 100644 crates/experimental_prover/keccak_gkr/src/skip_clmul.rs diff --git a/crates/experimental_prover/keccak_gkr/src/kernels.rs b/crates/experimental_prover/keccak_gkr/src/kernels.rs index c8f9b9db61..8083708c1c 100644 --- a/crates/experimental_prover/keccak_gkr/src/kernels.rs +++ b/crates/experimental_prover/keccak_gkr/src/kernels.rs @@ -3,8 +3,8 @@ //! //! Each has a portable form, an AVX-512 one (AVX-512BW, VPCLMULQDQ, GFNI), an AVX2 one //! (PCLMULQDQ, and VPCLMULQDQ where it is) and a NEON one (PMULL), which agree bit for bit. The -//! vector forms are compiled only where their features are. With AVX2 and GFNI a block's -//! [`skip_row`]s are summed on residues instead (`skip_gfni`). +//! vector forms are compiled only where their features are. On x86 a block's [`skip_row`]s are +//! summed on residues with GFNI (`skip_gfni`), and through cofactors without it (`skip_clmul`). use primitives::field::gf2_64::mul_wide; @@ -256,6 +256,14 @@ cfg_select! { all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni") => { pub(crate) use crate::skip_gfni::SkipBlock; } + all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + not(all(target_feature = "avx512f", target_feature = "vpclmulqdq")) + ) => { + pub(crate) use crate::skip_clmul::SkipBlock; + } _ => { /// [`skip_row`] over every row and slice of a block, a position at a time. pub(crate) struct SkipBlock<'a> { diff --git a/crates/experimental_prover/keccak_gkr/src/lib.rs b/crates/experimental_prover/keccak_gkr/src/lib.rs index a0970c1bcc..b1b2008378 100644 --- a/crates/experimental_prover/keccak_gkr/src/lib.rs +++ b/crates/experimental_prover/keccak_gkr/src/lib.rs @@ -86,9 +86,10 @@ //! of eight positions' weights applied to bit-transposed products (`products_gfni`); elsewhere, //! the positions grouped by each byte of their weight (`prover`'s digit buckets). //! -//! The four rounds' sums are binary polynomials of degree at most 173. With AVX2 and GFNI they -//! are taken on residues modulo 24 irreducibles of degree 8, where a product is a byte product -//! (`skip_gfni`). +//! The four rounds' sums are binary polynomials of degree at most 173. With GFNI (x86-64) they are +//! taken on residues modulo 24 irreducibles of degree 8, where a product is a byte product +//! (`skip_gfni`). Without it, x86-64 multiplies out the weights on the packs first, so that a +//! sum takes fewer carry-less products (`skip_clmul`). //! //! The rounds that remain fold a table of one field element per half block and position. A pass over it //! binds two rounds' variables: over each quad of the rows it writes it sums the next round and, as a @@ -114,6 +115,14 @@ mod products_gfni; mod protocol; mod prover; pub mod rmfe; +#[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + not(target_feature = "gfni"), + not(all(target_feature = "avx512f", target_feature = "vpclmulqdq")) +))] +mod skip_clmul; #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] mod skip_gfni; #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] diff --git a/crates/experimental_prover/keccak_gkr/src/skip_clmul.rs b/crates/experimental_prover/keccak_gkr/src/skip_clmul.rs new file mode 100644 index 0000000000..fcc3706adc --- /dev/null +++ b/crates/experimental_prover/keccak_gkr/src/skip_clmul.rs @@ -0,0 +1,212 @@ +//! The four bit-level rounds' sums over a block with carry-less products (x86 without GFNI): per +//! [`slot`], the polynomial [`skip_row`](crate::kernels::skip_row) sums over the block. +//! +//! A row's sum `sum over x of w[x] e[x + 1] e[x + 2]` is taken as `e[2] A + e[4] B + e[0] C`, with +//! the cofactors `A = w[0] e[1] + w[1] e[3]`, `B = w[2] e[3] + w[3] e[0]` and `C = w[4] e[1]`. +//! They are linear in the values, so they are multiplied out on the row's packs and extended like +//! them: a cofactor has three words, and a slot takes nine products of an embedding and a word +//! where [`skip_row`](crate::kernels::skip_row) takes fifteen of an embedding, a product and a +//! weight's word. + +use primitives::hash::positions::row_positions; + +use crate::clmul::{V, Vector}; +use crate::kernels::{GROUP, MIXED, PACKS, embed}; +use crate::witness::words; + +cfg_select! { + target_feature = "vpclmulqdq" => { + use core::arch::x86_64::*; + + /// Groups a register holds, a lane each. + const GROUPS: usize = 2; + + /// Packs `q` and `q + 1` of every group, a lane each. + #[inline(always)] + fn load_packs(e: &[u32; PACKS], q: usize) -> V { + // SAFETY: AVX2 is enabled at compile time (the module's cfg), and each load is of two + // packs of `e`. + unsafe { + let [low, high] = [0, 1].map(|g| _mm_loadl_epi64(e[GROUP * g + q..][..2].as_ptr().cast())); + _mm256_cvtepu32_epi64(_mm_unpacklo_epi64(low, high)) + } + } + + /// Where a sum's lanes hold output `i` of group `half`: a register's lanes are the + /// groups' outputs `i`. + const fn at(half: usize, i: usize) -> usize { + GROUPS * i + half + } + } + _ => { + /// Groups a register holds, a lane each. + const GROUPS: usize = 1; + + /// Packs `q` and `q + 1`. + #[inline(always)] + fn load_packs(e: &[u32; PACKS], q: usize) -> V { + // SAFETY: two packs of `e`. + unsafe { V::load_packs(e[q..][..2].as_ptr()) } + } + + /// Where a sum's lanes hold output `i` of group `half`: a group's outputs, then the + /// other's. + const fn at(half: usize, i: usize) -> usize { + MIXED * half + i + } + } +} +const _: () = assert!(GROUPS == V::N); + +/// The lanes of two words a slot's products read: `[e[2], e[4]]`, `[A0, B0]`, `[A1, B1]`, +/// `[A2, B2]`, `[C0, C1]` and `[C2, e[0]]`, word `k` of a cofactor its coefficients from `64 k`. +const FIELDS: usize = 6; + +/// Per word of the cofactors, a block's sums of the products of the word, at [`at`] each output +/// of [`extend_group`](crate::kernels::extend_group) of each group. +type Acc = [[[u64; 2]; 2 * MIXED]; 3]; + +/// The pairs of lanes of a vector of four whose sums are its low mixed points, in the order of +/// [`extend_group`](crate::kernels::extend_group). +const LOW_MIXED: [(usize, usize); 4] = [(0, 1), (2, 3), (0, 2), (1, 3)]; + +/// [`skip_row`](crate::kernels::skip_row) over every row and slice of a block, a row at a time. +pub struct SkipBlock<'a> { + weights: &'a [[u64; 2]], + /// The embeddings of a row's positions. + e: [[u32; PACKS]; 5], + /// Per field, the nine vectors of four of [`extend_group`](crate::kernels::extend_group) of + /// [`GROUPS`] groups, a register each lane: the packs, then their sums. + vectors: [[[V; 4]; 9]; FIELDS], + /// The block's sums. + acc: Acc, +} + +impl<'a> SkipBlock<'a> { + /// For blocks summed at the position weights `weights`, as polynomials. + pub fn new(weights: &'a [[u64; 2]]) -> Box { + Box::new(Self { + weights, + e: [[0; PACKS]; 5], + vectors: [[[V::zero(); 4]; 9]; FIELDS], + acc: [[[0; 2]; 2 * MIXED]; 3], + }) + } + + /// Sum a block. + pub fn run(&mut self, block: &[u64]) { + self.acc = [[[0; 2]; 2 * MIXED]; 3]; + for r in 0..5 { + for z in 0..64 { + let js = row_positions(r, z); + for (e, &j) in self.e.iter_mut().zip(&js) { + *e = embed(words(block, j)); + } + // Each lane a weight's two words. + let w = js.map(|j| { + let [low, high] = self.weights[j]; + V::splat(low).unpacklo(V::splat(high)) + }); + for pass in (0..2).step_by(GROUPS) { + self.cofactors(pass, w); + self.products(pass); + } + } + } + } + + /// The [`FIELDS`] of the packs of the groups from `pass`, at the row's weights `w`, and the + /// vectors their sums make. + #[inline(always)] + fn cofactors(&mut self, pass: usize, w: [V; 5]) { + for q in (0..GROUP).step_by(2) { + let e: [V; 5] = self.e.each_ref().map(|e| load_packs(e, GROUP * pass + q)); + // An embedding times a weight: per pack of the pair, the products of its words. + let product = |e: V, w: V| { + [ + [e.clmul::<0x00>(w), e.clmul::<0x10>(w)], + [e.clmul::<0x01>(w), e.clmul::<0x11>(w)], + ] + }; + let sum = |a: [[V; 2]; 2], b: [[V; 2]; 2]| -> [[V; 2]; 2] { + std::array::from_fn(|h| std::array::from_fn(|k| a[h][k].xor(b[h][k]))) + }; + let a = sum(product(e[1], w[0]), product(e[3], w[1])); + let b = sum(product(e[3], w[2]), product(e[0], w[3])); + let c = product(e[1], w[4]); + for h in 0..2 { + // Pack `q + h`'s embeddings in the low word. + let e = e.map(|e| if h == 0 { e } else { e.high() }); + let ([a0, a1], [b0, b1], [c0, c1]) = (a[h], b[h], c[h]); + let fields = [ + e[2].unpacklo(e[4]), + a0.unpacklo(b0), + a0.high().xor(a1).unpacklo(b0.high().xor(b1)), + a1.high().unpacklo(b1.high()), + c0.unpacklo(c0.high().xor(c1)), + c1.high().unpacklo(e[0]), + ]; + for (vectors, field) in self.vectors.iter_mut().zip(fields) { + vectors[(q + h) / 4][(q + h) % 4] = field; + } + } + } + for v in &mut self.vectors { + for l in 0..4 { + let (v4, v5) = (v[0][l].xor(v[1][l]), v[2][l].xor(v[3][l])); + [v[4][l], v[5][l], v[6][l], v[7][l], v[8][l]] = + [v4, v5, v[0][l].xor(v[2][l]), v[1][l].xor(v[3][l]), v4.xor(v5)]; + } + } + } + + /// Adds the products of the groups from `pass` to their sums, the extension's low points + /// summed as the products read them. + #[inline(always)] + fn products(&mut self, pass: usize) { + let Self { vectors, acc, .. } = self; + for k in 0..9 { + let v = |l: usize| vectors.each_ref().map(|v| v[k][l]); + if k >= 4 { + for l in 0..4 { + accumulate(acc, pass, 4 * (k - 4) + l, v(l)); + } + } + for (j, (a, b)) in LOW_MIXED.into_iter().enumerate() { + let (a, b) = (v(a), v(b)); + accumulate(acc, pass, 20 + 4 * k + j, std::array::from_fn(|f| a[f].xor(b[f]))); + } + let all = vectors + .each_ref() + .map(|v| v[k][0].xor(v[k][1]).xor(v[k][2].xor(v[k][3]))); + accumulate(acc, pass, 56 + k, all); + } + } + + /// The sum at slot `i`: the products of word `k` of the cofactors from coefficient `64 k`. + pub fn get(&self, slot: usize) -> [u64; 3] { + let (half, i) = if slot < 128 { + (slot / 64, slot % 64) + } else { + (slot - 128, 64) + }; + let [s0, s1, s2] = self.acc.each_ref().map(|acc| acc[at(half, i)]); + [s0[0], s0[1] ^ s1[0], s1[1] ^ s2[0]] + } +} + +/// Adds the products of the [`FIELDS`] at output `i` of the groups from `pass` to their sums. +#[inline(always)] +fn accumulate(acc: &mut Acc, pass: usize, i: usize, [e, w0, w1, w2, wc, wd]: [V; FIELDS]) { + let sums = [ + e.clmul::<0x00>(w0).xor(e.clmul::<0x11>(w0)).xor(wd.clmul::<0x01>(wc)), + e.clmul::<0x00>(w1).xor(e.clmul::<0x11>(w1)).xor(wd.clmul::<0x11>(wc)), + e.clmul::<0x00>(w2).xor(e.clmul::<0x11>(w2)).xor(wd.clmul::<0x01>(wd)), + ]; + for (acc, sum) in acc.iter_mut().zip(sums) { + // Lane `g` is group `pass + g`. + let acc = acc[at(pass, i)..].as_mut_ptr().cast::(); + // SAFETY: `GROUPS` lanes of `acc`. + unsafe { V::store(acc, V::load(acc).xor(sum)) }; + } +} From b68a08f507c95c4f270caa4258ff3b0e72d3dc1c Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:11:10 +0000 Subject: [PATCH 08/18] experimental keccak_gkr: GFNI product sums on values at six points The GFNI outer products staged each lane byte's products from CLMUL, a word per pack, then transposed them to column bytes. A product of two embeddings has degree below 48, so its values at six points of GF(2^8) with distinct minimal polynomials fix it: the staging now takes the embeddings' values by affine byte maps of the witness bytes, the products' values by GF2P8MULB, and one 8x32 byte transpose per point. The sums run over the value bits, and the inverse of the evaluation map sends them back to coefficients once a layer. Single thread, `aggregate --xmss 100`, Zen 4 core 24, same core back to back, last pass, ms: span gfni base gfni native base native Product 89.0 49.6 50.7 39.3 Keccak 399.0 352.0 160.0 144.0 Per block (pack_products microbench, n = 4096, gfni): 87.4 -> 51.2 us; staging 49.7 -> 13.2 us, accumulation unchanged at about 30 us. --- .../keccak_gkr/src/products_gfni.rs | 371 +++++++++++++----- .../keccak_gkr/src/prover.rs | 5 +- 2 files changed, 285 insertions(+), 91 deletions(-) diff --git a/crates/experimental_prover/keccak_gkr/src/products_gfni.rs b/crates/experimental_prover/keccak_gkr/src/products_gfni.rs index 0461f90f1e..08373427b6 100644 --- a/crates/experimental_prover/keccak_gkr/src/products_gfni.rs +++ b/crates/experimental_prover/keccak_gkr/src/products_gfni.rs @@ -2,36 +2,43 @@ //! same values as [`crate::prover`]'s digit buckets, `sum over positions j of weights[j] (x) //! product(q, j)` for every pack `q` of a block and every coefficient of its product. //! +//! A product of two embeddings has degree below `8 * POINTS`, so it is fixed by its values at +//! [`POINTS`] points of GF(2^8), the field of GF2P8MULB, with distinct minimal polynomials. A value +//! is a byte: the embeddings' values are affine byte maps of the witness's bytes ([`EVAL`]), and +//! a product's are byte products. The sums are taken over the values' bits, which the inverse of +//! the evaluation map ([`CRT`]) sends back to coefficients once a layer ([`coefficients`]). +//! //! Eight consecutive `z` of one lane are a *lane byte*. For a lane byte's positions `j_0 .. j_7` //! and one byte `b` of their weights, GF2P8AFFINEQB with the matrix `M_b` maps a *column byte* -//! (bit `k` = one coefficient of `product(q, j_k)`) to byte `b` of the sum of the weights of the -//! positions whose coefficient is set: the lane byte's share of that coefficient's sum. One -//! instruction does this for 32 (pack, coefficient) column bytes at once. +//! (bit `k` = one bit of a value of `product(q, j_k)`) to byte `b` of the sum of the weights of +//! the positions whose bit is set: the lane byte's share of that bit's sum. One instruction does +//! this for 32 (pack, bit) column bytes at once. //! //! Per block, two phases: //! 1. per *row byte* with a nonzero weight (the same byte of the five lanes of a row, which `chi` -//! mixes): the five lane bytes' embeddings, the products of each active lane byte, and their -//! bit transposes into column bytes, staged register by register in a per-thread buffer; +//! mixes): the five lane bytes' values, the products of each active lane byte, and their byte +//! and bit transposes into column bytes, staged register by register in a per-thread buffer; //! 2. per staged register, every active lane byte's 16 matrices applied into accumulators held in //! registers (eight at a time on AVX2, all 16 with AVX-512), then one byte transpose back to //! field elements. //! -//! The 8x8 byte transpose without VPERMB ([`transpose_bytes8x8`]) leaves the positions in a fixed -//! order ([`TRANSPOSED_ORDER`]), which the matrices absorb ([`COLUMN_BIT_POSITION`]). -//! //! The two-phase outer-product scheme follows hashmancer-keccak's GFNI kernels; the code here is //! written for this crate's layout (16-instance packs, 128-bit weights). use core::arch::x86_64::*; use primitives::field::F128; -use verifier::keccak::BITS; +use verifier::keccak::{BITS, EMBED_BITS, PRODUCT_BITS}; -use crate::kernels::{PACKS, embed, products}; +use crate::kernels::PACKS; use crate::prover::PAYLOAD_BITS; -use crate::witness::words; +use crate::rmfe::RMFE; +use crate::witness::{BLOCK_WORDS, words}; -/// Staged registers per active lane byte: 32 column bytes each, six coefficient bytes per pack. +/// Points a product is held at, a byte each. +const POINTS: usize = 6; +const _: () = assert!(PRODUCT_BITS <= 8 * POINTS && PAYLOAD_BITS == 8 * POINTS * PACKS); +/// Staged registers per active lane byte: 32 column bytes each, one per pack and bit of a value. const STAGED: usize = PAYLOAD_BITS / 32; /// Bytes of a weight. const WEIGHT_BYTES: usize = 16; @@ -41,68 +48,237 @@ type V = __m256i; /// The GF2P8AFFINEQB source that turns the instruction into an 8x8 bit transpose of its matrix. const BIT_TRANSPOSE: i64 = 0x8040_2010_0804_0201_u64 as i64; -/// The qword order [`transpose_bytes8x8`] leaves: output qword `d` byte `n` is byte `d` of input -/// `TRANSPOSED_ORDER[n]`. -const TRANSPOSED_ORDER: [usize; 8] = [0, 1, 4, 5, 2, 3, 6, 7]; /// The position that bit `i` of a column byte stands for, after the byte and bit transposes. -const COLUMN_BIT_POSITION: [usize; 8] = { - let mut p = [0; 8]; +const COLUMN_BIT_POSITION: [usize; 8] = [7, 6, 5, 4, 3, 2, 1, 0]; + +/// The byte product: binary polynomials modulo `x^8 + x^4 + x^3 + x + 1`. +const fn gf_mul(a: u8, b: u8) -> u8 { + let (mut a, mut product) = (a, 0); let mut i = 0; while i < 8 { - p[i] = TRANSPOSED_ORDER[7 - i]; + if b >> i & 1 == 1 { + product ^= a; + } + a = a << 1 ^ if a >> 7 == 1 { 0x1b } else { 0 }; + i += 1; + } + product +} + +/// `POWERS[i][n]` is point `i` to the power `n`. A point is the least of its eight conjugates, the +/// roots of its irreducible, so no two share one; an element of a proper subfield has fewer. +const POWERS: [[u8; 8 * POINTS]; POINTS] = { + let mut powers = [[1u8; 8 * POINTS]; POINTS]; + let (mut i, mut t) = (0, 0u8); + while i < POINTS { + t += 1; + let (mut conjugate, mut least) = (t, true); + let mut k = 1; + while k < 8 { + conjugate = gf_mul(conjugate, conjugate); + least &= conjugate > t; + k += 1; + } + if least { + let mut n = 1; + while n < 8 * POINTS { + powers[i][n] = gf_mul(powers[i][n - 1], t); + n += 1; + } + i += 1; + } + } + powers +}; + +/// The value at point `i` of the polynomial with coefficient bits `p`. +const fn value(p: u32, i: usize) -> u8 { + let (mut v, mut n) = (0, 0); + while n < EMBED_BITS { + if p >> n & 1 == 1 { + v ^= POWERS[i][n]; + } + n += 1; + } + v +} + +/// The matrix of the affine byte map sending bit `i` of a byte to `images[i]`: the map reads +/// output bit `t` off matrix byte `7 - t`. +const fn byte_map(images: [u8; 8]) -> u64 { + let mut matrix = 0u64; + let mut t = 0; + while t < 8 { + let mut i = 0; + while i < 8 { + matrix |= ((images[i] >> t & 1) as u64) << (8 * (7 - t) + i); + i += 1; + } + t += 1; + } + matrix +} + +/// `EVAL[h][i]` sends byte `h` of a pack to its share of the value of the pack's embedding at +/// point `i`. +const EVAL: [[u64; POINTS]; 2] = { + let mut maps = [[0u64; POINTS]; 2]; + let mut h = 0; + while h < 2 { + let mut i = 0; + while i < POINTS { + let mut images = [0u8; 8]; + let mut b = 0; + while b < 8 { + images[b] = value(RMFE.emb[8 * h + b], i); + b += 1; + } + maps[h][i] = byte_map(images); + i += 1; + } + h += 1; + } + maps +}; + +/// The value of `ones` at each point. +const ONES: [u8; POINTS] = { + let mut ones = [0u8; POINTS]; + let mut i = 0; + while i < POINTS { + ones[i] = value(RMFE.ones, i); + i += 1; + } + ones +}; + +/// `CRT[n]` is the mask of the value bits (bit `t` of the value at point `i` at bit `8 i + t`) +/// whose sum is coefficient `n`: row `n` of the inverse of the evaluation map on polynomials of +/// fewer than `8 * POINTS` coefficients, by Gauss-Jordan elimination. +const CRT: [u64; 8 * POINTS] = { + const N: usize = 8 * POINTS; + // Row `8 i + t` sends a polynomial to bit `t` of its value at point `i`, beside the row of the + // identity that the row operations turn into the inverse's. + let mut rows = [[0u64; 2]; N]; + let mut r = 0; + while r < N { + let mut n = 0; + while n < N { + rows[r][0] |= ((POWERS[r / 8][n] >> (r % 8) & 1) as u64) << n; + n += 1; + } + rows[r][1] = 1 << r; + r += 1; + } + let mut c = 0; + while c < N { + let mut p = c; + while rows[p][0] >> c & 1 == 0 { + p += 1; + } + let pivot = rows[p]; + rows.swap(p, c); + let mut r = 0; + while r < N { + if r != c && rows[r][0] >> c & 1 == 1 { + rows[r][0] ^= pivot[0]; + rows[r][1] ^= pivot[1]; + } + r += 1; + } + c += 1; + } + let mut crt = [0u64; N]; + let mut n = 0; + while n < N { + crt[n] = rows[n][1]; + n += 1; + } + crt +}; + +/// `SLOT[q][8 i + t]` is the entry of [`Scratch::out`] holding bit `t` of pack `q`'s values at +/// point `i`. Value registers ([`evaluate`]) hold pack `16 s + 8 l + u` at byte `16 l + 8 s + u`, +/// and [`ProductSums::stage_lane_byte`] puts byte `16 l + 2 j + d` at qword `2 l + d` of register +/// `8 i + j`, whose byte `n` is entry `32 r + n`. +const SLOT: [[u16; 8 * POINTS]; PACKS] = { + let mut slot = [[0u16; 8 * POINTS]; PACKS]; + let mut i = 0; + while i < POINTS { + let mut j = 0; + while j < 8 { + let mut l = 0; + while l < 2 { + let mut d = 0; + while d < 2 { + let byte = 2 * j + d; + let q = 16 * (byte / 8) + 8 * l + byte % 8; + let mut t = 0; + while t < 8 { + slot[q][8 * i + t] = (32 * (8 * i + j) + 8 * (2 * l + d) + t) as u16; + t += 1; + } + d += 1; + } + l += 1; + } + j += 1; + } i += 1; } - p + slot }; -/// In-lane byte interleave of the lane's two qwords (16-bit element `d` = byte `d` of both). -const INTERLEAVE: [u8; 32] = { +/// Per-lane byte shuffle of a pack's two bytes apart: the lane's low bytes, then its high ones. +const SPLIT: [u8; 32] = { let mut t = [0u8; 32]; let mut i = 0; while i < 32 { let j = i % 16; - t[i] = if j % 2 == 0 { (j / 2) as u8 } else { (8 + j / 2) as u8 }; + t[i] = if j < 8 { 2 * j as u8 } else { 2 * (j - 8) as u8 + 1 }; i += 1; } t }; -/// 8x8 byte transpose of the qwords `a_0 .. a_7` (`p` = `a_0 .. a_3`, `q` = `a_4 .. a_7`): output -/// qword `d` (`d < 4` in the first register) byte `n` = byte `d` of `a_{TRANSPOSED_ORDER[n]}`. Two -/// in-lane byte shuffles, two 16-bit unpacks and two dword permutes. -/// -/// # Safety -/// AVX2 (the module's build). +/// The values of a block position's 32 packs' embeddings at each point: register `i` holds point +/// `i`, pack `16 s + 8 l + u` at byte `16 l + 8 s + u`. #[inline(always)] -unsafe fn transpose_bytes8x8(p: V, q: V) -> [V; 2] { +fn evaluate(words: &[u64; BLOCK_WORDS]) -> [V; POINTS] { + // SAFETY: AVX2 and GFNI are enabled at compile time (module cfg); the loads are of the eight + // words and of `SPLIT`. unsafe { - let il = _mm256_loadu_si256(INTERLEAVE.as_ptr().cast()); - let (p, q) = (_mm256_shuffle_epi8(p, il), _mm256_shuffle_epi8(q, il)); - // Dword d of lane 0: bytes d of (a0, a1, a4, a5); of lane 1: of (a2, a3, a6, a7). - let (e, f) = (_mm256_unpacklo_epi16(p, q), _mm256_unpackhi_epi16(p, q)); - let idx = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); - [_mm256_permutevar8x32_epi32(e, idx), _mm256_permutevar8x32_epi32(f, idx)] + let split = _mm256_loadu_si256(SPLIT.as_ptr().cast()); + let w0 = _mm256_shuffle_epi8(_mm256_loadu_si256(words.as_ptr().cast()), split); + let w1 = _mm256_shuffle_epi8(_mm256_loadu_si256(words.as_ptr().add(4).cast()), split); + let (low, high) = (_mm256_unpacklo_epi64(w0, w1), _mm256_unpackhi_epi64(w0, w1)); + std::array::from_fn(|i| { + _mm256_xor_si256( + _mm256_gf2p8affine_epi64_epi8(low, _mm256_set1_epi64x(EVAL[0][i] as i64), 0), + _mm256_gf2p8affine_epi64_epi8(high, _mm256_set1_epi64x(EVAL[1][i] as i64), 0), + ) + }) } } -/// 4x4 transpose of qwords: `out[i]` qword `k` = `r[k]` qword `i`. -/// -/// # Safety -/// AVX2. -#[inline(always)] -unsafe fn transpose_qwords4x4(r: [V; 4]) -> [V; 4] { - unsafe { - let t0 = _mm256_unpacklo_epi64(r[0], r[1]); - let t1 = _mm256_unpackhi_epi64(r[0], r[1]); - let t2 = _mm256_unpacklo_epi64(r[2], r[3]); - let t3 = _mm256_unpackhi_epi64(r[2], r[3]); - [ - _mm256_permute2x128_si256(t0, t2, 0x20), - _mm256_permute2x128_si256(t1, t3, 0x20), - _mm256_permute2x128_si256(t0, t2, 0x31), - _mm256_permute2x128_si256(t1, t3, 0x31), - ] - } +/// Per pack, the coefficients of the weighted sums of its products, from the weighted sums of +/// their value bits (entry [`SLOT`]`[q][8 i + t]` of `sums`). +pub fn coefficients(sums: &[F128]) -> Vec<[F128; PRODUCT_BITS]> { + assert_eq!(sums.len(), PAYLOAD_BITS); + SLOT.iter() + .map(|slot| { + let bits: [F128; 8 * POINTS] = std::array::from_fn(|r| sums[slot[r] as usize]); + std::array::from_fn(|n| { + let mut mask = CRT[n]; + let mut c = F128::ZERO; + while mask != 0 { + c += bits[mask.trailing_zeros() as usize]; + mask &= mask - 1; + } + c + }) + }) + .collect() } /// Where staged register `r` of active lane byte `a` lies in the buffer (`n_active` active lane @@ -189,7 +365,7 @@ impl ProductSums { } /// The weighted bit sums of a block (`BITS` positions of eight words) into `s.out`: entry - /// `48 q + c` is `sum over j of weights[j] * coefficient c of product(q, j)`. + /// [`SLOT`]`[q][8 i + t]` is `sum over j of weights[j] * bit t of product(q, j) at point i`. pub fn block_sums(&self, s: &mut Scratch, block: &[u64]) { let n_active = self.matrices.len(); self.stage(block, &mut s.staged, n_active); @@ -200,26 +376,29 @@ impl ProductSums { /// Phase 1: every active lane byte's column bytes into `staged[staged_index(r, a, n_active)]`. fn stage(&self, block: &[u64], staged: &mut [V], n_active: usize) { assert!(staged.len() >= STAGED * n_active); - let mut embedded = [[[0u32; PACKS]; 8]; 5]; - let mut prod = [[0u64; PACKS]; 8]; + // SAFETY: AVX2 is enabled at compile time. + let mut values = [[[unsafe { _mm256_setzero_si256() }; POINTS]; 8]; 5]; let mut a = 0; for &(y, zb, mask) in &self.row_bytes { let (y, zb) = (y as usize, zb as usize); - for (x, lane) in embedded.iter_mut().enumerate() { - for (k, e) in lane.iter_mut().enumerate() { - *e = embed(words(block, 64 * (x + 5 * y) + 8 * zb + k)); + for (x, lane) in values.iter_mut().enumerate() { + for (k, v) in lane.iter_mut().enumerate() { + *v = evaluate(words(block, 64 * (x + 5 * y) + 8 * zb + k)); } } for x in (0..5).filter(|x| mask >> x & 1 == 1) { - let (e0, e1, e2) = (&embedded[x], &embedded[(x + 1) % 5], &embedded[(x + 2) % 5]); - for (k, p) in prod.iter_mut().enumerate() { - *p = products(&e0[k], &e1[k], &e2[k]); - } assert!(a < n_active, "one staged column per active lane byte"); // SAFETY: AVX2 and GFNI are enabled at compile time; `a < n_active` (asserted) and - // `stage_lane_byte` writes registers `r < STAGED` only (`3 * 15 + 2`), so every store + // `stage_lane_byte` writes registers `r < STAGED` only (`8 * 5 + 7`), so every store // is inside `staged` (asserted size). - unsafe { Self::stage_lane_byte(&prod, staged.as_mut_ptr(), a, n_active) }; + unsafe { + Self::stage_lane_byte( + [&values[x], &values[(x + 1) % 5], &values[(x + 2) % 5]], + staged.as_mut_ptr(), + a, + n_active, + ) + }; a += 1; } } @@ -227,38 +406,54 @@ impl ProductSums { assert_eq!(a, n_active, "every active lane byte staged"); } - /// The column bytes of one lane byte's products (`prod[k][q]`: position `k`, pack `q`): register - /// `r` holds column qwords `4 r .. 4 r + 4`, qword `6 q + m` being coefficient byte `m` of pack - /// `q` (bit `i` of its byte `b`: coefficient `8 m + b` of position `COLUMN_BIT_POSITION[i]`). + /// The column bytes of one lane byte's products, from the values of its three factors' + /// embeddings (`e[f][k][i]`: factor `f`, position `k`, point `i`): register `8 i + j` holds, in + /// qword `2 l + d`, the column byte of each bit of the products' values at point `i` for the + /// pack at byte `16 l + 2 j + d` of a value register ([`SLOT`]). /// /// # Safety /// AVX2 and GFNI; `staged + staged_index(r, a, n_active)` writable for `r < STAGED`. #[inline(always)] - unsafe fn stage_lane_byte(prod: &[[u64; PACKS]; 8], staged: *mut V, a: usize, n_active: usize) { + unsafe fn stage_lane_byte(e: [&[[V; POINTS]; 8]; 3], staged: *mut V, a: usize, n_active: usize) { unsafe { let bit_transpose = _mm256_set1_epi64x(BIT_TRANSPOSE); - let ld = |k: usize, q: usize| _mm256_loadu_si256(prod[k].as_ptr().add(q).cast()); - for qq in (0..PACKS).step_by(4) { - // t[h][i] qword k = the product of pack qq + i at position 4 h + k. - let t = [ - transpose_qwords4x4([ld(0, qq), ld(1, qq), ld(2, qq), ld(3, qq)]), - transpose_qwords4x4([ld(4, qq), ld(5, qq), ld(6, qq), ld(7, qq)]), - ]; - let columns: [[V; 2]; 4] = std::array::from_fn(|i| { - let [lo, hi] = transpose_bytes8x8(t[0][i], t[1][i]); + for i in 0..POINTS { + let ones = _mm256_set1_epi8(ONES[i] as i8); + // `ones * (a + c) + b * c` at point `i`, per position. + let p: [V; 8] = std::array::from_fn(|k| { + let (a, b, c) = (e[0][k][i], e[1][k][i], e[2][k][i]); + _mm256_xor_si256( + _mm256_gf2p8mul_epi8(_mm256_xor_si256(a, c), ones), + _mm256_gf2p8mul_epi8(b, c), + ) + }); + // 8x32 byte transpose: qword `d` of lane `l` of `w[j]` holds byte `16 l + 2 j + d` + // of each position, in order. + let y: [[V; 2]; 4] = std::array::from_fn(|m| { [ - _mm256_gf2p8affine_epi64_epi8(bit_transpose, lo, 0), - _mm256_gf2p8affine_epi64_epi8(bit_transpose, hi, 0), + _mm256_unpacklo_epi8(p[2 * m], p[2 * m + 1]), + _mm256_unpackhi_epi8(p[2 * m], p[2 * m + 1]), ] }); - for pair in 0..2 { - // Packs q, q + 1 take three registers: bytes 0..4 of q; 4, 5 of q and 0, 1 - // of q + 1; 2..4 and 4, 5 of q + 1. - let (c0, c1) = (columns[2 * pair], columns[2 * pair + 1]); - let r = 3 * ((qq + 2 * pair) / 2); - *staged.add(staged_index(r, a, n_active)) = c0[0]; - *staged.add(staged_index(r + 1, a, n_active)) = _mm256_permute2x128_si256(c0[1], c1[0], 0x20); - *staged.add(staged_index(r + 2, a, n_active)) = _mm256_permute2x128_si256(c1[0], c1[1], 0x21); + let z: [[V; 4]; 2] = std::array::from_fn(|s| { + std::array::from_fn(|h| { + let (u, v) = (y[2 * s][h / 2], y[2 * s + 1][h / 2]); + if h % 2 == 0 { + _mm256_unpacklo_epi16(u, v) + } else { + _mm256_unpackhi_epi16(u, v) + } + }) + }); + for h in 0..4 { + let w = [ + _mm256_unpacklo_epi32(z[0][h], z[1][h]), + _mm256_unpackhi_epi32(z[0][h], z[1][h]), + ]; + for (d, w) in w.into_iter().enumerate() { + let r = 8 * i + 2 * h + d; + *staged.add(staged_index(r, a, n_active)) = _mm256_gf2p8affine_epi64_epi8(bit_transpose, w, 0); + } } } } diff --git a/crates/experimental_prover/keccak_gkr/src/prover.rs b/crates/experimental_prover/keccak_gkr/src/prover.rs index 680b8d741f..a45659337e 100644 --- a/crates/experimental_prover/keccak_gkr/src/prover.rs +++ b/crates/experimental_prover/keccak_gkr/src/prover.rs @@ -272,9 +272,8 @@ impl Layer<'_> { a }, ); - (0..PACKS) - .map(|q| std::array::from_fn(|c| sums[48 * q + c].reduce())) - .collect() + let sums: Vec = sums.iter().map(|s| s.reduce()).collect(); + crate::products_gfni::coefficients(&sums) } /// [`Self::pack_products`] by digit buckets: the block's positions grouped by each digit of From 6425dae6cc43285caefe4fa7e08b804b40c17efc Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:22:31 +0000 Subject: [PATCH 09/18] experimental keccak_gkr: whole-position parts on AVX2 The digit buckets summed each block position's products in two parts of three AVX2 registers, so each bucket's loop ran twice per digit. A part of the whole position, six registers, runs it once. Single thread, `aggregate --xmss 100`, Zen 4 core 24, two runs each, same core back to back, Product span, ms: vp: base 115.7 / 115.8 this 110.3 / 109.7 hsw: with the next commit's PCLMULQDQ row products only 135.6 / 135.5, with this commit as well 127.7 / 127.9 --- crates/experimental_prover/keccak_gkr/src/prover.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/experimental_prover/keccak_gkr/src/prover.rs b/crates/experimental_prover/keccak_gkr/src/prover.rs index a45659337e..c89af3cc3b 100644 --- a/crates/experimental_prover/keccak_gkr/src/prover.rs +++ b/crates/experimental_prover/keccak_gkr/src/prover.rs @@ -33,12 +33,10 @@ const DIGITS: usize = 128 / DIGIT_BITS; /// The bits of a block position's products, 48 to each pack. pub(crate) const PAYLOAD_BITS: usize = 48 * PACKS; /// The packs of a part of a block position, and the words of their products: with AVX-512 the -/// whole position, three registers; with AVX2 half, three registers; elsewhere a quarter, a -/// block's worth of which stays in cache. -#[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] +/// whole position, three registers; with AVX2 the whole position, six registers; elsewhere a +/// quarter, a block's worth of which stays in cache. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] const PART: usize = PACKS; -#[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] -const PART: usize = PACKS / 2; #[cfg(not(all(target_arch = "x86_64", target_feature = "avx2")))] const PART: usize = 8; const PART_WORDS: usize = 48 * PART / 64; @@ -92,7 +90,7 @@ impl Part { } } #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] - // SAFETY: both parts are three registers, and AVX2 is enabled at compile time. + // SAFETY: both parts are six registers, and AVX2 is enabled at compile time. unsafe { use core::arch::x86_64::*; let (dst, src) = (self.0.as_mut_ptr(), other.0.as_ptr()); From ae7b9f5d529257cfb28af0dd7702a26e2409abaf Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:22:40 +0000 Subject: [PATCH 10/18] experimental keccak_gkr: PCLMULQDQ row products with ones by lookup The digit buckets' products are `ones * (a + c) + b * c`, two carry-less products a pack. `ones * a` is a linear map of the pack, so with PCLMULQDQ alone (a product a lane) a row now takes it once a position by nibble lookups, an 8x32 byte transpose to a word a pack, and the products of the row XOR two of them. That halves the carry-less products. With VPCLMULQDQ the lookups cost more than they save (row products per block, microbenchmark: 62.3 us with them, 54.4 us without), so that tier keeps the products. `kernels::row_products` hands the five products of a row to a closure, so the other arms keep their shape. Single thread, `aggregate --xmss 100`, Zen 4 core 24, two runs each, same core back to back, Product span, ms: hsw: base 148.9 / 148.9 lookups 135.6 / 135.5 with the previous commit as well 127.7 / 127.9 vp: unchanged code path --- .../keccak_gkr/src/kernels.rs | 174 +++++++++++++++++- .../keccak_gkr/src/prover.rs | 10 +- 2 files changed, 176 insertions(+), 8 deletions(-) diff --git a/crates/experimental_prover/keccak_gkr/src/kernels.rs b/crates/experimental_prover/keccak_gkr/src/kernels.rs index 8083708c1c..8525bd3679 100644 --- a/crates/experimental_prover/keccak_gkr/src/kernels.rs +++ b/crates/experimental_prover/keccak_gkr/src/kernels.rs @@ -36,7 +36,7 @@ cfg_select! { mod simd { pub use super::{ embed_extend_portable as embed_extend, embed_portable as embed, fold_groups_portable as fold_groups, - products_portable as products, skip_row_portable as skip_row, + products_portable as products, row_products_portable as row_products, skip_row_portable as skip_row, }; } } @@ -176,6 +176,22 @@ pub fn products_portable(a: &[u32; PACKS], b: &[u32; PACKS], c: &[u32; PACKS]) - std::array::from_fn(|q| (mul_wide(ones, (a[q] ^ c[q]) as u64) ^ mul_wide(b[q] as u64, c[q] as u64)) as u64) } +/// The products of a row's five positions, which `chi` mixes cyclically, each handed to `f` as +/// it is made: `x` takes the embeddings of positions `x`, `x + 1` and `x + 2`, as [`products`] +/// does. +#[inline(always)] +pub fn row_products(words: [&[u64; BLOCK_WORDS]; 5], f: impl FnMut(usize, &[u64; PACKS])) { + simd::row_products(words, f) +} + +#[inline(always)] +pub fn row_products_portable(words: [&[u64; BLOCK_WORDS]; 5], mut f: impl FnMut(usize, &[u64; PACKS])) { + let e: [[u32; PACKS]; 5] = std::array::from_fn(|x| embed(words[x])); + for x in 0..5 { + f(x, &products(&e[x], &e[(x + 1) % 5], &e[(x + 2) % 5])); + } +} + /// One row of five positions, which `chi` mixes cyclically: for every pattern, /// `acc[p] += sum over x of weights[x] * e[x + 1][p] * e[x + 2][p]`. #[inline(always)] @@ -341,7 +357,10 @@ mod x86 { use core::arch::x86_64::*; use super::*; - pub use super::{embed_extend_portable as embed_extend, products_clmul as products, skip_row_portable as skip_row}; + pub use super::{ + embed_extend_portable as embed_extend, products_clmul as products, row_products_portable as row_products, + skip_row_portable as skip_row, + }; /// The matrices sending a pack's low and high byte to each byte of its embedding. An affine /// byte map reads output bit `t` off matrix byte `7 - t`. @@ -477,6 +496,8 @@ mod avx2 { use core::arch::x86_64::*; pub use super::products_clmul as products; + #[cfg(target_feature = "vpclmulqdq")] + pub use super::row_products_portable as row_products; use super::*; use crate::clmul::{V, Vector}; @@ -640,6 +661,147 @@ mod avx2 { acc.map(V::sum) }) } + + /// Table `6 * n + m`: byte `m` of `ones` times the embedding of each value of nibble `n` of a + /// pack, in both lanes. + #[cfg(not(target_feature = "vpclmulqdq"))] + static ONES_NIBBLES: [[u8; 32]; 24] = { + let mut tables = [[0u8; 32]; 24]; + let mut i = 0; + while i < 24 { + let (n, m) = (i / 6, i % 6); + let mut v = 0; + while v < 16 { + let mut e = 0u64; + let mut t = 0; + while t < 4 { + if v >> t & 1 == 1 { + e ^= RMFE.emb[4 * n + t] as u64; + } + t += 1; + } + let mut product = 0u64; + let mut k = 0; + while k < 32 { + if RMFE.ones >> k & 1 == 1 { + product ^= e << k; + } + k += 1; + } + tables[i][v] = (product >> (8 * m)) as u8; + tables[i][16 + v] = tables[i][v]; + v += 1; + } + i += 1; + } + tables + }; + + /// Per lane of eight packs, the low bytes of packs `0, 1, 4, 5`, then of `2, 3, 6, 7`, then + /// their high bytes. + #[cfg(not(target_feature = "vpclmulqdq"))] + const SPLIT: [u8; 32] = { + const LANE: [u8; 16] = [0, 2, 8, 10, 4, 6, 12, 14, 1, 3, 9, 11, 5, 7, 13, 15]; + let mut t = [0u8; 32]; + let mut i = 0; + while i < 32 { + t[i] = LANE[i % 16]; + i += 1; + } + t + }; + + /// `ones` times each pack's embedding, a linear map of the pack: per byte of the products, the + /// sum of four nibble lookups, then an 8x32 byte transpose to a word per pack. + #[cfg(not(target_feature = "vpclmulqdq"))] + #[inline(always)] + fn ones_times(words: &[u64; BLOCK_WORDS]) -> [u64; PACKS] { + // SAFETY: the loads are of the eight words and the tables, the stores of `out`. + unsafe { + let split = _mm256_loadu_si256(SPLIT.as_ptr().cast()); + let idx = _mm256_setr_epi32(0, 4, 2, 6, 1, 5, 3, 7); + let half = |h: usize| { + let v = _mm256_shuffle_epi8(_mm256_loadu_si256(words.as_ptr().add(4 * h).cast()), split); + _mm256_permutevar8x32_epi32(v, idx) + }; + let (w0, w1) = (half(0), half(1)); + // Byte `16 l + 2 j + d` of each: pack `4 j + 2 l + d`. + let (low, high) = (_mm256_unpacklo_epi64(w0, w1), _mm256_unpackhi_epi64(w0, w1)); + let mask = _mm256_set1_epi8(0x0f); + let nibbles = [ + _mm256_and_si256(low, mask), + _mm256_and_si256(_mm256_srli_epi16(low, 4), mask), + _mm256_and_si256(high, mask), + _mm256_and_si256(_mm256_srli_epi16(high, 4), mask), + ]; + let look = |n: usize, m: usize| { + _mm256_shuffle_epi8(_mm256_loadu_si256(ONES_NIBBLES[6 * n + m].as_ptr().cast()), nibbles[n]) + }; + let zero = _mm256_setzero_si256(); + let p: [__m256i; 8] = std::array::from_fn(|m| { + if m < 6 { + _mm256_xor_si256( + _mm256_xor_si256(look(0, m), look(1, m)), + _mm256_xor_si256(look(2, m), look(3, m)), + ) + } else { + zero + } + }); + let y: [[__m256i; 2]; 4] = std::array::from_fn(|k| { + [ + _mm256_unpacklo_epi8(p[2 * k], p[2 * k + 1]), + _mm256_unpackhi_epi8(p[2 * k], p[2 * k + 1]), + ] + }); + let z: [[__m256i; 4]; 2] = std::array::from_fn(|s| { + std::array::from_fn(|h| { + let (u, v) = (y[2 * s][h / 2], y[2 * s + 1][h / 2]); + if h % 2 == 0 { + _mm256_unpacklo_epi16(u, v) + } else { + _mm256_unpackhi_epi16(u, v) + } + }) + }); + let mut out = [0u64; PACKS]; + for h in 0..4 { + // Register `2 h + d` holds packs `8 h + 4 d ..`, in order. + let w = [ + _mm256_unpacklo_epi32(z[0][h], z[1][h]), + _mm256_unpackhi_epi32(z[0][h], z[1][h]), + ]; + for (d, w) in w.into_iter().enumerate() { + _mm256_storeu_si256(out.as_mut_ptr().add(8 * h + 4 * d).cast(), w); + } + } + out + } + } + + /// [`products`] of a row, `ones * (a + c)` from [`ones_times`] of each position: with + /// PCLMULQDQ alone, a product a lane, that halves the carry-less products. With VPCLMULQDQ + /// the products are cheaper than the lookups. + #[cfg(not(target_feature = "vpclmulqdq"))] + #[inline(always)] + pub fn row_products(words: [&[u64; BLOCK_WORDS]; 5], mut f: impl FnMut(usize, &[u64; PACKS])) { + let e: [[u32; PACKS]; 5] = std::array::from_fn(|x| embed(words[x])); + let t: [[u64; PACKS]; 5] = std::array::from_fn(|x| ones_times(words[x])); + for x in 0..5 { + let (b, c, ta, tc) = (&e[(x + 1) % 5], &e[(x + 2) % 5], &t[x], &t[(x + 2) % 5]); + let mut out = [0u64; PACKS]; + for q in (0..PACKS).step_by(2 * V::N) { + // SAFETY: `2 * N` packs and words at `q`. + unsafe { + let (zb, zc) = (V::load_packs(b.as_ptr().add(q)), V::load_packs(c.as_ptr().add(q))); + let linear = V::load(ta.as_ptr().add(q)).xor(V::load(tc.as_ptr().add(q))); + let product = zb.clmul::<0x00>(zc).unpacklo(zb.clmul::<0x11>(zc)); + V::store(out.as_mut_ptr().add(q), product.xor(linear)); + } + } + f(x, &out); + } + } } /// Compiled only where its feature is, so its kernels are safe to call. @@ -652,6 +814,7 @@ mod neon { pmull, pmull_hi as mul_hi_scheduled, pmull_hi_asm as mul_hi, pmull_lo_asm as mul_lo, }; + pub use super::row_products_portable as row_products; use super::*; /// A linear map is the sum of its nibbles' images, so each byte of sixteen embeddings is four @@ -876,6 +1039,13 @@ mod tests { embed(&std::array::from_fn(|_| rng.next_u64())), ); assert_eq!(products(&e, &b, &c), products_portable(&e, &b, &c)); + let row: [[u64; BLOCK_WORDS]; 5] = std::array::from_fn(|_| std::array::from_fn(|_| rng.next_u64())); + let er: [[u32; PACKS]; 5] = std::array::from_fn(|x| embed_portable(&row[x])); + let expected: [[u64; PACKS]; 5] = + std::array::from_fn(|x| products_portable(&er[x], &er[(x + 1) % 5], &er[(x + 2) % 5])); + let mut got = [[0u64; PACKS]; 5]; + row_products(row.each_ref(), |x, p| got[x] = *p); + assert_eq!(got, expected); let (mut xe, mut xp) = ([0u32; SLOTS], [0u32; SLOTS]); embed_extend(&words, &mut xe); diff --git a/crates/experimental_prover/keccak_gkr/src/prover.rs b/crates/experimental_prover/keccak_gkr/src/prover.rs index c89af3cc3b..9c1413781b 100644 --- a/crates/experimental_prover/keccak_gkr/src/prover.rs +++ b/crates/experimental_prover/keccak_gkr/src/prover.rs @@ -16,7 +16,7 @@ use primitives::transcript::ProverState; use verifier::keccak::{BITS, LOCAL_BITS, along, linear_transpose}; use crate::basis::{PolyEval, add_mul}; -use crate::kernels::{GROUP, MIXED, PACKS, SKIP, SkipBlock, TERNARY, embed, mixed_points, products, slot}; +use crate::kernels::{GROUP, MIXED, PACKS, SKIP, SkipBlock, TERNARY, mixed_points, row_products, slot}; use crate::power::{Element, PowerField, get, set}; use crate::protocol::{Claim, chi_weights, mu_at}; use crate::witness::{Shape, Witness, transpose64, words}; @@ -298,13 +298,11 @@ impl Layer<'_> { for r in 0..5 { for z in 0..64 { let js = row_positions(r, z); - let e: [[u32; PACKS]; 5] = std::array::from_fn(|x| embed(words(block, js[x]))); - for x in 0..5 { - let p = products(&e[x], &e[(x + 1) % 5], &e[(x + 2) % 5]); + row_products(js.map(|j| words(block, j)), |x, p| { for (q, part) in payloads.iter_mut().enumerate() { part[js[x]] = Part::pack(p[PART * q..][..PART].try_into().unwrap()); } - } + }); } } for (payloads, rows) in payloads.iter().zip(rows.iter_mut()) { @@ -428,7 +426,7 @@ impl Layer<'_> { _ => { // Both groups' unreduced sums, three words each and laid out by word, then reduced. for j in 0..BITS { - let sums = crate::kernels::fold_groups(&embed(words(block, j)), &fold); + let sums = crate::kernels::fold_groups(&crate::kernels::embed(words(block, j)), &fold); for (wide, sum) in wide.as_chunks_mut::<{ 3 * BITS }>().0[..halves].iter_mut().zip(&sums) { let sum: [u64; 3] = crate::basis::carry(sum); for (k, &word) in sum.iter().enumerate() { From ff71b8aa12eccf1a9fd0ee63ebab1ed11f0c4e06 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:37:22 +0000 Subject: [PATCH 11/18] experimental keccak_gkr: pack AVX2 row products in registers The digit buckets' payloads hold a position's products 48 bits each. On AVX2 every product went through memory as a word, then four at a time through scalar shifts into three words, then into the payload. Now a lane's two products come out of the carry-less products already packed (the second shifted up six bytes), and each lane's twelve bytes are stored straight into the payload, twelve bytes apart, the last lane's without the four bytes past it (`kernels::row_packed`). With PCLMULQDQ alone the nibble lookups of `ones * a` produce that packed layout too. A part is a whole position on x86, so nothing else is copied; the other arms keep products and `Part::pack`. Single thread, `aggregate --xmss 100`, Zen 4 core 24, same core back to back, Product span, ms: vp: before 109.9 / 110.0 after 101.8 / 101.5 hsw: before 129.5 / 128.4 / 135.4 after 119.7 / 119.3 / 119.5 pack_products microbench (n = 4096) per block: vp 122.7 -> 114.2 us, hsw 143.3 -> 130.5 us; the row kernel alone per block: vp 49.7 -> 40.5 us, hsw 82.0 -> 63.8 us. --- .../keccak_gkr/src/kernels.rs | 159 +++++++++++++++--- .../keccak_gkr/src/prover.rs | 26 ++- 2 files changed, 157 insertions(+), 28 deletions(-) diff --git a/crates/experimental_prover/keccak_gkr/src/kernels.rs b/crates/experimental_prover/keccak_gkr/src/kernels.rs index 8525bd3679..820fe64607 100644 --- a/crates/experimental_prover/keccak_gkr/src/kernels.rs +++ b/crates/experimental_prover/keccak_gkr/src/kernels.rs @@ -192,6 +192,30 @@ pub fn row_products_portable(words: [&[u64; BLOCK_WORDS]; 5], mut f: impl FnMut( } } +/// The words of a block position's products packed 48 bits each. +pub const PACKED_WORDS: usize = 3 * PACKS / 4; + +/// Products packed 48 bits each, product `q` at bits `48 q ..` of `out`: four to three words. +#[inline(always)] +pub fn pack_products(products: &[u64], out: &mut [u64]) { + for (out, p) in out.as_chunks_mut::<3>().0.iter_mut().zip(products.as_chunks::<4>().0) { + *out = [p[0] | p[1] << 48, p[1] >> 16 | p[2] << 32, p[2] >> 32 | p[3] << 16]; + } +} + +/// [`row_products`], each packed ([`pack_products`]) into `out[x]`. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq"))] +#[inline(always)] +pub fn row_packed(words: [&[u64; BLOCK_WORDS]; 5], out: [&mut [u64; PACKED_WORDS]; 5]) { + simd::row_packed(words, out) +} + +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq"))] +#[inline(always)] +pub fn row_packed_portable(words: [&[u64; BLOCK_WORDS]; 5], mut out: [&mut [u64; PACKED_WORDS]; 5]) { + row_products(words, |x, p| pack_products(p, &mut out[x][..])); +} + /// One row of five positions, which `chi` mixes cyclically: for every pattern, /// `acc[p] += sum over x of weights[x] * e[x + 1][p] * e[x + 2][p]`. #[inline(always)] @@ -358,8 +382,8 @@ mod x86 { use super::*; pub use super::{ - embed_extend_portable as embed_extend, products_clmul as products, row_products_portable as row_products, - skip_row_portable as skip_row, + embed_extend_portable as embed_extend, products_clmul as products, row_packed_portable as row_packed, + row_products_portable as row_products, skip_row_portable as skip_row, }; /// The matrices sending a pack's low and high byte to each byte of its embedding. An affine @@ -495,10 +519,8 @@ mod x86 { mod avx2 { use core::arch::x86_64::*; - pub use super::products_clmul as products; - #[cfg(target_feature = "vpclmulqdq")] - pub use super::row_products_portable as row_products; use super::*; + pub use super::{products_clmul as products, row_products_portable as row_products}; use crate::clmul::{V, Vector}; /// The 32 packs of a block position, embedded, four to a register. @@ -712,7 +734,8 @@ mod avx2 { }; /// `ones` times each pack's embedding, a linear map of the pack: per byte of the products, the - /// sum of four nibble lookups, then an 8x32 byte transpose to a word per pack. + /// sum of four nibble lookups, then an 8x32 byte transpose to a word per pack and a shuffle to + /// the packed layout of [`Chunks`], two packs to a lane. #[cfg(not(target_feature = "vpclmulqdq"))] #[inline(always)] fn ones_times(words: &[u64; BLOCK_WORDS]) -> [u64; PACKS] { @@ -764,6 +787,11 @@ mod avx2 { } }) }); + // A lane's second word's six bytes beside its first word's. + let pack = _mm256_setr_epi8( + 0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, -1, -1, 0, 1, 2, 3, 4, 5, 8, 9, 10, 11, 12, 13, -1, -1, + -1, -1, + ); let mut out = [0u64; PACKS]; for h in 0..4 { // Register `2 h + d` holds packs `8 h + 4 d ..`, in order. @@ -772,34 +800,110 @@ mod avx2 { _mm256_unpackhi_epi32(z[0][h], z[1][h]), ]; for (d, w) in w.into_iter().enumerate() { - _mm256_storeu_si256(out.as_mut_ptr().add(8 * h + 4 * d).cast(), w); + _mm256_storeu_si256(out.as_mut_ptr().add(8 * h + 4 * d).cast(), _mm256_shuffle_epi8(w, pack)); } } out } } - /// [`products`] of a row, `ones * (a + c)` from [`ones_times`] of each position: with - /// PCLMULQDQ alone, a product a lane, that halves the carry-less products. With VPCLMULQDQ - /// the products are cheaper than the lookups. - #[cfg(not(target_feature = "vpclmulqdq"))] + /// Lane by lane, two packs' products packed into the lane's low twelve bytes, as + /// [`pack_products`] lays them out. + trait Chunks: Vector { + /// The products in each lane's low word of `low` and of `high`, the second 48 bits up. + fn chunks(low: Self, high: Self) -> Self; + /// Each lane's twelve bytes at `p + 12 l`. A lane's store runs four bytes past them, but + /// for the last lane of a `last` register. + /// + /// # Safety + /// `p` is valid for writing `12 * N` bytes, and four more unless `last`. + unsafe fn store_chunks(self, p: *mut u8, last: bool); + } + + /// Twelve bytes of a lane at `p`. + /// + /// # Safety + /// `p` is valid for writing twelve bytes. #[inline(always)] - pub fn row_products(words: [&[u64; BLOCK_WORDS]; 5], mut f: impl FnMut(usize, &[u64; PACKS])) { + unsafe fn store_twelve(p: *mut u8, v: __m128i) { + unsafe { + _mm_storel_epi64(p.cast(), v); + _mm_storeu_si32(p.add(8).cast(), _mm_srli_si128::<8>(v)); + } + } + + impl Chunks for __m128i { + #[inline(always)] + fn chunks(low: Self, high: Self) -> Self { + // SAFETY: SSE2. + unsafe { _mm_xor_si128(low, _mm_slli_si128::<6>(high)) } + } + #[inline(always)] + unsafe fn store_chunks(self, p: *mut u8, last: bool) { + unsafe { + if last { + store_twelve(p, self); + } else { + _mm_storeu_si128(p.cast(), self); + } + } + } + } + + #[cfg(target_feature = "vpclmulqdq")] + impl Chunks for __m256i { + #[inline(always)] + fn chunks(low: Self, high: Self) -> Self { + // SAFETY: AVX2. + unsafe { _mm256_xor_si256(low, _mm256_bslli_epi128::<6>(high)) } + } + #[inline(always)] + unsafe fn store_chunks(self, p: *mut u8, last: bool) { + unsafe { + _mm_storeu_si128(p.cast(), _mm256_castsi256_si128(self)); + let high = _mm256_extracti128_si256::<1>(self); + if last { + store_twelve(p.add(12), high); + } else { + _mm_storeu_si128(p.add(12).cast(), high); + } + } + } + } + + /// [`row_products`] packed into `out` as [`pack_products`] lays them out, a lane's two + /// products packed in registers and stored twelve bytes apart. With PCLMULQDQ alone, a + /// product a lane, `ones * (a + c)` comes from [`ones_times`] of each position, which halves + /// the carry-less products; with VPCLMULQDQ the products are cheaper than the lookups. + #[inline(always)] + pub fn row_packed(words: [&[u64; BLOCK_WORDS]; 5], out: [&mut [u64; PACKED_WORDS]; 5]) { let e: [[u32; PACKS]; 5] = std::array::from_fn(|x| embed(words[x])); + #[cfg(not(target_feature = "vpclmulqdq"))] let t: [[u64; PACKS]; 5] = std::array::from_fn(|x| ones_times(words[x])); - for x in 0..5 { - let (b, c, ta, tc) = (&e[(x + 1) % 5], &e[(x + 2) % 5], &t[x], &t[(x + 2) % 5]); - let mut out = [0u64; PACKS]; + #[cfg(target_feature = "vpclmulqdq")] + let ones = V::splat(RMFE.ones as u64); + for (x, out) in out.into_iter().enumerate() { + let (b, c) = (&e[(x + 1) % 5], &e[(x + 2) % 5]); + let dst = out.as_mut_ptr().cast::(); for q in (0..PACKS).step_by(2 * V::N) { - // SAFETY: `2 * N` packs and words at `q`. + // SAFETY: `2 * N` packs and words at `q`; the stores are of packs `q ..` (six + // bytes each), and four bytes past them but at the last packs. unsafe { let (zb, zc) = (V::load_packs(b.as_ptr().add(q)), V::load_packs(c.as_ptr().add(q))); - let linear = V::load(ta.as_ptr().add(q)).xor(V::load(tc.as_ptr().add(q))); - let product = zb.clmul::<0x00>(zc).unpacklo(zb.clmul::<0x11>(zc)); - V::store(out.as_mut_ptr().add(q), product.xor(linear)); + let (low, high) = (zb.clmul::<0x00>(zc), zb.clmul::<0x11>(zc)); + let chunks = cfg_select! { + target_feature = "vpclmulqdq" => {{ + let linear = V::load_packs(e[x].as_ptr().add(q)).xor(zc); + V::chunks(low.xor(linear.clmul::<0x00>(ones)), high.xor(linear.clmul::<0x11>(ones))) + }} + _ => {{ + let linear = V::load(t[x].as_ptr().add(q)).xor(V::load(t[(x + 2) % 5].as_ptr().add(q))); + V::chunks(low, high).xor(linear) + }} + }; + chunks.store_chunks(dst.add(6 * q), q + 2 * V::N == PACKS); } } - f(x, &out); } } } @@ -1046,6 +1150,21 @@ mod tests { let mut got = [[0u64; PACKS]; 5]; row_products(row.each_ref(), |x, p| got[x] = *p); assert_eq!(got, expected); + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq"))] + { + // A guard word after each row: the packed stores run past a product but not past the row. + let mut packed = [[u64::MAX; PACKED_WORDS + 1]; 5]; + row_packed( + row.each_ref(), + packed.each_mut().map(|p| (&mut p[..PACKED_WORDS]).try_into().unwrap()), + ); + for (packed, expected) in packed.iter().zip(&expected) { + let mut want = [0u64; PACKED_WORDS + 1]; + pack_products(expected, &mut want); + want[PACKED_WORDS] = u64::MAX; + assert_eq!(*packed, want); + } + } let (mut xe, mut xp) = ([0u32; SLOTS], [0u32; SLOTS]); embed_extend(&words, &mut xe); diff --git a/crates/experimental_prover/keccak_gkr/src/prover.rs b/crates/experimental_prover/keccak_gkr/src/prover.rs index 9c1413781b..941c7767f6 100644 --- a/crates/experimental_prover/keccak_gkr/src/prover.rs +++ b/crates/experimental_prover/keccak_gkr/src/prover.rs @@ -16,7 +16,7 @@ use primitives::transcript::ProverState; use verifier::keccak::{BITS, LOCAL_BITS, along, linear_transpose}; use crate::basis::{PolyEval, add_mul}; -use crate::kernels::{GROUP, MIXED, PACKS, SKIP, SkipBlock, TERNARY, mixed_points, row_products, slot}; +use crate::kernels::{GROUP, MIXED, PACKS, SKIP, SkipBlock, TERNARY, mixed_points, slot}; use crate::power::{Element, PowerField, get, set}; use crate::protocol::{Claim, chi_weights, mu_at}; use crate::witness::{Shape, Witness, transpose64, words}; @@ -62,12 +62,11 @@ impl Part { const ZERO: Self = Self([0; PART_WORDS]); /// Product `q` at bits `48 * q ..`: four products to three words. + #[cfg(not(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq")))] #[inline(always)] fn pack(products: &[u64; PART]) -> Self { let mut out = [0u64; PART_WORDS]; - for (out, p) in out.as_chunks_mut::<3>().0.iter_mut().zip(products.as_chunks::<4>().0) { - *out = [p[0] | p[1] << 48, p[1] >> 16 | p[2] << 32, p[2] >> 32 | p[3] << 16]; - } + crate::kernels::pack_products(products, &mut out); Self(out) } @@ -298,11 +297,22 @@ impl Layer<'_> { for r in 0..5 { for z in 0..64 { let js = row_positions(r, z); - row_products(js.map(|j| words(block, j)), |x, p| { - for (q, part) in payloads.iter_mut().enumerate() { - part[js[x]] = Part::pack(p[PART * q..][..PART].try_into().unwrap()); + let row = js.map(|j| words(block, j)); + cfg_select! { + all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq") => { + // A part is a whole position, so the products pack straight into it. + // SAFETY: a row's five positions are distinct and below `BITS`. + let parts = unsafe { payloads[0].get_disjoint_unchecked_mut(js) }; + crate::kernels::row_packed(row, parts.map(|p| &mut p.0)); + } + _ => { + crate::kernels::row_products(row, |x, p| { + for (q, part) in payloads.iter_mut().enumerate() { + part[js[x]] = Part::pack(p[PART * q..][..PART].try_into().unwrap()); + } + }); } - }); + } } } for (payloads, rows) in payloads.iter().zip(rows.iter_mut()) { From 658fd2049988368f45d9d005a9f559ca797d19e6 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:43:51 +0000 Subject: [PATCH 12/18] experimental keccak_gkr: fold digit bucket sums by nibbles as they are made Each digit's 256 bucket sums were stored, then halved eight times into the digit's rows, which with whole-position parts on AVX2 spills past L1. Now each bucket's sum, made in registers, adds into the sum of its value's high nibble (kept in registers across the 16 values that share it) and into that of its low nibble (16 parts), and the two sets of 16 make the rows of the digit's high and low bits. Shared by every arm that uses the digit buckets. Single thread, `aggregate --xmss 100`, Zen 4 core 24, two runs each, same core back to back, Product span, ms: vp: before 101.2 / 103.1 after 93.2 / 93.5 hsw: before 119.3 / 119.5 after 108.8 / 109.1 pack_products microbench (n = 4096) per block: vp 113.8 -> 104.8 us, hsw 131.4 / 134.2 -> 121.9 / 121.4 us. --- .../keccak_gkr/src/prover.rs | 42 ++++++++++++------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/crates/experimental_prover/keccak_gkr/src/prover.rs b/crates/experimental_prover/keccak_gkr/src/prover.rs index 941c7767f6..435c774650 100644 --- a/crates/experimental_prover/keccak_gkr/src/prover.rs +++ b/crates/experimental_prover/keccak_gkr/src/prover.rs @@ -152,7 +152,9 @@ impl Groups { /// `sum over positions of weight[j] (x) payloads[j]` into `rows`: row `DIGIT_BITS * d + i` /// sums the payloads whose weight has that bit set. Each group is summed in registers, so - /// the payloads are only read. + /// the payloads are only read, then added into the sums of its value's low nibble and of its + /// high one (`sums[..16]`, `sums[16..32]`), the latter kept in registers; those make the + /// rows of the digit's low bits and of its high bits. fn add_rows(&self, payloads: &[Part; BITS], rows: &mut [Part; 128], sums: &mut [Part; 1 << DIGIT_BITS]) { for (rows, (order, starts)) in rows .as_chunks_mut::() @@ -160,23 +162,33 @@ impl Groups { .iter_mut() .zip(self.order.iter().zip(&self.starts)) { - for (v, sum) in sums.iter_mut().enumerate() { - let mut acc = Part::ZERO; - for &p in &order[starts[v] as usize..starts[v + 1] as usize] { - // SAFETY: `order` holds positions. - acc.add(unsafe { payloads.get_unchecked(p as usize) }); + let (low, high) = sums[..32].split_at_mut(16); + low.fill(Part::ZERO); + for (h, high) in high.iter_mut().enumerate() { + let mut nibble = Part::ZERO; + for (l, low) in low.iter_mut().enumerate() { + let v = 16 * h + l; + let mut acc = Part::ZERO; + for &p in &order[starts[v] as usize..starts[v + 1] as usize] { + // SAFETY: `order` holds positions. + acc.add(unsafe { payloads.get_unchecked(p as usize) }); + } + nibble.add(&acc); + low.add(&acc); } - *sum = acc; + *high = nibble; } - // Bit `i` of the digit: the sums whose value has it, then fold it away. - for (i, row) in rows.iter_mut().enumerate().rev() { - let (low, high) = sums[..2 << i].split_at_mut(1 << i); - let mut acc = Part::ZERO; - for (low, high) in low.iter_mut().zip(&*high) { - acc.add(high); - low.add(high); + for (sums, rows) in [low, high].into_iter().zip(rows.as_chunks_mut::<4>().0) { + // Bit `i` of the nibble: the sums whose value has it, then fold it away. + for (i, row) in rows.iter_mut().enumerate().rev() { + let (low, high) = sums[..2 << i].split_at_mut(1 << i); + let mut acc = Part::ZERO; + for (low, high) in low.iter_mut().zip(&*high) { + acc.add(high); + low.add(high); + } + *row = acc; } - *row = acc; } } } From 8a3fe5188356766f22cdaafa8a4571d537043dd5 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:04:20 +0000 Subject: [PATCH 13/18] experimental pcs: fold level 0's lanes two coefficients a register on AVX2 With VPCLMULQDQ and no AVX-512, the level-0 rounds wrote each folded lane (`P = G + alpha F + beta X F`) and multiplied it by `X` through the plane form of `F128x8`, which transposes eight elements to two coefficient planes on load and back on store. `primitives::field::x4` gains `F128x2` and `F64x2`: two elements a register in memory order, with Karatsuba (`Prod2`) and mixed (`Mixed2`) products that reduce straight back to that order. `Folded::block` and `mul_x_block` run on them, and an `E` block's `X` product multiplies only its even columns, the odd columns' constant being one; a `K` block's gathers its four even sums into one register. Single thread, `aggregate --xmss 100`, last pass, same core, back to back, ms: vp: base / new PCS open 237 238 / 230 Fold round L0 / Project 59.3 59.8 / 51.9 gfni: PCS open 233 / 222 224 Fold round L0 / Project 52.8 / 44.2 44.2 hsw and AVX-512 compile the unchanged arms. --- .../experimental_prover/pcs/src/whir_lanes.rs | 76 ++++- crates/experimental_prover/pcs/src/xmul.rs | 201 ++++++++++++- .../primitives/src/field/x4.rs | 271 ++++++++++++++++++ 3 files changed, 536 insertions(+), 12 deletions(-) diff --git a/crates/experimental_prover/pcs/src/whir_lanes.rs b/crates/experimental_prover/pcs/src/whir_lanes.rs index 535eb88e54..9507d46f97 100644 --- a/crates/experimental_prover/pcs/src/whir_lanes.rs +++ b/crates/experimental_prover/pcs/src/whir_lanes.rs @@ -23,13 +23,13 @@ //! Where the stack's blocks under a lane `X F` repeat, its blocks repeat past their first few words (one more per //! round, from multiplying by `X`), so a block of `X F` is a repeat when those words are too. -use primitives::field::{F64, F64x8, F128, F128x8, mul_base_splat8}; +use primitives::field::{F64, F128}; use primitives::multilinear::interp; use primitives::stream::Stream; use crate::stack_open::{SWEEP, Terms}; use crate::whir::{FoldChallenges, FoldTable, InitialBasis}; -use crate::xmul::{exponents8, mul_x_block, multiples, sums8, x_at}; +use crate::xmul::{mul_x_block, multiples, x_at}; /// The words of a block of a round-`r` lane `X F` that can differ from the block before when the stack's blocks under /// it repeat: its first `r`. Later rounds skip no blocks. @@ -112,7 +112,16 @@ impl Folded<'_> { /// /// `X` multiplies column `c` by `A_t` after summing two coefficients, so a multiple of `X` enters as the sums, the /// `A_t` folded into its challenge. + #[cfg(not(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "vpclmulqdq", + not(target_feature = "avx512f") + )))] fn block(&self, i: usize, first: usize, out: &mut [F128]) { + use crate::xmul::{exponents8, sums8}; + use primitives::field::{F64x8, F128x8, mul_base_splat8}; + let n = out.len(); let alpha = self.ch.alpha; let out = out.as_chunks_mut::<8>().0; @@ -146,6 +155,69 @@ impl Folded<'_> { } } } + + /// [`Self::block`] two coefficients a register, in the order they lie in memory. + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "vpclmulqdq", + not(target_feature = "avx512f") + ))] + fn block(&self, i: usize, first: usize, out: &mut [F128]) { + use crate::xmul::head8; + use crate::xmul::lanes::{load8, load8_base, store8, sums}; + use primitives::field::x4::{F64x2, F128x2}; + + let n = out.len(); + let cx = &self.cx; + // `beta A_t` of the columns of an aligned eight, two to a register, all but the first column's. + let (c23, c45) = (F128x2::new(cx[1], cx[0]), F128x2::new(cx[2], cx[0])); + let out = out.as_chunks_mut::<8>().0; + match self.pairs { + Pairs::Stack => { + let f = self.stack_lane(2 * i).expect("a folded lane has its first"); + let g = self + .stack_lane(2 * i + 1) + .map(|g| g[first..first + n].as_chunks::<8>().0); + let (fb, alpha) = (&f[first..first + n], F128x2::splat(self.ch.alpha)); + for (q, (b, o)) in fb.as_chunks::<8>().0.iter().zip(out).enumerate() { + let (s0, t0) = head8(fb, q, first, outside(f)); + let fv = load8_base(b); + let s = sums(fv, F64x2::new(s0, F64::ZERO)); + let c = [F128x2::new(cx[t0], cx[0]), c23, c45, c23]; + let mut v: [F128x2; 4] = std::array::from_fn(|j| { + let mut p = alpha.mul_base_unreduced(fv[j]); + p ^= c[j].mul_base_unreduced(s[j]); + p.reduce() + }); + if let Some(g) = g { + let gv = load8_base(&g[q]); + v = std::array::from_fn(|j| v[j] + F128x2::from(gv[j])); + } + store8(v, o); + } + } + Pairs::Stored { f, g, len } => { + let (f, g) = (&f[i * len..(i + 1) * len], &g[i * len..(i + 1) * len]); + let (fb, gb) = (&f[first..first + n], g[first..first + n].as_chunks::<8>().0); + let alpha = F128x2::splat(self.ch.alpha).factor(); + let (c23, c45) = (c23.factor(), c45.factor()); + for (q, (b, o)) in fb.as_chunks::<8>().0.iter().zip(out).enumerate() { + let (s0, t0) = head8(fb, q, first, outside(f)); + let fv = load8(b); + let s = sums(fv, F128x2::new(s0, F128::ZERO)); + let c = [F128x2::new(cx[t0], cx[0]).factor(), c23, c45, c23]; + let gv = load8(&gb[q]); + let v = std::array::from_fn(|j| { + let mut p = fv[j].mul_unreduced(alpha); + p ^= s[j].mul_unreduced(c[j]); + gv[j] + p.reduce() + }); + store8(v, o); + } + } + } + } } /// What a round's table leaves for its fold: the projections of `X F` per pair, and the pairs. diff --git a/crates/experimental_prover/pcs/src/xmul.rs b/crates/experimental_prover/pcs/src/xmul.rs index ac14a4929a..0889405cac 100644 --- a/crates/experimental_prover/pcs/src/xmul.rs +++ b/crates/experimental_prover/pcs/src/xmul.rs @@ -20,6 +20,15 @@ pub(crate) trait Coef: Copy + Send + Sync + std::ops::Add { fn scale(self, k: F64) -> Self; /// `v[j] * k[j]`, where `k[j]` is one at every odd `j`. fn scale8(v: [Self; 8], k: [F64; 8]) -> [Self; 8]; + + /// [`mul_x_block`]. + #[inline(always)] + fn x_block(block: &[Self], first: usize, outside: impl Fn(usize) -> Self, out: &mut [Self]) { + for (q, out) in out.as_chunks_mut::<8>().0.iter_mut().enumerate() { + let (s, t0) = sums8(block, q, first, &outside); + *out = Self::scale8(s, multipliers8(exponents8(t0))); + } + } } impl Coef for F64 { @@ -32,6 +41,16 @@ impl Coef for F64 { fn scale8(v: [Self; 8], k: [F64; 8]) -> [Self; 8] { std::array::from_fn(|j| if j % 2 == 0 { v[j] * k[j] } else { v[j] }) } + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "vpclmulqdq", + not(target_feature = "avx512f") + ))] + #[inline(always)] + fn x_block(block: &[Self], first: usize, outside: impl Fn(usize) -> Self, out: &mut [Self]) { + lanes::x_block_k(block, first, outside, out); + } } impl Coef for F128 { @@ -47,6 +66,16 @@ impl Coef for F128 { .reduce_lanes() .to_array() } + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "vpclmulqdq", + not(target_feature = "avx512f") + ))] + #[inline(always)] + fn x_block(block: &[Self], first: usize, outside: impl Fn(usize) -> Self, out: &mut [Self]) { + lanes::x_block_e(block, first, outside, out); + } } /// `A_t` for every `t < 64`. @@ -110,13 +139,10 @@ pub(crate) fn multiples(c: F128) -> [F128; 64] { constants().map(|a| c.mul_base(a)) } -/// The sums `s` of columns `8q..8q + 8` of a block of a lane `h`, `(X h)[c] = A_t s[c]` with `t` from [`exponents8`], -/// and `t0`. `block[i] = h[first + i]`, and `outside(c)` gives the coefficients outside the block that the first column -/// reads; the block's length is a power of two that divides `first`. +/// `s[0]` of [`sums8`], and `t0`. #[inline(always)] -pub(crate) fn sums8(block: &[T], q: usize, first: usize, outside: impl Fn(usize) -> T) -> ([T; 8], usize) { - let b = &block[8 * q..8 * q + 8]; - let (s0, t0) = match (q, first) { +pub(crate) fn head8(block: &[T], q: usize, first: usize, outside: impl Fn(usize) -> T) -> (T, usize) { + match (q, first) { (0, 0) => (T::ZERO, 0), (0, _) => { let t = first.trailing_zeros() as usize; @@ -130,7 +156,16 @@ pub(crate) fn sums8(block: &[T], q: usize, first: usize, outside: impl let t = 3 + q.trailing_zeros() as usize; (block[8 * q - 1] + block[8 * q + (1 << t) - 1], t) } - }; + } +} + +/// The sums `s` of columns `8q..8q + 8` of a block of a lane `h`, `(X h)[c] = A_t s[c]` with `t` from [`exponents8`], +/// and `t0`. `block[i] = h[first + i]`, and `outside(c)` gives the coefficients outside the block that the first column +/// reads; the block's length is a power of two that divides `first`. +#[inline(always)] +pub(crate) fn sums8(block: &[T], q: usize, first: usize, outside: impl Fn(usize) -> T) -> ([T; 8], usize) { + let b = &block[8 * q..8 * q + 8]; + let (s0, t0) = head8(block, q, first, outside); let s = [ s0, b[0] + b[1], @@ -149,9 +184,142 @@ pub(crate) fn sums8(block: &[T], q: usize, first: usize, outside: impl /// 8, that divides `first`. pub(crate) fn mul_x_block(block: &[T], first: usize, outside: impl Fn(usize) -> T, out: &mut [T]) { debug_assert!(block.len() >= 8 && block.len().is_power_of_two() && first.is_multiple_of(block.len())); - for (q, out) in out.as_chunks_mut::<8>().0.iter_mut().enumerate() { - let (s, t0) = sums8(block, q, first, &outside); - *out = T::scale8(s, multipliers8(exponents8(t0))); + debug_assert_eq!(block.len(), out.len()); + T::x_block(block, first, outside, out); +} + +/// [`sums8`] and [`mul_x_block`] on two coefficients a register, in the order they lie in memory, where VPCLMULQDQ +/// has no AVX-512 beside it. +#[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "vpclmulqdq", + not(target_feature = "avx512f") +))] +pub(crate) mod lanes { + use core::arch::x86_64::*; + + use primitives::field::gf2_64x2::x86_64::reduce_halves256; + use primitives::field::x4::{F64x2, F64x4, F128x2}; + use primitives::field::{F64, F128}; + + use super::{constants, head8}; + + /// A register of two coefficients. + pub(crate) trait Lane2: Copy + std::ops::Add { + /// The 128-bit lanes `vperm2i128` picks with immediate `I`. + fn select(a: Self, b: Self) -> Self; + } + + impl Lane2 for F128x2 { + #[inline(always)] + fn select(a: Self, b: Self) -> Self { + F128x2::select::(a, b) + } + } + + impl Lane2 for F64x2 { + #[inline(always)] + fn select(a: Self, b: Self) -> Self { + F64x2::select::(a, b) + } + } + + /// [`super::sums8`] on the registers `[b0 b1, b2 b3, b4 b5, b6 b7]`, with `s0` the low lane of its register. + #[inline(always)] + pub(crate) fn sums([f01, f23, f45, f67]: [L; 4], s0: L) -> [L; 4] { + let pair = f01 + L::select::<0x01>(f01, f01); + [ + L::select::<0x30>(s0, pair), + L::select::<0x21>(f01, f23) + L::select::<0x11>(f23, f23), + L::select::<0x21>(f23, f45) + L::select::<0x31>(f67, f45), + L::select::<0x21>(f45, f67) + L::select::<0x11>(f67, f67), + ] + } + + /// Eight `E` coefficients as four registers. + #[inline(always)] + pub(crate) fn load8(e: &[F128; 8]) -> [F128x2; 4] { + let p = e.as_chunks::<2>().0; + [ + F128x2::load(&p[0]), + F128x2::load(&p[1]), + F128x2::load(&p[2]), + F128x2::load(&p[3]), + ] + } + + #[inline(always)] + pub(crate) fn store8([a, b, c, d]: [F128x2; 4], out: &mut [F128; 8]) { + let p = out.as_chunks_mut::<2>().0; + a.store(&mut p[0]); + b.store(&mut p[1]); + c.store(&mut p[2]); + d.store(&mut p[3]); + } + + /// Eight `K` coefficients as four registers. + #[inline(always)] + pub(crate) fn load8_base(k: &[F64; 8]) -> [F64x2; 4] { + let q = k.as_chunks::<4>().0; + let ([a, b], [c, d]) = (F64x4::load(&q[0]).spread(), F64x4::load(&q[1]).spread()); + [a, b, c, d] + } + + /// [`super::mul_x_block`] on `E` coefficients: the odd columns' constant is one, so only the even ones multiply. + #[inline(always)] + pub(super) fn x_block_e(block: &[F128], first: usize, outside: impl Fn(usize) -> F128, out: &mut [F128]) { + let a = constants(); + let k46 = F64x2::new(a[2], a[1]); + let chunks = block.as_chunks::<8>().0.iter().zip(out.as_chunks_mut::<8>().0); + for (q, (b, o)) in chunks.enumerate() { + let (s0, t0) = head8(block, q, first, &outside); + let s = sums(load8(b), F128x2::new(s0, F128::ZERO)); + let r02 = F128x2::select::<0x20>(s[0], s[1]) + .mul_base_unreduced(F64x2::new(a[t0], a[1])) + .reduce(); + let r46 = F128x2::select::<0x20>(s[2], s[3]).mul_base_unreduced(k46).reduce(); + let out = [ + F128x2::select::<0x30>(r02, s[0]), + F128x2::select::<0x31>(r02, s[1]), + F128x2::select::<0x30>(r46, s[2]), + F128x2::select::<0x31>(r46, s[3]), + ]; + store8(out, o); + } + } + + /// [`super::mul_x_block`] on `K` coefficients: the even columns' sums in one register, multiplied at once. + #[inline(always)] + pub(super) fn x_block_k(block: &[F64], first: usize, outside: impl Fn(usize) -> F64, out: &mut [F64]) { + let a = constants(); + let chunks = block.as_chunks::<8>().0.iter().zip(out.as_chunks_mut::<8>().0); + for (q, (b, o)) in chunks.enumerate() { + let (s0, t0) = head8(block, q, first, &outside); + // SAFETY: the module's cfg enables avx2 and vpclmulqdq; each access is four whole qwords of a chunk of eight. + unsafe { + let (lo, hi) = ( + _mm256_loadu_si256(b.as_ptr().cast()), + _mm256_loadu_si256(b[4..].as_ptr().cast()), + ); + // `[b0 b4 b2 b6]` and `[b1 b5 b3 b7]`. + let (u, h) = (_mm256_unpacklo_epi64(lo, hi), _mm256_unpackhi_epi64(lo, hi)); + let odd = _mm256_xor_si256(u, h); + // `[s0 s4 s2 s6]`: the shuffles cancel in the first qword, where `s0` goes. + let even = _mm256_xor_si256( + _mm256_permute4x64_epi64::<0b01_00_10_00>(h), + _mm256_permute4x64_epi64::<0b11_10_11_00>(h), + ); + let even = _mm256_xor_si256(even, _mm256_zextsi128_si256(_mm_cvtsi64_si128(s0.0 as i64))); + let m = _mm256_set_epi64x(a[1].0 as i64, a[1].0 as i64, a[2].0 as i64, a[t0].0 as i64); + let r = reduce_halves256( + _mm256_clmulepi64_epi128::<0x00>(even, m), + _mm256_clmulepi64_epi128::<0x11>(even, m), + ); + _mm256_storeu_si256(o.as_mut_ptr().cast(), _mm256_unpacklo_epi64(r, odd)); + _mm256_storeu_si256(o[4..].as_mut_ptr().cast(), _mm256_unpackhi_epi64(r, odd)); + } + } } } @@ -261,6 +429,19 @@ mod tests { let mut part = vec![F128::ONE; (n + 1 - first) * stride]; mul_x(&h, stride, 1, first, &mut part); assert_eq!(part, got[first * stride..], "E range, mu={mu}, high={high}"); + + // Blocks of the first E lane. + let e: Vec = h.iter().step_by(stride).copied().collect(); + let want: Vec = got.iter().step_by(stride).copied().collect(); + for log_block in 3..=mu { + let n_block = 1usize << log_block; + for first in (0..1usize << mu).step_by(n_block) { + let mut part = vec![F128::ONE; n_block]; + let outside = |c: usize| e.get(c).copied().unwrap_or(F128::ZERO); + mul_x_block(&e[first..first + n_block], first, outside, &mut part); + assert_eq!(part, want[first..first + n_block], "E block {first}+{n_block}, mu={mu}"); + } + } } } } diff --git a/crates/experimental_prover/primitives/src/field/x4.rs b/crates/experimental_prover/primitives/src/field/x4.rs index 2cdf76aa95..bc4bd11853 100644 --- a/crates/experimental_prover/primitives/src/field/x4.rs +++ b/crates/experimental_prover/primitives/src/field/x4.rs @@ -258,6 +258,236 @@ impl BitXorAssign for Mixed4 { } } +/// Two `E` elements in one register as they lie in memory: 128-bit lane `i` is `[c0, c1]` of element `i`. +/// +/// Products keep this layout from load to store, where the plane form of [`F128x4`] transposes on both sides. +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +pub struct F128x2(__m256i); + +/// Two `K` elements in one register, each repeated across its 128-bit lane, so that it meets either coefficient of +/// the [`F128x2`] lane it multiplies. +#[derive(Clone, Copy, Debug)] +#[repr(transparent)] +pub struct F64x2(__m256i); + +/// A [`F128x2`] multiplier with `c0 + c1` of each lane in its low qword, the operand Karatsuba's middle product reads. +#[derive(Clone, Copy, Debug)] +pub struct Mul2 { + v: __m256i, + s: __m256i, +} + +/// Unreduced sums of [`F128x2`] products: the Karatsuba partials `a0 b0`, `a1 b1` and `(a0 + a1)(b0 + b1)` of each +/// lane, y-folded once at the reduction. +#[derive(Clone, Copy, Debug)] +pub struct Prod2 { + p0: __m256i, + p1: __m256i, + p01: __m256i, +} + +/// Unreduced sums of mixed products `e * k`: the coefficients of `y^0` and `y^1` of each lane. +#[derive(Clone, Copy, Debug)] +pub struct Mixed2 { + d0: __m256i, + d1: __m256i, +} + +/// `[c1, c0]` of each lane: the two qwords swapped. +#[inline(always)] +fn swap_qwords(v: __m256i) -> __m256i { + // SAFETY: the module's cfg enables avx2. + unsafe { _mm256_shuffle_epi32::<0x4E>(v) } +} + +/// The 128-bit lanes `vperm2i128` picks with immediate `I`: per half of the result, `0..=3` names `a.lo`, `a.hi`, +/// `b.lo`, `b.hi`, and bit 3 zeroes it. +#[inline(always)] +fn select(a: __m256i, b: __m256i) -> __m256i { + // SAFETY: the module's cfg enables avx2. + unsafe { _mm256_permute2x128_si256::(a, b) } +} + +impl F128x2 { + pub const ZERO: Self = Self( + // SAFETY: an all-zero bit pattern is a valid `__m256i`. + unsafe { std::mem::zeroed() }, + ); + + #[inline(always)] + pub fn load(e: &[F128; 2]) -> Self { + // SAFETY: the module's cfg enables avx2; two elements are 32 bytes. + Self(unsafe { _mm256_loadu_si256(e.as_ptr().cast()) }) + } + + #[inline(always)] + pub fn store(self, dst: &mut [F128; 2]) { + // SAFETY: the module's cfg enables avx2; two elements are 32 bytes. + unsafe { _mm256_storeu_si256(dst.as_mut_ptr().cast(), self.0) } + } + + #[inline(always)] + pub fn new(lo: F128, hi: F128) -> Self { + // SAFETY: the module's cfg enables avx2. + Self(unsafe { _mm256_set_epi64x(hi.c1 as i64, hi.c0 as i64, lo.c1 as i64, lo.c0 as i64) }) + } + + #[inline(always)] + pub fn splat(e: F128) -> Self { + Self::new(e, e) + } + + /// The lanes [`select`] picks from `a` and `b` with immediate `I`. + #[inline(always)] + pub fn select(a: Self, b: Self) -> Self { + Self(select::(a.0, b.0)) + } + + /// The coefficients `c0` of two pairs of lanes, then their coefficients `c1`, in lane order. + #[inline(always)] + pub fn limbs(a: Self, b: Self) -> (F64x4, F64x4) { + // SAFETY: the module's cfg enables avx2. + unsafe { + ( + F64x4(_mm256_permute4x64_epi64::<0xD8>(_mm256_unpacklo_epi64(a.0, b.0))), + F64x4(_mm256_permute4x64_epi64::<0xD8>(_mm256_unpackhi_epi64(a.0, b.0))), + ) + } + } + + /// This pair as a multiplier. + #[inline(always)] + pub fn factor(self) -> Mul2 { + Mul2 { + v: self.0, + s: xor(self.0, swap_qwords(self.0)), + } + } + + /// The lane-wise products by `m`, unreduced. + #[inline(always)] + pub fn mul_unreduced(self, m: Mul2) -> Prod2 { + let s = xor(self.0, swap_qwords(self.0)); + Prod2 { + p0: clmul::<0x00>(self.0, m.v), + p1: clmul::<0x11>(self.0, m.v), + p01: clmul::<0x00>(s, m.s), + } + } + + /// The lane-wise mixed products by `k`, unreduced. + #[inline(always)] + pub fn mul_base_unreduced(self, k: F64x2) -> Mixed2 { + // Immediate 0x01 multiplies the high qword of the first operand by the low qword of the second. + Mixed2 { + d0: clmul::<0x00>(self.0, k.0), + d1: clmul::<0x01>(self.0, k.0), + } + } +} + +impl Add for F128x2 { + type Output = Self; + #[inline(always)] + fn add(self, rhs: Self) -> Self { + Self(xor(self.0, rhs.0)) + } +} + +/// The embedding of the base field. +impl From for F128x2 { + #[inline(always)] + fn from(k: F64x2) -> Self { + // SAFETY: the module's cfg enables avx2. + Self(unsafe { _mm256_blend_epi32::<0xCC>(k.0, _mm256_setzero_si256()) }) + } +} + +impl F64x2 { + #[inline(always)] + pub fn new(lo: F64, hi: F64) -> Self { + // SAFETY: the module's cfg enables avx2. + Self(unsafe { _mm256_set_epi64x(hi.0 as i64, hi.0 as i64, lo.0 as i64, lo.0 as i64) }) + } + + #[inline(always)] + pub fn splat(k: F64) -> Self { + Self::new(k, k) + } + + /// The lanes [`select`] picks from `a` and `b` with immediate `I`. + #[inline(always)] + pub fn select(a: Self, b: Self) -> Self { + Self(select::(a.0, b.0)) + } +} + +impl Add for F64x2 { + type Output = Self; + #[inline(always)] + fn add(self, rhs: Self) -> Self { + Self(xor(self.0, rhs.0)) + } +} + +impl F64x4 { + /// Lanes `0, 1` and lanes `2, 3`, each as a [`F64x2`]. + #[inline(always)] + pub fn spread(self) -> [F64x2; 2] { + // SAFETY: the module's cfg enables avx2. + unsafe { + [ + F64x2(_mm256_permute4x64_epi64::<0x50>(self.0)), + F64x2(_mm256_permute4x64_epi64::<0xFA>(self.0)), + ] + } + } + + #[inline(always)] + pub fn store(self, dst: &mut [F64; 4]) { + // SAFETY: the module's cfg enables avx2; four qwords are 32 bytes. + unsafe { _mm256_storeu_si256(dst.as_mut_ptr().cast(), self.0) } + } +} + +impl Prod2 { + /// Both lanes reduced. + #[inline(always)] + pub fn reduce(self) -> F128x2 { + // SAFETY: the module's cfg enables avx2. + let [d0, d1] = fold(self.p0, self.p1, self.p01, xor, |p| unsafe { times_x256(p) }); + // SAFETY: the module's cfg enables avx2. + F128x2(unsafe { reduce_halves(d0, d1) }) + } +} + +impl BitXorAssign for Prod2 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + self.p0 = xor(self.p0, rhs.p0); + self.p1 = xor(self.p1, rhs.p1); + self.p01 = xor(self.p01, rhs.p01); + } +} + +impl Mixed2 { + /// Both lanes reduced. + #[inline(always)] + pub fn reduce(self) -> F128x2 { + // SAFETY: the module's cfg enables avx2. + F128x2(unsafe { reduce_halves(self.d0, self.d1) }) + } +} + +impl BitXorAssign for Mixed2 { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) { + self.d0 = xor(self.d0, rhs.d0); + self.d1 = xor(self.d1, rhs.d1); + } +} + #[cfg(test)] mod tests { use super::*; @@ -303,4 +533,45 @@ mod tests { assert_eq!(F128x4::from(F64x4::new(k)).to_array(), k.map(F128::from)); } } + + #[test] + fn pairs_match_scalar() { + let arr = |v: F128x2| { + let mut out = [F128::ZERO; 2]; + v.store(&mut out); + out + }; + let mut rng = Rng::new(31); + for _ in 0..200 { + let (a, b, c, d): ([F128; 2], [F128; 2], [F128; 2], [F128; 2]) = ( + [rng.ext(), F128::new(u64::MAX, 1 << 63)], + [rng.ext(), rng.ext()], + [rng.ext(), rng.ext()], + [rng.ext(), rng.ext()], + ); + let k: [F64; 4] = std::array::from_fn(|_| F64(rng.next_u64())); + let (va, vb, vc, vd) = (F128x2::load(&a), F128x2::load(&b), F128x2::load(&c), F128x2::load(&d)); + let mut acc = va.mul_unreduced(vb.factor()); + acc ^= vc.mul_unreduced(vd.factor()); + assert_eq!(arr(acc.reduce()), std::array::from_fn(|i| a[i] * b[i] + c[i] * d[i])); + let [k01, k23] = F64x4::load(&k).spread(); + let mut mixed = va.mul_base_unreduced(k01); + mixed ^= vb.mul_base_unreduced(k23); + let want = std::array::from_fn(|i| a[i].mul_base(k[i]) + b[i].mul_base(k[2 + i]) + F128::from(k[i])); + assert_eq!(arr(mixed.reduce() + F128x2::from(k01)), want); + assert_eq!(arr(F128x2::select::<0x21>(va, vb)), [a[1], b[0]]); + let (lo, hi) = F128x2::limbs(va, vb); + assert_eq!(lo.to_array(), [a[0].c0, a[1].c0, b[0].c0, b[1].c0].map(F64)); + assert_eq!(hi.to_array(), [a[0].c1, a[1].c1, b[0].c1, b[1].c1].map(F64)); + let mut stored = [F64::ZERO; 4]; + F64x4::load(&k).store(&mut stored); + assert_eq!(stored, k); + assert_eq!( + arr(F128x2::new(a[0], a[1]) + F128x2::splat(c[0])), + [a[0] + c[0], a[1] + c[0]] + ); + let sum = F64x2::new(k[0], k[1]) + F64x2::select::<0x01>(F64x2::splat(k[2]), F64x2::splat(k[2])); + assert_eq!(arr(F128x2::from(sum)), [k[0] + k[2], k[1] + k[2]].map(F128::from)); + } + } } From 836ec09f0c2378b4004cab49dbe1b911575403e1 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:49:00 +0000 Subject: [PATCH 14/18] experimental pcs: run the two-lane level-0 kernels on AVX2 without VPCLMULQDQ `primitives::field::x4` now builds with AVX2 and PCLMULQDQ, a 256-bit carry-less product being two 128-bit ones where VPCLMULQDQ is missing (`x4::clmul`). The level-0 lane fold (`Folded::block`) and the `X` products (`mul_x_block`) take their `F128x2` arms on the haswell tier too, in place of the portable eight-lane path. The leaf builder and `f128x8` keep their VPCLMULQDQ gates. Single thread, `aggregate --xmss 100`, last pass, same core, back to back, ms (base is upstream main): hsw: base / new PCS open 289 288 / 279 282 Fold round L0 / Project 82.7 82.2 / 74.0 75.1 (78.5 78.7 before) --- .../experimental_prover/pcs/src/whir_lanes.rs | 4 +-- crates/experimental_prover/pcs/src/xmul.rs | 19 +++++----- .../primitives/src/field/gf2_64x2.rs | 2 +- .../primitives/src/field/mod.rs | 2 +- .../primitives/src/field/x4.rs | 35 +++++++++++-------- 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/crates/experimental_prover/pcs/src/whir_lanes.rs b/crates/experimental_prover/pcs/src/whir_lanes.rs index 9507d46f97..d2ade143ef 100644 --- a/crates/experimental_prover/pcs/src/whir_lanes.rs +++ b/crates/experimental_prover/pcs/src/whir_lanes.rs @@ -115,7 +115,7 @@ impl Folded<'_> { #[cfg(not(all( target_arch = "x86_64", target_feature = "avx2", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", not(target_feature = "avx512f") )))] fn block(&self, i: usize, first: usize, out: &mut [F128]) { @@ -160,7 +160,7 @@ impl Folded<'_> { #[cfg(all( target_arch = "x86_64", target_feature = "avx2", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", not(target_feature = "avx512f") ))] fn block(&self, i: usize, first: usize, out: &mut [F128]) { diff --git a/crates/experimental_prover/pcs/src/xmul.rs b/crates/experimental_prover/pcs/src/xmul.rs index 0889405cac..cba16f82b4 100644 --- a/crates/experimental_prover/pcs/src/xmul.rs +++ b/crates/experimental_prover/pcs/src/xmul.rs @@ -44,7 +44,7 @@ impl Coef for F64 { #[cfg(all( target_arch = "x86_64", target_feature = "avx2", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", not(target_feature = "avx512f") ))] #[inline(always)] @@ -69,7 +69,7 @@ impl Coef for F128 { #[cfg(all( target_arch = "x86_64", target_feature = "avx2", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", not(target_feature = "avx512f") ))] #[inline(always)] @@ -188,19 +188,19 @@ pub(crate) fn mul_x_block(block: &[T], first: usize, outside: impl Fn(u T::x_block(block, first, outside, out); } -/// [`sums8`] and [`mul_x_block`] on two coefficients a register, in the order they lie in memory, where VPCLMULQDQ -/// has no AVX-512 beside it. +/// [`sums8`] and [`mul_x_block`] on two coefficients a register, in the order they lie in memory, where AVX2 has no +/// AVX-512 beside it. #[cfg(all( target_arch = "x86_64", target_feature = "avx2", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", not(target_feature = "avx512f") ))] pub(crate) mod lanes { use core::arch::x86_64::*; use primitives::field::gf2_64x2::x86_64::reduce_halves256; - use primitives::field::x4::{F64x2, F64x4, F128x2}; + use primitives::field::x4::{F64x2, F64x4, F128x2, clmul}; use primitives::field::{F64, F128}; use super::{constants, head8}; @@ -296,7 +296,7 @@ pub(crate) mod lanes { let chunks = block.as_chunks::<8>().0.iter().zip(out.as_chunks_mut::<8>().0); for (q, (b, o)) in chunks.enumerate() { let (s0, t0) = head8(block, q, first, &outside); - // SAFETY: the module's cfg enables avx2 and vpclmulqdq; each access is four whole qwords of a chunk of eight. + // SAFETY: the module's cfg enables avx2; each access is four whole qwords of a chunk of eight. unsafe { let (lo, hi) = ( _mm256_loadu_si256(b.as_ptr().cast()), @@ -312,10 +312,7 @@ pub(crate) mod lanes { ); let even = _mm256_xor_si256(even, _mm256_zextsi128_si256(_mm_cvtsi64_si128(s0.0 as i64))); let m = _mm256_set_epi64x(a[1].0 as i64, a[1].0 as i64, a[2].0 as i64, a[t0].0 as i64); - let r = reduce_halves256( - _mm256_clmulepi64_epi128::<0x00>(even, m), - _mm256_clmulepi64_epi128::<0x11>(even, m), - ); + let r = reduce_halves256(clmul::<0x00>(even, m), clmul::<0x11>(even, m)); _mm256_storeu_si256(o.as_mut_ptr().cast(), _mm256_unpacklo_epi64(r, odd)); _mm256_storeu_si256(o[4..].as_mut_ptr().cast(), _mm256_unpackhi_epi64(r, odd)); } diff --git a/crates/experimental_prover/primitives/src/field/gf2_64x2.rs b/crates/experimental_prover/primitives/src/field/gf2_64x2.rs index 83443b02ef..cc1d06d5a7 100644 --- a/crates/experimental_prover/primitives/src/field/gf2_64x2.rs +++ b/crates/experimental_prover/primitives/src/field/gf2_64x2.rs @@ -410,7 +410,7 @@ impl F128Unreduced { } /// Build from two 128-bit coefficients given as `[low, high]` words. - #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2"))] + #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq", target_feature = "avx2"))] #[inline] pub(crate) const fn from_coeffs(coeffs: [[u64; 2]; 2]) -> Self { Self { coeffs } diff --git a/crates/experimental_prover/primitives/src/field/mod.rs b/crates/experimental_prover/primitives/src/field/mod.rs index 5dafc4d88d..6fd0afcf20 100644 --- a/crates/experimental_prover/primitives/src/field/mod.rs +++ b/crates/experimental_prover/primitives/src/field/mod.rs @@ -18,7 +18,7 @@ pub mod gf2_64x2; #[cfg(all( target_arch = "x86_64", target_feature = "avx2", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", not(target_feature = "avx512f") ))] pub mod x4; diff --git a/crates/experimental_prover/primitives/src/field/x4.rs b/crates/experimental_prover/primitives/src/field/x4.rs index bc4bd11853..8e29126f65 100644 --- a/crates/experimental_prover/primitives/src/field/x4.rs +++ b/crates/experimental_prover/primitives/src/field/x4.rs @@ -1,6 +1,7 @@ -//! Four lanes of products in YMM registers, for a machine with VPCLMULQDQ but no AVX-512: the -//! 256-bit counterpart of the AVX-512 `x8`'s `F64x8`, `F128x8` and `Mixed8` for the leaf builder, -//! and the halves of the eight-lane `f128x8` types. +//! Four lanes of products in YMM registers, for a machine with AVX2 but no AVX-512: the 256-bit +//! counterpart of the AVX-512 `x8`'s `F64x8`, `F128x8` and `Mixed8` for the leaf builder, and the +//! halves of the eight-lane `f128x8` types. Without VPCLMULQDQ a 256-bit carry-less product is two +//! 128-bit ones. use core::arch::x86_64::*; use core::ops::{Add, BitXor, BitXorAssign, Mul}; @@ -36,11 +37,22 @@ fn xor(a: __m256i, b: __m256i) -> __m256i { unsafe { _mm256_xor_si256(a, b) } } -/// The products of the even lanes (`I = 0x00`) or the odd ones (`I = 0x11`) of two registers. +/// `vpclmulqdq` with immediate `I`: in each 128-bit lane, the product of the qwords `I` picks, bit 0 in `a` and bit 4 in +/// `b`. #[inline(always)] -fn clmul(a: __m256i, b: __m256i) -> __m256i { - // SAFETY: the module's cfg enables vpclmulqdq. - unsafe { _mm256_clmulepi64_epi128::(a, b) } +pub fn clmul(a: __m256i, b: __m256i) -> __m256i { + #[cfg(target_feature = "vpclmulqdq")] + // SAFETY: the cfg enables vpclmulqdq. + return unsafe { _mm256_clmulepi64_epi128::(a, b) }; + #[cfg(not(target_feature = "vpclmulqdq"))] + // SAFETY: the module's cfg enables avx2 and pclmulqdq. + unsafe { + let lane = |a: __m128i, b: __m128i| _mm_clmulepi64_si128::(a, b); + _mm256_set_m128i( + lane(_mm256_extracti128_si256::<1>(a), _mm256_extracti128_si256::<1>(b)), + lane(_mm256_castsi256_si128(a), _mm256_castsi256_si128(b)), + ) + } } /// The y-folded Karatsuba product of the even or the odd lanes. @@ -88,13 +100,8 @@ impl Mul for F64x4 { type Output = Self; #[inline(always)] fn mul(self, rhs: Self) -> Self { - // SAFETY: the module's cfg enables avx2 and vpclmulqdq. - Self(unsafe { - reduce_halves( - _mm256_clmulepi64_epi128::<0x00>(self.0, rhs.0), - _mm256_clmulepi64_epi128::<0x11>(self.0, rhs.0), - ) - }) + // SAFETY: the module's cfg enables avx2. + Self(unsafe { reduce_halves(clmul::<0x00>(self.0, rhs.0), clmul::<0x11>(self.0, rhs.0)) }) } } From 491743d4a8a231a58b353a69719168cabbab63af Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 20:55:23 +0000 Subject: [PATCH 15/18] experimental keccak_gkr: reduce the skip pass's cofactors modulo the minimal polynomial Only a block's skip sums at `s` matter: the prover evaluates them there. So `skip_clmul` reduces each cofactor modulo the minimal polynomial of `s` (Barrett's reduction from `PowerField`'s modulus, three carry-less products) from three words to two, and a slot takes six products instead of nine: per row 32 * 19 + 130 * 6 = 1388 products instead of 1490. `SkipBlock::new` takes the `PowerField`; the other arms keep exact sums, and the kernel test compares sums modulo the minimal polynomial. `SkipBlock::run` on one random block (microbenchmark, best of 6 rounds, one core of an EPYC 9454P, against kw/avx2-single-thread at e557ff799): vp 154.9 us before, 145.0 us after; hsw 319.4 us before, 301.9 us after. On an earlier tip of the branch: vp 152.3 to 142.8, hsw 291.9 to 275.7. `aggregate --xmss 100`, single thread, last pass, Skip span in ms, ref and new back to back twice on core 8 (host load average about 50): hsw: 262.7, 267.9 before; 255.1, 250.5 after. vp: unusable, the reference alone swung from 142 to 272. --- .../keccak_gkr/src/kernels.rs | 12 ++-- .../keccak_gkr/src/power.rs | 4 +- .../keccak_gkr/src/prover.rs | 2 +- .../keccak_gkr/src/skip_clmul.rs | 64 +++++++++++-------- .../keccak_gkr/src/skip_gfni.rs | 5 +- 5 files changed, 53 insertions(+), 34 deletions(-) diff --git a/crates/experimental_prover/keccak_gkr/src/kernels.rs b/crates/experimental_prover/keccak_gkr/src/kernels.rs index 820fe64607..7e78f8b0f1 100644 --- a/crates/experimental_prover/keccak_gkr/src/kernels.rs +++ b/crates/experimental_prover/keccak_gkr/src/kernels.rs @@ -313,8 +313,9 @@ cfg_select! { } impl<'a> SkipBlock<'a> { - /// For blocks summed at the position weights `weights`, as polynomials. - pub fn new(weights: &'a [[u64; 2]]) -> Box { + /// For blocks summed at the position weights `weights`, as polynomials. The sums are + /// exact, with no reduction modulo the minimal polynomial of the field's `s`. + pub fn new(weights: &'a [[u64; 2]], _field: &crate::power::PowerField) -> Box { Box::new(Self { acc: SkipAcc::new(), extended: [[0; SLOTS]; 5], @@ -1195,11 +1196,14 @@ mod tests { let weights: Vec<[u64; 2]> = (0..BITS) .map(|_| std::array::from_fn(|_| if dense { u64::MAX } else { rng.next_u64() })) .collect(); - let mut skip = SkipBlock::new(&weights); + // A kernel's sums may be reduced modulo the minimal polynomial of `s`. + let field = crate::power::PowerField::new(rng.ext()).expect("a generator"); + let reduce = |[a, b, c]: [u64; 3]| field.reduce(&[a, b, c, 0]); + let mut skip = SkipBlock::new(&weights, &field); for (b, block) in blocks.iter().enumerate() { skip.run(block); for (i, expected) in skip_block_portable(block, &weights).into_iter().enumerate() { - assert_eq!(skip.get(i), expected, "block {b} slot {i}"); + assert_eq!(reduce(skip.get(i)), reduce(expected), "block {b} slot {i}"); } } } diff --git a/crates/experimental_prover/keccak_gkr/src/power.rs b/crates/experimental_prover/keccak_gkr/src/power.rs index f3e4264ef1..2fc94cb317 100644 --- a/crates/experimental_prover/keccak_gkr/src/power.rs +++ b/crates/experimental_prover/keccak_gkr/src/power.rs @@ -44,9 +44,9 @@ macro_rules! row_kernel { pub struct PowerField { pub basis: PowerBasis, /// `s^128`, the modulus without its leading term. - modulus: Element, + pub(crate) modulus: Element, /// `floor(x^256 / modulus)` without its leading term, for Barrett's reduction. - barrett: Element, + pub(crate) barrett: Element, eval: PolyEval, } diff --git a/crates/experimental_prover/keccak_gkr/src/prover.rs b/crates/experimental_prover/keccak_gkr/src/prover.rs index 435c774650..6e6a0137dc 100644 --- a/crates/experimental_prover/keccak_gkr/src/prover.rs +++ b/crates/experimental_prover/keccak_gkr/src/prover.rs @@ -380,7 +380,7 @@ impl Layer<'_> { let (nb, written) = (self.witness.shape.n_blocks(), self.witness.written()); let mixed = parallel::map_reduce_with_state( self.rounds.len() * written, - || SkipBlock::new(weights), + || SkipBlock::new(weights, field), || vec![[0u64; 5]; MIXED], |skip, total, task| { let (segment, g) = (task / written, task % written); diff --git a/crates/experimental_prover/keccak_gkr/src/skip_clmul.rs b/crates/experimental_prover/keccak_gkr/src/skip_clmul.rs index fcc3706adc..92098dce4d 100644 --- a/crates/experimental_prover/keccak_gkr/src/skip_clmul.rs +++ b/crates/experimental_prover/keccak_gkr/src/skip_clmul.rs @@ -4,14 +4,16 @@ //! A row's sum `sum over x of w[x] e[x + 1] e[x + 2]` is taken as `e[2] A + e[4] B + e[0] C`, with //! the cofactors `A = w[0] e[1] + w[1] e[3]`, `B = w[2] e[3] + w[3] e[0]` and `C = w[4] e[1]`. //! They are linear in the values, so they are multiplied out on the row's packs and extended like -//! them: a cofactor has three words, and a slot takes nine products of an embedding and a word -//! where [`skip_row`](crate::kernels::skip_row) takes fifteen of an embedding, a product and a -//! weight's word. +//! them. Only a block's sums at `s` matter, so a cofactor is reduced modulo the minimal polynomial +//! of `s` to two words, and a slot takes six products of an embedding and a word where +//! [`skip_row`](crate::kernels::skip_row) takes fifteen of an embedding, a product and a weight's +//! word. use primitives::hash::positions::row_positions; use crate::clmul::{V, Vector}; use crate::kernels::{GROUP, MIXED, PACKS, embed}; +use crate::power::PowerField; use crate::witness::words; cfg_select! { @@ -59,12 +61,12 @@ cfg_select! { const _: () = assert!(GROUPS == V::N); /// The lanes of two words a slot's products read: `[e[2], e[4]]`, `[A0, B0]`, `[A1, B1]`, -/// `[A2, B2]`, `[C0, C1]` and `[C2, e[0]]`, word `k` of a cofactor its coefficients from `64 k`. -const FIELDS: usize = 6; +/// `[C0, C1]` and `e[0]` in the low word, word `k` of a cofactor its coefficients from `64 k`. +const FIELDS: usize = 5; /// Per word of the cofactors, a block's sums of the products of the word, at [`at`] each output /// of [`extend_group`](crate::kernels::extend_group) of each group. -type Acc = [[[u64; 2]; 2 * MIXED]; 3]; +type Acc = [[[u64; 2]; 2 * MIXED]; 2]; /// The pairs of lanes of a vector of four whose sums are its low mixed points, in the order of /// [`extend_group`](crate::kernels::extend_group). @@ -73,6 +75,9 @@ const LOW_MIXED: [(usize, usize); 4] = [(0, 1), (2, 3), (0, 2), (1, 3)]; /// [`skip_row`](crate::kernels::skip_row) over every row and slice of a block, a row at a time. pub struct SkipBlock<'a> { weights: &'a [[u64; 2]], + /// In every lane's low word: the high word of the Barrett constant of the minimal polynomial + /// of `s`, then the minimal polynomial's two words below its leading term. + reduction: [V; 3], /// The embeddings of a row's positions. e: [[u32; PACKS]; 5], /// Per field, the nine vectors of four of [`extend_group`](crate::kernels::extend_group) of @@ -83,19 +88,21 @@ pub struct SkipBlock<'a> { } impl<'a> SkipBlock<'a> { - /// For blocks summed at the position weights `weights`, as polynomials. - pub fn new(weights: &'a [[u64; 2]]) -> Box { + /// For blocks summed at the position weights `weights`, as polynomials in the power basis of + /// `field`. + pub fn new(weights: &'a [[u64; 2]], field: &PowerField) -> Box { Box::new(Self { weights, + reduction: [field.barrett[1], field.modulus[0], field.modulus[1]].map(V::splat), e: [[0; PACKS]; 5], vectors: [[[V::zero(); 4]; 9]; FIELDS], - acc: [[[0; 2]; 2 * MIXED]; 3], + acc: [[[0; 2]; 2 * MIXED]; 2], }) } /// Sum a block. pub fn run(&mut self, block: &[u64]) { - self.acc = [[[0; 2]; 2 * MIXED]; 3]; + self.acc = [[[0; 2]; 2 * MIXED]; 2]; for r in 0..5 { for z in 0..64 { let js = row_positions(r, z); @@ -137,15 +144,8 @@ impl<'a> SkipBlock<'a> { for h in 0..2 { // Pack `q + h`'s embeddings in the low word. let e = e.map(|e| if h == 0 { e } else { e.high() }); - let ([a0, a1], [b0, b1], [c0, c1]) = (a[h], b[h], c[h]); - let fields = [ - e[2].unpacklo(e[4]), - a0.unpacklo(b0), - a0.high().xor(a1).unpacklo(b0.high().xor(b1)), - a1.high().unpacklo(b1.high()), - c0.unpacklo(c0.high().xor(c1)), - c1.high().unpacklo(e[0]), - ]; + let [a, b, c] = [a[h], b[h], c[h]].map(|x| reduce(x, self.reduction)); + let fields = [e[2].unpacklo(e[4]), a.unpacklo(b), a.high().unpacklo(b.high()), c, e[0]]; for (vectors, field) in self.vectors.iter_mut().zip(fields) { vectors[(q + h) / 4][(q + h) % 4] = field; } @@ -190,18 +190,32 @@ impl<'a> SkipBlock<'a> { } else { (slot - 128, 64) }; - let [s0, s1, s2] = self.acc.each_ref().map(|acc| acc[at(half, i)]); - [s0[0], s0[1] ^ s1[0], s1[1] ^ s2[0]] + let [s0, s1] = self.acc.each_ref().map(|acc| acc[at(half, i)]); + [s0[0], s0[1] ^ s1[0], s1[1]] } } +/// A cofactor `low + high x^64`, `high`'s high word below 64 coefficients, modulo the minimal +/// polynomial of `s` by Barrett's reduction, `reduction` as [`SkipBlock`] holds it: its words in a +/// lane each. +#[inline(always)] +fn reduce([low, high]: [V; 2], [barrett, modulus_low, modulus_high]: [V; 3]) -> V { + // The quotient, from the cofactor's word 2 alone. + let top = high.high(); + let q = top.xor(top.clmul::<0x00>(barrett).high()); + // The low two words of the cofactor and of `q` times the modulus. + let shifted = |x: V| V::zero().unpacklo(x); + low.xor(shifted(high)) + .xor(q.clmul::<0x00>(modulus_low)) + .xor(shifted(q.clmul::<0x00>(modulus_high))) +} + /// Adds the products of the [`FIELDS`] at output `i` of the groups from `pass` to their sums. #[inline(always)] -fn accumulate(acc: &mut Acc, pass: usize, i: usize, [e, w0, w1, w2, wc, wd]: [V; FIELDS]) { +fn accumulate(acc: &mut Acc, pass: usize, i: usize, [e, w0, w1, c, e0]: [V; FIELDS]) { let sums = [ - e.clmul::<0x00>(w0).xor(e.clmul::<0x11>(w0)).xor(wd.clmul::<0x01>(wc)), - e.clmul::<0x00>(w1).xor(e.clmul::<0x11>(w1)).xor(wd.clmul::<0x11>(wc)), - e.clmul::<0x00>(w2).xor(e.clmul::<0x11>(w2)).xor(wd.clmul::<0x01>(wd)), + e.clmul::<0x00>(w0).xor(e.clmul::<0x11>(w0)).xor(e0.clmul::<0x00>(c)), + e.clmul::<0x00>(w1).xor(e.clmul::<0x11>(w1)).xor(e0.clmul::<0x10>(c)), ]; for (acc, sum) in acc.iter_mut().zip(sums) { // Lane `g` is group `pass + g`. diff --git a/crates/experimental_prover/keccak_gkr/src/skip_gfni.rs b/crates/experimental_prover/keccak_gkr/src/skip_gfni.rs index 1e41431080..4acd2c8ebd 100644 --- a/crates/experimental_prover/keccak_gkr/src/skip_gfni.rs +++ b/crates/experimental_prover/keccak_gkr/src/skip_gfni.rs @@ -369,8 +369,9 @@ pub struct SkipBlock { } impl SkipBlock { - /// For blocks summed at the position weights `weights`, as polynomials. - pub fn new(weights: &[[u64; 2]]) -> Box { + /// For blocks summed at the position weights `weights`, as polynomials. The sums are exact, + /// with no reduction modulo the minimal polynomial of the field's `s`. + pub fn new(weights: &[[u64; 2]], _field: &crate::power::PowerField) -> Box { let mut residues = vec![[splat(0); RESIDUES]; BITS / 8]; for (residues, weights) in residues.iter_mut().zip(weights[..BITS].as_chunks::<8>().0) { let bytes = transpose_bytes(weights.map(|w| [w[0], w[1], 0, 0])); From c528130100e195de22a48f4ef18199d181fc4fbc Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Tue, 17 Nov 2026 21:19:06 +0000 Subject: [PATCH 16/18] experimental lean_vm_core: bus GKR and leaf lanes on AVX2 without VPCLMULQDQ With `x4` building on AVX2 and PCLMULQDQ alone (a 256-bit carry-less product as two 128-bit ones), `f128x8`'s AVX2 arm, the bus GKR's vertical kernels (`products_into`, the round message, the fused fold) and the lane-wise leaf fill now run on the haswell tier too, in place of the portable eight-lane types and the scalar row loops. They are gated on AVX2 and PCLMULQDQ, with VPCLMULQDQ where AVX-512 is present (the `x8` path), so an AVX-512 machine without VPCLMULQDQ keeps the portable path. Single thread, core 32 of an EPYC 9454P, `aggregate --xmss 100`, `target-cpu=haswell`, the previous commit vs this, two quiet pairs back to back, ms: Bus leaves 49.5 49.3 -> 35.5 34.5 Bus GKR 72.8 72.4 -> 57.9 57.7 Prove bus 156 156 -> 127 126 Prove constraints 22.0 21.8 -> 19.8 20.3 proving time 1138 1134 -> 1135 (noisy) 1103 The other stages are unchanged. vp, gfni and native compile to the same code as before. --- .../lean_vm_core/src/gkr.rs | 112 +++++++++++++++--- .../lean_vm_core/src/leaf.rs | 23 +++- .../primitives/src/field/f128x8.rs | 30 +++-- 3 files changed, 136 insertions(+), 29 deletions(-) diff --git a/crates/experimental_prover/lean_vm_core/src/gkr.rs b/crates/experimental_prover/lean_vm_core/src/gkr.rs index 1711e1317a..1038404216 100644 --- a/crates/experimental_prover/lean_vm_core/src/gkr.rs +++ b/crates/experimental_prover/lean_vm_core/src/gkr.rs @@ -6,8 +6,9 @@ //! Leaves and every layer are `E`-valued (the bus fingerprints mix `K`-columns //! into `E` upstream, [`crate::leaf`]). //! -//! With VPCLMULQDQ the large rounds run eight pairs of rows at a time in transposed form -//! (`vertical`), and skip the runs of rows known to be the identity. +//! On x86 with AVX2 and a carry-less multiplier (VPCLMULQDQ with AVX-512) the large rounds run +//! eight pairs of rows at a time in transposed form (`vertical`), and skip the runs of rows known +//! to be the identity. use std::ops::Range; @@ -41,9 +42,19 @@ fn sum_tasks(tasks: usize, parallel: bool, f: impl Fn(usize) -> [F128; 4] + Sync /// `out[i] = ∏ current[4i..4i+4]`, for `out.len() == current.len() / 4`. pub(crate) fn products_into(current: &[F128], out: &mut [F128]) { debug_assert_eq!(out.len(), current.len() / 4); - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] let done = vertical::products_into(current, out); - #[cfg(not(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq")))] + #[cfg(not(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + )))] let done = 0; for (row, slot) in out.iter_mut().enumerate().skip(done) { let [left, right] = mul2( @@ -161,7 +172,12 @@ struct SplitEq { hi: Vec, lo_bits: usize, /// `lo` eight entries at a time, for the vertical kernels. - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] lo8: Vec, } @@ -171,7 +187,12 @@ impl SplitEq { let (low, high) = vars.split_at(lo_bits); let lo = eq_table(low); Self { - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] lo8: lo.as_chunks::<8>().0.iter().map(vertical::load_eight).collect(), lo, hi: eq_table(high), @@ -259,7 +280,12 @@ fn runs<'a>(range: Range, ones: &'a [Range]) -> impl Iterator, - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] spare: Vec, /// Runs of rows known to be the identity, sorted: the kernels skip them, and /// the vertical ones leave their groups unwritten. @@ -290,9 +326,19 @@ impl QuaternaryLayerState { Self { values, logical_rows: width, - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] groups: Vec::new(), - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] spare: Vec::new(), ones: Vec::new(), } @@ -300,7 +346,12 @@ impl QuaternaryLayerState { /// `(q(0)+q(1), [X²]q, [X³]q, [X⁴]q)`. fn round_message(&self, eq: &SplitEq) -> [F128; 4] { - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] if self.logical_rows >= VERTICAL_ROWS { return vertical::message(&self.values, &self.ones, eq); } @@ -308,13 +359,23 @@ impl QuaternaryLayerState { } fn fold(&mut self, challenge: F128) { - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] self.leave_groups(); self.fold_scalar(challenge); } /// The vertical kernels' view of the rows, and the buffer the fold writes. - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] fn split(&mut self) -> (vertical::Input<'_>, &mut Vec) { let data = if self.groups.is_empty() { vertical::Data::Rows(&self.values) @@ -325,7 +386,12 @@ impl QuaternaryLayerState { } /// Move the rows back into `values` for the scalar kernels. - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] fn leave_groups(&mut self) { if !self.groups.is_empty() { self.values = vertical::to_rows(&self.groups, &self.ones); @@ -510,7 +576,12 @@ fn fold_and_message( challenge: F128, eq: &SplitEq, ) -> [[F128; 4]; N_BUS_TREES] { - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] { if trees[0].logical_rows >= VERTICAL_ROWS { let [a, b, c] = trees.each_mut().map(QuaternaryLayerState::split); @@ -530,11 +601,16 @@ fn fold_and_message( trees.each_mut().map(|tree| tree.fold_and_message_scalar(challenge, eq)) } -/// The VPCLMULQDQ kernels: eight pairs of rows at a time, transposed so that every +/// The vector kernels: eight pairs of rows at a time, transposed so that every /// product is lane-wise (`primitives::field::x8` on AVX-512, `f128x8`'s two halves /// of `x4` lanes on AVX2). After its first fold a layer keeps its rows transposed, /// in [`Group`]s, until it is small enough for the scalar kernels. -#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] +#[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) +))] mod vertical { use super::{N_BUS_TREES, SplitEq, add4, coarsen, sum_tasks}; use primitives::field::F128; diff --git a/crates/experimental_prover/lean_vm_core/src/leaf.rs b/crates/experimental_prover/lean_vm_core/src/leaf.rs index 0c54a64a1c..cfaef01e12 100644 --- a/crates/experimental_prover/lean_vm_core/src/leaf.rs +++ b/crates/experimental_prover/lean_vm_core/src/leaf.rs @@ -219,9 +219,19 @@ impl LeafBlock { let (real_out, padding) = out.split_at_mut(real); // A padding row's leaf is the identity, so it leaves the product alone. padding.fill(F128::ONE); - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] let done = self.fill_lanes(z0, real_out, cols); - #[cfg(not(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq")))] + #[cfg(not(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + )))] let done = 0; let mut index = g_pow(z0 + done); for (i, slot) in real_out.iter_mut().enumerate().skip(done) { @@ -233,7 +243,12 @@ impl LeafBlock { /// [`fill`](Self::fill) [`lanes::N`] rows at a time, every term one lane-wise mixed /// product; returns how many rows it wrote. - #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "vpclmulqdq"))] + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + any(target_feature = "vpclmulqdq", not(target_feature = "avx512f")) + ))] fn fill_lanes(&self, z0: usize, out: &mut [F128], cols: &[&[F64]]) -> usize { use lanes::{E, K, Mixed, N}; let load = |source: &[F64], z: usize| K::load(source[z..z + N].try_into().unwrap()); @@ -271,7 +286,7 @@ mod lanes { #[cfg(all( target_arch = "x86_64", target_feature = "avx2", - target_feature = "vpclmulqdq", + target_feature = "pclmulqdq", not(target_feature = "avx512f") ))] mod lanes { diff --git a/crates/experimental_prover/primitives/src/field/f128x8.rs b/crates/experimental_prover/primitives/src/field/f128x8.rs index 3bceab7abe..597b3c6fcd 100644 --- a/crates/experimental_prover/primitives/src/field/f128x8.rs +++ b/crates/experimental_prover/primitives/src/field/f128x8.rs @@ -19,14 +19,22 @@ use super::gf2_64x2::{F128, F128Unreduced}; #[cfg(all( target_arch = "x86_64", - target_feature = "vpclmulqdq", target_feature = "avx2", + target_feature = "pclmulqdq", not(target_feature = "avx512f") ))] pub use avx2::{F64x8, F128x8, Prod8}; #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] pub use avx512::{Acc8, F64x8, F128x8}; -#[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2")))] +#[cfg(not(any( + all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + not(target_feature = "avx512f") + ), + all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f") +)))] pub use portable::{F64x8, F128x8, Prod8}; #[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f")))] pub use sum::Acc8; @@ -36,8 +44,16 @@ pub use sum::Acc8; #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] pub type Prod8 = Acc8; -/// The eight-lane types on a target without the wide multiplier. -#[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2")))] +/// The eight-lane types on a target without the `x4` or `x8` multipliers. +#[cfg(not(any( + all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "pclmulqdq", + not(target_feature = "avx512f") + ), + all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f") +)))] mod portable { use super::sum::halves; use super::{F64, F128, F128Unreduced}; @@ -318,12 +334,12 @@ pub fn xor_into8(out: &mut [F128; 8], add: &[F128; 8]) { } } -/// The eight-lane types with VPCLMULQDQ but no AVX-512: two halves of four lanes, each the `x4` form, a -/// YMM plane per coefficient, so a product is lane-wise, two lanes to an instruction. +/// The eight-lane types with AVX2 but no AVX-512: two halves of four lanes, each the `x4` form, a YMM plane per +/// coefficient, so a product is lane-wise, two lanes to an instruction with VPCLMULQDQ and one without. #[cfg(all( target_arch = "x86_64", - target_feature = "vpclmulqdq", target_feature = "avx2", + target_feature = "pclmulqdq", not(target_feature = "avx512f") ))] mod avx2 { From 9d7e41f76daf2b7aa6673aef4c7866fbc812672e Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Wed, 18 Nov 2026 01:21:34 +0000 Subject: [PATCH 17/18] experimental pcs: pad the base encode's rows to four lanes on x86 The base commit encoded n_lanes data lanes as next_multiple_of(8), so the NTT only shrank when the committed cells crossed an eighth of the stack. Rows are now padded to `ntt::ROW_LANES`: four on x86 with AVX2 (AVX-512 included), eight on NEON and the portable build, where the kernels' tail goes word by word. - The lane-major first gathered pass and the opening's rebuild of it move whole octets of lanes and then a quad where the row ends in one (`stage::lane_groups`), with a four-line stage tile. - The AVX2 register kernels take the count of vectors that hold lanes, so a last chunk of one vector no longer multiplies a vector of zeros with VPCLMULQDQ (two vectors in flight). - Leaf images are unchanged: the padding is the image's leading zeros, and only the data lanes ride the proof. A narrower row also absorbs more of the leading zeros as the shared zero-prefix state, so a leaf can take one Keccak permutation fewer. - The base encode's span reports the data lanes beside the row width. Single thread, EPYC 9454P core 44, `aggregate --xmss N`, three passes, A/B back to back against origin/kw/avx2-single-thread (ms): | tier | N | data / row lanes | Commit/NTT | Open L0 | Commit L1..L3 NTT | total | | gfni | 200 | 42: 48 -> 44 | 356 -> 323 | 13.5 -> 13.0 | 168 -> 169 | 1276 -> 1244 | | gfni | 1300 | 43: 48 -> 44 | 1470 -> 1340 | 16.5 -> 15.7 | 693 -> 695 | 5811 -> 5666 | | gfni | 1440 | 38: 40 -> 40 | 2480 -> 2490 | 370 -> 374 | 1412 -> 1424 | 8744 -> 8761 | | hsw | 200 | 42: 48 -> 44 | 401 -> 364 | 18.7 -> 17.6 | 186 -> 186 | 2115 -> 2000 | | hsw | 1300 | 43: 48 -> 44 | 1670 -> 1510 | 23.7 -> 21.7 | 775 -> 775 | 10321 -> 10179 | | hsw | 1440 | 38: 40 -> 40 | 2840 -> 2830 | 512 -> 510 | 1588 -> 1592 | 14297 -> 14370 | | native (AVX-512) | 200 | 42: 48 -> 44 | 227 -> 220 | 9.6 -> 9.6 | 102 -> 103 | 932 -> 928 | | native (AVX-512) | 1300 | 43: 48 -> 44 | 941 -> 913 | 11.9 -> 12.1 | 425 -> 424 | 4422 -> 4382 | | native (AVX-512) | 1440 | 38: 40 -> 40 | 1640 -> 1640 | 301 -> 301 | 866 -> 867 | 6555 -> 6568 | At 1440 the row is 40 lanes either way, the no-regression shape; its differences are within the runs' spread (hsw ref 200 had 11% spread). On AVX-512 the same tree with eight-lane rows measured 225 ms at 200 and 942 ms at 1300, so four lanes is kept there too. The extension commits encode 16 lanes of data each and need no padding. --- crates/experimental_prover/pcs/src/ntt.rs | 2 +- .../pcs/src/ntt/gathered.rs | 45 ++++++---- .../pcs/src/ntt/kernels.rs | 48 +++++++--- .../pcs/src/ntt/openings.rs | 32 +++---- .../experimental_prover/pcs/src/ntt/plan.rs | 6 +- .../experimental_prover/pcs/src/ntt/stage.rs | 89 ++++++++++++------- .../experimental_prover/pcs/src/ntt/tests.rs | 30 ++++--- crates/experimental_prover/pcs/src/whir.rs | 41 +++++---- 8 files changed, 175 insertions(+), 118 deletions(-) diff --git a/crates/experimental_prover/pcs/src/ntt.rs b/crates/experimental_prover/pcs/src/ntt.rs index 23b9c83907..b2d322e246 100644 --- a/crates/experimental_prover/pcs/src/ntt.rs +++ b/crates/experimental_prover/pcs/src/ntt.rs @@ -69,5 +69,5 @@ mod twiddles; mod window; pub(crate) use encode::{Codeword, Keep}; -pub(crate) use kernels::transpose_butterfly_lanes; +pub(crate) use kernels::{ROW_LANES, transpose_butterfly_lanes}; pub use twiddles::AdditiveNttF64; diff --git a/crates/experimental_prover/pcs/src/ntt/gathered.rs b/crates/experimental_prover/pcs/src/ntt/gathered.rs index 3ab5e7b70f..c7863fa34d 100644 --- a/crates/experimental_prover/pcs/src/ntt/gathered.rs +++ b/crates/experimental_prover/pcs/src/ntt/gathered.rs @@ -6,7 +6,7 @@ use primitives::stream::Stream; use super::plan::Plan; use super::scratch::{Line, with_scratch}; use super::source::Source; -use super::stage::{BATCH, Stage}; +use super::stage::{BATCH, Stage, lane_groups}; use super::twiddles::AdditiveNttF64; use super::window::Window; @@ -47,9 +47,10 @@ use super::window::Window; /// /// - Word `w` of message row `j` is word `j` of lane block `num_ntts - 1 - w`, and zero below the padding. /// - A lane block's cache line holds [`BATCH`] consecutive rows, so a task takes that many residues. -/// - It goes one octet of lanes at a time, and the octet of the residues is 64 lanes of one transform. -/// - It transposes those lanes into a stage of `2^g` rows of 64 words, one line per residue. -/// - Each block transforms a copy of the stage and scatters it line by line. +/// - It goes one group of lanes at a time, whole octets and then a quad where the row ends in one ([`lane_groups`]). +/// - With the residues, a group of `lanes` lanes is `BATCH * lanes` lanes of one transform. +/// - It transposes those lanes into a stage of `2^g` rows of `BATCH * lanes` words, `lanes` words per residue. +/// - Each block transforms a copy of the stage and scatters it residue by residue. pub(super) struct GatheredPass<'a> { src: Source<'a>, /// The transpose of a lane-major message, if the source is one. @@ -63,13 +64,14 @@ pub(super) struct GatheredPass<'a> { /// Rows of a layer-`layer` block, and the rows between the rows of a group. block_rows: usize, step: usize, - /// Words of a group's row, and of the group. + /// Words of a group's row, and of the group: for a lane-major message, those of an octet's. width: usize, group_len: usize, /// Residues one task step takes, and the words of its stage. batch: usize, stage_len: usize, - /// How a finished group drains: in pieces of `piece` words, `piece_stride` apart. + /// How a finished group drains: in pieces of `piece` words, `piece_stride` apart. A lane-major group's pieces + /// are its lanes of each residue. piece: usize, piece_stride: usize, stream: bool, @@ -85,7 +87,7 @@ impl<'a> GatheredPass<'a> { // The blocks the buffer holds. let first_block = win.first_row() / block_rows; let blocks = first_block..first_block + (win.len() / num_ntts) / block_rows; - // A lane-major group is 64 lanes of a batch of residues, transposed into a stage: one line per residue in each row. + // A lane-major group is at most an octet of lanes of a batch of residues, transposed into a stage. let staged = matches!(src, Source::Lanes(_)); let batch = if staged { BATCH } else { 1 }; let width = if staged { BATCH * 8 } else { num_ntts }; @@ -135,11 +137,14 @@ impl<'a> GatheredPass<'a> { match &self.staging { Some(staging) => { let r = t * self.batch; - for octet in 0..self.num_ntts / 8 { - staging.fill(stage, r, octet); + for (lane0, lanes) in lane_groups(self.num_ntts) { + let width = BATCH * lanes; + let stage = &mut stage[..width << self.g]; + staging.fill(stage, r, lane0, lanes); for block in self.blocks.clone().rev() { - let dst = win.row(self.block_rows, block, r) + 8 * octet; - self.group(ntt, scratch, stream, win, base, block, r, Some(&*stage), dst); + let dst = win.row(self.block_rows, block, r) + lane0; + let (from, piece) = (Some(&*stage), lanes); + self.group(ntt, scratch, stream, win, base, block, r, from, dst, width, piece); } } } @@ -151,7 +156,9 @@ impl<'a> GatheredPass<'a> { for block in self.blocks.clone().rev() { let dst = win.row(self.block_rows, block, t); let from = msg.map(|_| &*stage); - self.group(ntt, scratch, stream, win, base, block, t, from, dst); + self.group( + ntt, scratch, stream, win, base, block, t, from, dst, self.width, self.piece, + ); } } } @@ -185,7 +192,8 @@ impl<'a> GatheredPass<'a> { /// Transform residue `r`'s group as layer-`layer` block `block`, and send it back to word `dst` on. /// - /// The rows are `from` the stage, or gathered from the block's own rows of the buffer. + /// The rows, `width` words each, are `from` the stage, or gathered from the block's own rows of the buffer. + /// They drain in pieces of `piece` words. #[inline(always)] fn group( &self, @@ -198,8 +206,11 @@ impl<'a> GatheredPass<'a> { r: usize, from: Option<&[F64]>, dst: usize, + width: usize, + piece: usize, ) { let (layer, g, num_ntts, step) = (self.layer, self.g, self.num_ntts, self.step); + let scratch = &mut scratch[..width << g]; if from.is_none() { // Replicas: each block transforms its own rows of the buffer. self.gather( @@ -207,11 +218,11 @@ impl<'a> GatheredPass<'a> { base.0.cast_const().wrapping_add(win.row(self.block_rows, block, r)), ); } - ntt.run_layers(scratch, from, layer + g, self.width, layer, layer, block, None); + ntt.run_layers(scratch, from, layer + g, width, layer, layer, block, None); // Write the group back: scratch row `i` lands `i * step` rows after word `dst`, in pieces of `piece` words // `piece_stride` apart. A streaming write moves whole aligned cache lines: `piece` is then 8. - for (i, row) in scratch.chunks_exact(self.width).enumerate() { - for (k, piece) in row.chunks_exact(self.piece).enumerate() { + for (i, row) in scratch.chunks_exact(width).enumerate() { + for (k, piece) in row.chunks_exact(piece).enumerate() { // SAFETY: // - The group lies inside the buffer, so every piece is in bounds, and line-aligned when streamed. // - Only this group's task writes these rows of the buffer. @@ -219,7 +230,7 @@ impl<'a> GatheredPass<'a> { let at = base.add(dst + i * step * num_ntts + k * self.piece_stride); match stream { Some(s) => s.copy_lines(at.cast::(), &primitives::cast_slice::(piece)[0]), - None => at.copy_from_nonoverlapping(piece.as_ptr(), self.piece), + None => at.copy_from_nonoverlapping(piece.as_ptr(), piece.len()), } } } diff --git a/crates/experimental_prover/pcs/src/ntt/kernels.rs b/crates/experimental_prover/pcs/src/ntt/kernels.rs index 706fa26a03..827d9a40aa 100644 --- a/crates/experimental_prover/pcs/src/ntt/kernels.rs +++ b/crates/experimental_prover/pcs/src/ntt/kernels.rs @@ -4,6 +4,17 @@ use core::arch::x86_64::__m512i; use primitives::field::F64; +/// The lanes a codeword row is padded to a multiple of, with zero lanes: whole vectors of the kernels, or quads. +/// +/// - AVX2 runs four lanes to a vector, and a last chunk of one vector multiplies only it. +/// - AVX-512 runs eight, but masks its last vector to the row: a narrower row still saves memory traffic and the +/// Merkle leaf's hashing, which absorbs whole blocks of leading zero lanes as one shared state. +/// - NEON runs eight, and the rest word by word. +#[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq", target_feature = "avx2"))] +pub(crate) const ROW_LANES: usize = 4; +#[cfg(not(all(target_arch = "x86_64", target_feature = "pclmulqdq", target_feature = "avx2")))] +pub(crate) const ROW_LANES: usize = 8; + /// Visit every row group of a fused multi-layer block. /// /// ```text @@ -146,10 +157,10 @@ pub(super) fn butterfly_interleaved_fused_3layer( num_ntts, t, #[inline(always)] - |v, t| { + |v, t, vectors| { radix8_schedule( #[inline(always)] - |top, bot, k| avx2::butterfly_regs(v, top, bot, t[k]), + |top, bot, k| avx2::butterfly_regs(v, top, bot, t[k], vectors), ); }, ); @@ -295,10 +306,10 @@ pub(super) fn butterfly_interleaved_fused_2layer( num_ntts, t, #[inline(always)] - |v, t| { + |v, t, vectors| { radix4_schedule( #[inline(always)] - |top, bot, k| avx2::butterfly_regs(v, top, bot, t[k]), + |top, bot, k| avx2::butterfly_regs(v, top, bot, t[k], vectors), ); }, ); @@ -825,12 +836,18 @@ mod avx2 { dst[4 * i..4 * i + n].copy_from_slice(&words[..n]); } - /// Butterfly rows `top` and `bot` of every vector in flight. + /// Butterfly rows `top` and `bot` of the first `vectors` vectors in flight. #[inline] #[target_feature(enable = "pclmulqdq", enable = "avx2")] #[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] - pub(super) fn butterfly_regs(v: &mut RowVectors, top: usize, bot: usize, (tw, zero): Twiddle) { - for k in 0..LANE_VECTORS { + pub(super) fn butterfly_regs( + v: &mut RowVectors, + top: usize, + bot: usize, + (tw, zero): Twiddle, + vectors: usize, + ) { + for k in 0..vectors { let (u, w) = (v[top][k], v[bot][k]); let new_u = if zero { u } else { mul_xor_ymm(u, w, tw) }; v[top][k] = new_u; @@ -848,7 +865,7 @@ mod avx2 { stride_rows: usize, num_ntts: usize, twiddles: &[F64; T], - kernel: impl Fn(&mut RowVectors, &[Twiddle; T]), + kernel: impl Fn(&mut RowVectors, &[Twiddle; T], usize), ) { fused_rows::(block, src, stride_rows, num_ntts, |src, rows| { fused_lanes( @@ -856,12 +873,13 @@ mod avx2 { rows, twiddles, #[inline(always)] - |v, t| kernel(v, t), + |v, t, vectors| kernel(v, t, vectors), ) }); } - /// `fused_lanes_avx512` at half the width: [`LANE_VECTORS`] four-lane vectors of every row at a time. + /// `fused_lanes_avx512` at half the width: [`LANE_VECTORS`] four-lane vectors of every row at a time, and in the + /// last chunk only those that hold lanes. #[inline] #[target_feature(enable = "pclmulqdq", enable = "avx2")] #[cfg_attr(target_feature = "vpclmulqdq", target_feature(enable = "vpclmulqdq"))] @@ -869,7 +887,7 @@ mod avx2 { src: Option<[&[F64]; N]>, rows: &mut [&mut [F64]; N], twiddles: &[F64; T], - kernel: impl Fn(&mut RowVectors, &[Twiddle; T]), + kernel: impl Fn(&mut RowVectors, &[Twiddle; T], usize), ) { const CHUNK: usize = 4 * LANE_VECTORS; let len = rows[0].len(); @@ -888,7 +906,7 @@ mod avx2 { }), None => std::array::from_fn(|i| std::array::from_fn(|k| load_at(&*chunks[i], 4 * k))), }; - kernel(&mut v, &tw); + kernel(&mut v, &tw, LANE_VECTORS); for i in 0..N { for k in 0..LANE_VECTORS { store_at(chunks[i], 4 * k, v[i][k]); @@ -906,7 +924,11 @@ mod avx2 { None => load_prefix(&*rows[i], reg + k, n(k)), }) }); - kernel(&mut v, &tw); + // Each count is its own call, so the kernel's loops over the vectors unroll. + match (len - lane).div_ceil(4) { + 1 => kernel(&mut v, &tw, 1), + _ => kernel(&mut v, &tw, LANE_VECTORS), + } for i in 0..N { for k in 0..LANE_VECTORS { store_prefix(rows[i], reg + k, n(k), v[i][k]); diff --git a/crates/experimental_prover/pcs/src/ntt/openings.rs b/crates/experimental_prover/pcs/src/ntt/openings.rs index 1176d145e8..e0afd2f04a 100644 --- a/crates/experimental_prover/pcs/src/ntt/openings.rs +++ b/crates/experimental_prover/pcs/src/ntt/openings.rs @@ -6,7 +6,7 @@ use super::encode::Codeword; use super::plan::Plan; use super::scratch::with_scratch; use super::source::Lanes; -use super::stage::{BATCH, Stage}; +use super::stage::{BATCH, Stage, lane_groups}; use super::twiddles::AdditiveNttF64; /// The rows at the sorted `positions` of a codeword kept whole, `row_words` words each, back to back. @@ -177,39 +177,27 @@ impl<'a> Rebuild<'a> { // A group writes row `r` of every queried sub-block, and the groups cover every residue `r`. let out = parallel::SendPtr(built.as_mut_ptr()); parallel::for_each_chunk((1usize << log_step) / BATCH, |lo, hi| { - // One octet of lanes of a batch of residues is 64 lanes of one transform. + // An octet of lanes of a batch of residues is 64 lanes of one transform, a quad 32. let stage_len = rows * BATCH * 8; // `built` is far larger than a node's L3 share and is read only after every task: its lines stream. let stream = primitives::stream::Stream::new(); with_scratch(2 * stage_len, |all| { let (stage, scratch) = all.split_at_mut(stage_len); for t in lo..hi { - for octet in 0..num_ntts / 8 { - self.stage.fill(stage, t * BATCH, octet); + for (lane0, lanes) in lane_groups(num_ntts) { + let width = BATCH * lanes; + let (stage, scratch) = (&mut stage[..rows * width], &mut scratch[..rows * width]); + self.stage.fill(stage, t * BATCH, lane0, lanes); for (coset, run, live) in &self.cosets { let live_rows = Some(&live[..]); - ntt.run_layers( - scratch, - Some(stage), - deep_start, - BATCH * 8, - start, - start, - *coset, - live_rows, - ); + ntt.run_layers(scratch, Some(stage), deep_start, width, start, start, *coset, live_rows); for (pos, &i) in run.clone().zip(live) { - for (s, line) in scratch[i * BATCH * 8..][..BATCH * 8] - .as_chunks::<8>() - .0 - .iter() - .enumerate() - { + for (s, lanes_of_s) in scratch[i * width..][..width].chunks_exact(lanes).enumerate() { // SAFETY: this task alone owns row `t * BATCH + s` of every sub-block, // and `built` outlives the tasks. let dst = - unsafe { out.slice(pos * sub_len + (t * BATCH + s) * num_ntts + 8 * octet, 8) }; - stream.copy(dst, line); + unsafe { out.slice(pos * sub_len + (t * BATCH + s) * num_ntts + lane0, lanes) }; + stream.copy(dst, lanes_of_s); } } } diff --git a/crates/experimental_prover/pcs/src/ntt/plan.rs b/crates/experimental_prover/pcs/src/ntt/plan.rs index 569f980dbc..d48871d360 100644 --- a/crates/experimental_prover/pcs/src/ntt/plan.rs +++ b/crates/experimental_prover/pcs/src/ntt/plan.rs @@ -34,11 +34,11 @@ impl Plan { } } - /// Whether the first gathered pass can read a lane-major message itself: it has a layer to run, whole cache - /// lines in a row, and enough residues to batch. + /// Whether the first gathered pass can read a lane-major message itself: it has a layer to run, whole quads of + /// lanes in a row, and enough residues to batch. pub(super) fn reads_lanes(&self) -> bool { let g = (self.deep_start - self.start).min(self.max_g); - self.start < self.deep_start && self.num_ntts.is_multiple_of(8) && self.log_d - self.start - g >= LOG_BATCH + self.start < self.deep_start && self.num_ntts.is_multiple_of(4) && self.log_d - self.start - g >= LOG_BATCH } /// Whether an opening can rebuild a row from a lane-major message alone: one gathered pass reads it and diff --git a/crates/experimental_prover/pcs/src/ntt/stage.rs b/crates/experimental_prover/pcs/src/ntt/stage.rs index d6e2312605..ff27220a80 100644 --- a/crates/experimental_prover/pcs/src/ntt/stage.rs +++ b/crates/experimental_prover/pcs/src/ntt/stage.rs @@ -19,7 +19,7 @@ use super::window::STREAM_MIN_WORDS; /// - A row is `row_words >= n` words: the first `row_words - n` lanes are zero padding. /// - Codeword lane `t` past the padding takes message block `row_words - 1 - t`. /// - Any lane count works: a commitment encodes only the lanes that carry data, and a caller pads -/// the row to the vector width. +/// the row to a multiple of [`ROW_LANES`](super::ROW_LANES). /// /// # Why descending /// @@ -83,10 +83,19 @@ pub(super) fn transpose_lane_major(out: &mut [F64], msg: &[F64], n_lanes: usize, pub(super) const LOG_BATCH: usize = 3; pub(super) const BATCH: usize = 1 << LOG_BATCH; +/// The lane groups a stage moves across a row of `row_words` lanes, as `(first lane, lanes)`: whole octets, then a +/// quad where the row ends four lanes past one. +pub(super) fn lane_groups(row_words: usize) -> impl Iterator { + assert!(row_words.is_multiple_of(4), "a staged row is whole quads of lanes"); + (0..row_words) + .step_by(8) + .map(move |lane| (lane, (row_words - lane).min(8))) +} + /// The transpose of a lane-major message through a stage, for one gathered pass. /// /// - It owns the message and the geometry of the fill: the step, the message rows and the row width. -/// - [`Stage::fill`] moves one octet of lanes of a batch of residues into a stage buffer. +/// - [`Stage::fill`] moves one group of lanes of a batch of residues into a stage buffer. /// - A task fills the stage, then each block of the pass transforms a copy of it. pub(super) struct Stage<'a> { lanes: Lanes<'a>, @@ -106,17 +115,27 @@ impl<'a> Stage<'a> { } } - /// Transpose one octet of lanes of a batch of [`BATCH`] consecutive residues from `t0` into `stage`. + /// Transpose lanes `lane0..lane0 + lanes`, a group of [`lane_groups`], of a batch of [`BATCH`] consecutive + /// residues from `t0` into `stage`. /// /// - Message row `j` is `msg_rows` words apart per lane block, and word `w` of a row of `row_words` words /// is lane block `row_words - 1 - w` of the message, or zero while `w` is below the padding. /// - Residue `t0 + s` takes rows `t0 + s + i * step` for `i < rows`, `step = 2^log_step`. - /// - The octet is lanes `8 * octet..8 * octet + 8`. Together with the residues, they are 64 lanes of one transform. - /// - `stage` has `rows` rows of 64 words. - /// Lane `8 * octet + k` of row `t0 + s + i * step` is word `(i * BATCH + s) * 8 + k`. - /// - One tile is 8 words of 8 lane blocks, so each 64 byte line of a lane block is read once, whole. + /// - Together with the residues, the group's lanes are `BATCH * lanes` lanes of one transform. + /// - `stage` has `rows` rows of `BATCH * lanes` words. + /// Lane `lane0 + k` of row `t0 + s + i * step` is word `(i * BATCH + s) * lanes + k`. + /// - One tile is 8 words of each lane block of the group, so each 64 byte line of a lane block is read once, whole. #[inline] - pub(super) fn fill(&self, stage: &mut [F64], t0: usize, octet: usize) { + pub(super) fn fill(&self, stage: &mut [F64], t0: usize, lane0: usize, lanes: usize) { + match lanes { + 8 => self.fill_group::<8>(stage, t0, lane0), + 4 => self.fill_group::<4>(stage, t0, lane0), + _ => unreachable!("a stage moves an octet or a quad of lanes"), + } + } + + #[inline(always)] + fn fill_group(&self, stage: &mut [F64], t0: usize, lane0: usize) { let Self { lanes: Lanes { words: msg, n_lanes }, log_step, @@ -126,22 +145,23 @@ impl<'a> Stage<'a> { // SAFETY: AVX-512F is enabled at compile time. #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] unsafe { - fill_stage_tiles(stage, msg, t0, log_step, msg_rows, n_lanes, row_words, octet) + fill_stage_tiles::(stage, msg, t0, log_step, msg_rows, n_lanes, row_words, lane0) } // SAFETY: AVX2 is enabled at compile time. #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] unsafe { - fill_stage_tiles(stage, msg, t0, log_step, msg_rows, n_lanes, row_words, octet) + fill_stage_tiles::(stage, msg, t0, log_step, msg_rows, n_lanes, row_words, lane0) } #[cfg(not(any( all(target_arch = "x86_64", target_feature = "avx512f"), all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")) )))] - fill_stage_tiles(stage, msg, t0, log_step, msg_rows, n_lanes, row_words, octet); + fill_stage_tiles::(stage, msg, t0, log_step, msg_rows, n_lanes, row_words, lane0); } } -/// [`Stage::fill`], a target-feature function where AVX-512 or AVX2 is enabled, so its tiles move through registers. +/// [`Stage::fill`] of `W` lanes, a target-feature function where AVX-512 or AVX2 is enabled, so its tiles move +/// through registers. #[cfg_attr( all(target_arch = "x86_64", target_feature = "avx512f"), target_feature(enable = "avx512f"), @@ -152,7 +172,7 @@ impl<'a> Stage<'a> { target_feature(enable = "avx2"), inline )] -fn fill_stage_tiles( +fn fill_stage_tiles( stage: &mut [F64], msg: &[F64], t0: usize, @@ -160,28 +180,32 @@ fn fill_stage_tiles( msg_rows: usize, n_lanes: usize, row_words: usize, - octet: usize, + lane0: usize, ) { - let rows = stage.len() / (BATCH * 8); + let rows = stage.len() / (BATCH * W); let pad = row_words - n_lanes; - // Lane block of line `k` of the tile, which is lane `8 * octet + k`. - let block = |k: usize| row_words - 1 - (8 * octet + k); + // Lane block of line `k` of the tile, which is lane `lane0 + k`. + let block = |k: usize| row_words - 1 - (lane0 + k); // The lines of a tile that are padding lanes. - let pad_lines = (0..8).filter(|k| 8 * octet + k < pad).fold(0u8, |m, k| m | 1 << k); + let pad_lines = (0..W).filter(|k| lane0 + k < pad).fold(0u8, |m, k| m | 1 << k); for i in 0..rows { // Line `k` of the tile: 8 residues of one lane block, or zeros for a padding lane. - let src: [usize; 8] = std::array::from_fn(|k| block(k) * msg_rows + t0 + (i << log_step)); - fill_tile(stage, i * BATCH, msg, src, pad_lines); + let src: [usize; W] = std::array::from_fn(|k| block(k) * msg_rows + t0 + (i << log_step)); + fill_tile::(stage, i * BATCH, msg, src, pad_lines); } } -/// Write `stage[tile + s][k] = msg[src[k] + s]` for the 8 x 8 words of eight lines, a line in `zeros` being zeros. +/// Write `stage[(tile + s) * W + k] = msg[src[k] + s]` for the 8 x `W` words of `W` lines, a line in `zeros` being +/// zeros. #[cfg(all(target_arch = "x86_64", target_feature = "avx512f"))] #[inline] #[target_feature(enable = "avx512f")] -fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; 8], zeros: u8) { +fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; W], zeros: u8) { use core::arch::x86_64::*; use primitives::simd::{load, store, transpose8}; + if W != 8 { + return fill_tile_ymm::(stage, tile, msg, src, zeros); + } let r: [__m512i; 8] = std::array::from_fn(|k| { if zeros >> k & 1 == 1 { _mm512_setzero_si512() @@ -195,24 +219,27 @@ fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; 8], zeros } } -/// [`fill_tile`] as two 4x4 transposes of YMM halves per half of the lines. #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")))] +use fill_tile_ymm as fill_tile; + +/// [`fill_tile`] as one 4x4 transpose of YMM halves per half of each quad of lines. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] #[inline] #[target_feature(enable = "avx2")] -fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; 8], zeros: u8) { +fn fill_tile_ymm(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; W], zeros: u8) { use core::arch::x86_64::*; use primitives::{load_at, store_at}; // Line `k`, half `h`: words `4h..4h + 4` of the line's 8 residues. - let r: [[__m256i; 2]; 8] = std::array::from_fn(|k| { + let r: [[__m256i; 2]; W] = std::array::from_fn(|k| { if zeros >> k & 1 == 1 { [_mm256_setzero_si256(); 2] } else { [load_at(msg, src[k]), load_at(msg, src[k] + 4)] } }); - // Residue `s` is word `s` of every line: lines 0..4 in its first half, 4..8 in its second. + // Residue `s` is word `s` of every line: lines `4q..4q + 4` at words `4q..4q + 4` of its row. for h in 0..2 { - for q in 0..2 { + for q in 0..W / 4 { let t = |k: usize| r[4 * q + k][h]; let (a0, a1) = (_mm256_unpacklo_epi64(t(0), t(1)), _mm256_unpackhi_epi64(t(0), t(1))); let (a2, a3) = (_mm256_unpacklo_epi64(t(2), t(3)), _mm256_unpackhi_epi64(t(2), t(3))); @@ -223,7 +250,7 @@ fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; 8], zeros _mm256_permute2x128_si256::<0x31>(a1, a3), ]; for (i, col) in cols.into_iter().enumerate() { - store_at(stage, (tile + 4 * h + i) * 8 + 4 * q, col); + store_at(stage, (tile + 4 * h + i) * W + 4 * q, col); } } } @@ -235,10 +262,10 @@ fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; 8], zeros all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "avx512f")) )))] #[inline(always)] -fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; 8], zeros: u8) { +fn fill_tile(stage: &mut [F64], tile: usize, msg: &[F64], src: [usize; W], zeros: u8) { for s in 0..8 { - for k in 0..8 { - stage[(tile + s) * 8 + k] = if zeros >> k & 1 == 1 { + for k in 0..W { + stage[(tile + s) * W + k] = if zeros >> k & 1 == 1 { F64::ZERO } else { msg[src[k] + s] diff --git a/crates/experimental_prover/pcs/src/ntt/tests.rs b/crates/experimental_prover/pcs/src/ntt/tests.rs index 78a6652151..4a2dc50826 100644 --- a/crates/experimental_prover/pcs/src/ntt/tests.rs +++ b/crates/experimental_prover/pcs/src/ntt/tests.rs @@ -201,7 +201,8 @@ fn fused_encode_matches_replicate_then_transform() { /// that lane's contiguous message block, which is what makes a commitment over /// `n_lanes` lanes equal to the `2^log_batch_size`-lane one with a zero tail. /// The shapes cover the transposing fused first pass, its fallback, and lane -/// counts that are not powers of two (the padding-free commit's whole point). +/// counts that are not powers of two (the padding-free commit's whole point), +/// in rows padded to four lanes. #[test] fn lane_major_msg_encode_matches_per_lane_reference() { let mut rng = Rng::new(0x1A2E); @@ -211,6 +212,7 @@ fn lane_major_msg_encode_matches_per_lane_reference() { (5, 2, 7), (9, 1, 5), (12, 2, 37), + (12, 2, 43), (14, 1, 64), ] { let log_d = log_rows + log_inv_rate; @@ -218,7 +220,7 @@ fn lane_major_msg_encode_matches_per_lane_reference() { let rows = 1usize << log_rows; let msg: Vec = (0..rows * n_lanes).map(|_| F64(rng.next_u64())).collect(); - let row_words = n_lanes.next_multiple_of(8); + let row_words = n_lanes.next_multiple_of(4); let mut got = vec![F64::ZERO; (row_words << log_rows) << log_inv_rate]; transpose_lane_major(&mut got[..row_words << log_rows], &msg, n_lanes, row_words, log_rows); ntt.encode_interleaved_in_place(&mut got, row_words, log_inv_rate); @@ -246,22 +248,23 @@ fn lane_major_msg_encode_matches_per_lane_reference() { /// The lane-major encode reads the message in its first gathered pass, so it must equal the /// transpose into block 0 followed by the in-place encode: rows handed over and rows opened. -/// The shapes cover a fused first pass (with and without padding lanes, at both rates), -/// a deep-only plan, and a domain too small for any gathered pass. +/// The shapes cover a fused first pass (with and without padding lanes, at both rates, in rows of whole octets +/// and in rows ending in a quad), a deep-only plan, and a domain too small for any gathered pass. #[test] fn lane_major_encode_matches_transpose_then_in_place() { let mut rng = Rng::new(0x7A5E); - for (log_rows, log_inv_rate, n_lanes) in [ - (13usize, 2usize, 47usize), - (14, 1, 40), - (13, 1, 17), - (12, 2, 64), - (5, 2, 7), + for (log_rows, log_inv_rate, n_lanes, row_words) in [ + (13usize, 2usize, 47usize, 48usize), + (14, 1, 40, 40), + (13, 1, 17, 24), + (13, 1, 17, 20), + (15, 1, 17, 20), + (12, 2, 64, 64), + (5, 2, 7, 8), ] { let log_d = log_rows + log_inv_rate; let ntt = AdditiveNttF64::standard(log_d); let msg: Vec = (0..n_lanes << log_rows).map(|_| F64(rng.next_u64())).collect(); - let row_words = n_lanes.next_multiple_of(8); let mut want = vec![F64::ZERO; (row_words << log_rows) << log_inv_rate]; transpose_lane_major(&mut want[..row_words << log_rows], &msg, n_lanes, row_words, log_rows); ntt.encode_interleaved_in_place(&mut want, row_words, log_inv_rate); @@ -272,8 +275,9 @@ fn lane_major_encode_matches_transpose_then_in_place() { let codeword = ntt.encode_lane_major(&msg, n_lanes, row_words, log_inv_rate, keep, &on_rows); drop(on_rows); let keep_nothing = keep == Keep::Nothing; - let context = - format!("log_rows={log_rows}, rate={log_inv_rate}, n_lanes={n_lanes}, keep_nothing={keep_nothing}"); + let context = format!( + "log_rows={log_rows}, rate={log_inv_rate}, n_lanes={n_lanes}, row_words={row_words}, keep_nothing={keep_nothing}" + ); assert_eq!(handed, want, "handed-over rows, {context}"); let mut positions: Vec = (0..40).map(|_| rng.next_u64() as usize % (1 << log_d)).collect(); positions.sort_unstable(); diff --git a/crates/experimental_prover/pcs/src/whir.rs b/crates/experimental_prover/pcs/src/whir.rs index b209663e63..66844d663a 100644 --- a/crates/experimental_prover/pcs/src/whir.rs +++ b/crates/experimental_prover/pcs/src/whir.rs @@ -23,7 +23,7 @@ //! `E`; the committed alphabet remains `K = GF(2^64)`. use crate::merkle; -use crate::ntt::{AdditiveNttF64, Codeword, Keep}; +use crate::ntt::{AdditiveNttF64, Codeword, Keep, ROW_LANES}; use crate::stack_open::Terms; use fiat_shamir::merkle::{Hash, MerklePath}; use primitives::field::{Acc8, F64, F128, F128Unreduced, F128x8, fold_pairs8_ext, msg_terms8_ext, powers}; @@ -88,7 +88,7 @@ pub struct Commitment { pub struct ProverData { ntt: AdditiveNttF64, codeword: Codeword, - /// Words per codeword row: the committed lanes, with zero lanes in front up to the vector width. + /// Words per codeword row: the committed lanes, with zero lanes in front up to a multiple of `ROW_LANES`. row_words: usize, pub(crate) merkle_tree: merkle::CutTree, } @@ -116,9 +116,8 @@ const KEEP_CODEWORD_MAX_BYTES: usize = 1 << 30; /// state every leaf shares. At most the image's tail, the committed lanes, /// rides the proof, the opening saying its width and the image's. /// -/// The encoded row is padded to a whole AVX-512 vector of lanes with zero lanes -/// in front, so the NTT has no masked tail. The padding is the image's zeros and -/// never rides the proof. +/// The encoded row is padded to a whole number of the NTT's vectors (`ROW_LANES`) with zero lanes in front, so the +/// NTT has no vector of padding lanes. The padding is the image's zeros and never rides the proof. pub fn commit(message: &[F64], log_n: usize, log_batch_size: usize, log_inv_rate: usize) -> (Commitment, ProverData) { assert!(log_inv_rate >= 1, "log_inv_rate must be >= 1 for a non-trivial RS code"); assert!(log_n > log_batch_size, "witness must be wider than the interleaving"); @@ -131,23 +130,29 @@ pub fn commit(message: &[F64], log_n: usize, log_batch_size: usize, log_inv_rate ); let k_code = log_rows + log_inv_rate; let n_positions = 1usize << k_code; - let row_words = n_lanes.next_multiple_of(8).min(1usize << log_batch_size); + let row_words = n_lanes.next_multiple_of(ROW_LANES).min(1usize << log_batch_size); // Leaves are hashed as the encode finishes each block of rows. let tree = merkle::MerkleBuilder::new(n_positions, row_words, 1usize << log_batch_size); let ntt = AdditiveNttF64::standard(k_code); - let codeword = - tracing::info_span!("NTT", kind = "base encode", log_domain = k_code, lanes = row_words).in_scope(|| { - let on_rows = |row, rows: &[F64]| tree.absorb(row, rows); - let keep = if !primitives::REBUILD_OVER_STORE { - Keep::Whole - } else if n_positions * row_words * size_of::() > KEEP_CODEWORD_MAX_BYTES { - Keep::Nothing - } else { - Keep::BeforeDeep - }; - ntt.encode_lane_major(message, n_lanes, row_words, log_inv_rate, keep, &on_rows) - }); + let span = tracing::info_span!( + "NTT", + kind = "base encode", + log_domain = k_code, + lanes = row_words, + data = n_lanes + ); + let codeword = span.in_scope(|| { + let on_rows = |row, rows: &[F64]| tree.absorb(row, rows); + let keep = if !primitives::REBUILD_OVER_STORE { + Keep::Whole + } else if n_positions * row_words * size_of::() > KEEP_CODEWORD_MAX_BYTES { + Keep::Nothing + } else { + Keep::BeforeDeep + }; + ntt.encode_lane_major(message, n_lanes, row_words, log_inv_rate, keep, &on_rows) + }); let merkle_tree = tracing::info_span!("Merkle").in_scope(|| tree.finish()); let root = merkle_tree.root(); From 08209e93a212e99d974978ee07da7e5d7e6aaaa1 Mon Sep 17 00:00:00 2001 From: Kevaundray Wedderburn Date: Wed, 18 Nov 2026 01:30:46 +0000 Subject: [PATCH 18/18] experimental pcs: hash the L0 leaves from their data lanes A base codeword row leads with zero lanes up to the NTT's lane quantum, and the leaf hasher counted only the leaf's own padding as the shared zero-prefix state. It now takes the data lanes (`MerkleBuilder::new` and `CutTree::open` gain `data_words`), so the whole zero prefix, the row's padding included, is one sponge state, and a leaf takes one Keccak permutation fewer whenever the padding crosses a rate block. The hashed bytes are unchanged; the extension commits pass their full row. Single thread, EPYC 9454P core 44, `aggregate --xmss 150` (46 data lanes in rows of 48, leaf images 512 -> 376 bytes), two A/B rounds back to back against the previous commit (Commit/NTT is the last pass's span; total the three-pass mean): | tier | Commit/NTT ms | total s | | gfni | 176 -> 152 | 0.849, 0.835 -> 0.813, 0.801 | | hsw | 196 -> 174 | 1.376, 1.371 -> 1.360, 1.352 | | native (AVX-512) | 112 -> 98.4 | 0.609, 0.611 -> 0.596, 0.594 | At `--xmss 200` (42 data lanes in rows of 44) both hash the same image; the totals matched within spread (gfni 1.253 both, hsw 2.091 -> 2.026 at 6% spread, native 0.935 -> 0.942 at 6% spread). --- crates/experimental_prover/pcs/src/merkle.rs | 129 ++++++++++++------- crates/experimental_prover/pcs/src/whir.rs | 13 +- 2 files changed, 91 insertions(+), 51 deletions(-) diff --git a/crates/experimental_prover/pcs/src/merkle.rs b/crates/experimental_prover/pcs/src/merkle.rs index 9af7b2ee28..2108ec0e47 100644 --- a/crates/experimental_prover/pcs/src/merkle.rs +++ b/crates/experimental_prover/pcs/src/merkle.rs @@ -38,7 +38,7 @@ const STAGE_TILE_BYTES: usize = 16 << 10; /// A Merkle tree filled one aligned block of leaves at a time, from any thread. /// -/// Leaf `i` is `zeros(leaf_words - row_words) ‖ row_i`. +/// Leaf `i` is `zeros(leaf_words - row_words) ‖ row_i`, and a row may lead with zero words of its own. /// /// Blocks must all have the same power-of-two size, and together cover every leaf once. pub(crate) struct MerkleBuilder { @@ -50,20 +50,21 @@ pub(crate) struct MerkleBuilder { } impl MerkleBuilder { - /// An empty tree of `num_leaves` leaves of `leaf_words`, each committing `row_words`. + /// An empty tree of `num_leaves` leaves of `leaf_words`, each committing `row_words`, of which all but the last + /// `data_words` are zero. /// /// A tree of `CUT_FROM` leaves or more is cut, so its blocks must hold whole leaf groups. - pub fn new(num_leaves: usize, row_words: usize, leaf_words: usize) -> Self { + pub fn new(num_leaves: usize, row_words: usize, data_words: usize, leaf_words: usize) -> Self { let cut = if num_leaves >= CUT_FROM { CUT } else { 0 }; - Self::with_bytes(num_leaves, 8 * row_words, 8 * leaf_words, cut) + Self::with_bytes(num_leaves, 8 * row_words, 8 * data_words, 8 * leaf_words, cut) } - fn with_bytes(num_leaves: usize, row_bytes: usize, leaf_bytes: usize, cut: usize) -> Self { + fn with_bytes(num_leaves: usize, row_bytes: usize, data_bytes: usize, leaf_bytes: usize, cut: usize) -> Self { assert!(num_leaves.is_power_of_two(), "num_leaves must be power of 2"); Self { nodes: Nodes::new(num_leaves, cut), cut, - leaves: LeafHasher::new(row_bytes, leaf_bytes), + leaves: LeafHasher::new(row_bytes, data_bytes, leaf_bytes), progress: Progress::new(num_leaves), } } @@ -341,12 +342,13 @@ impl CutTree { /// The paths of `positions`, with `row(q)` the row at `q`. /// - /// `group_rows` are the full rows of [`Self::group_positions`], `row_words` each, back to back, hashed - /// as the leaves `zeros(leaf_words - row_words) ‖ row`. + /// `group_rows` are the full rows of [`Self::group_positions`], `row_words` each, all but the last `data_words` + /// of each zero, back to back, hashed as the leaves `zeros(leaf_words - row_words) ‖ row`. pub fn open<'a>( &self, positions: &[usize], row_words: usize, + data_words: usize, leaf_words: usize, group_rows: &[F64], row: impl Fn(usize) -> &'a [F64], @@ -362,7 +364,8 @@ impl CutTree { let mut nodes = Nodes::new(n, 0); // SAFETY: the nodes are ours alone, the climb writes every level it reads, and the levels kept are written. unsafe { - LeafHasher::new(8 * row_words, 8 * leaf_words).hash(cast_slice(group_rows), nodes.level_mut(0, 0, n)); + LeafHasher::new(8 * row_words, 8 * data_words, 8 * leaf_words) + .hash(cast_slice(group_rows), nodes.level_mut(0, 0, n)); nodes.climb(0, 0, n, self.cut); nodes.tree.truncate(Self::start(n, self.cut)); nodes.assume_init() @@ -385,14 +388,15 @@ impl CutTree { /// How rows become leaf digests, chosen once from the leaf shape. /// -/// Whole blocks of leading zeros are one sponge state, absorbed once for every leaf. +/// Whole blocks of leading zeros, the row's own included, are one sponge state, absorbed once for every leaf. enum LeafHasher { /// Past the shared prefix, each row is its own image. Direct { row_bytes: usize, state: [u64; STATE_LANES], }, - /// The rest of the padding does not fill a block, so rows are zero-extended in a tile first. + /// The rest of the padding does not fill a block, or the row leads with more zeros than the image, so each row's + /// image is laid out in a tile first. Staged { row_bytes: usize, image: usize, @@ -401,9 +405,11 @@ enum LeafHasher { } impl LeafHasher { - fn new(row_bytes: usize, leaf_bytes: usize) -> Self { - assert!(0 < row_bytes && row_bytes <= leaf_bytes, "a leaf holds its row"); - let zero_blocks = (leaf_bytes - row_bytes) / RATE; + /// Rows of `row_bytes`, zero but for their last `data_bytes`, hashed as leaves of `leaf_bytes`. + fn new(row_bytes: usize, data_bytes: usize, leaf_bytes: usize) -> Self { + assert!(0 < data_bytes && data_bytes <= row_bytes, "a row holds its data"); + assert!(row_bytes <= leaf_bytes, "a leaf holds its row"); + let zero_blocks = (leaf_bytes - data_bytes) / RATE; let state = zero_prefix_state(zero_blocks); let image = leaf_bytes - zero_blocks * RATE; if image == row_bytes { @@ -460,14 +466,16 @@ impl Tile { Self([0; STAGE_TILE_BYTES / 8]) } - /// Lay each `row_bytes` row at the end of its own `image`-byte slot, and return the slots. + /// Lay each `row_bytes` row, cut to its last `image` bytes when longer (the rest is zeros), at the end of its own + /// `image`-byte slot, and return the slots. /// - /// The bytes before each row are never written, so they stay zero. + /// A slot's bytes before its row are never written, so they stay zero. fn extend(&mut self, rows: &[u8], row_bytes: usize, image: usize) -> &[u8] { let tile: &mut [u8] = cast_slice_mut(&mut self.0); let n = rows.len() / row_bytes; + let take = row_bytes.min(image); for (slot, row) in tile.chunks_exact_mut(image).zip(rows.chunks_exact(row_bytes)) { - slot[image - row_bytes..].copy_from_slice(row); + slot[image - take..].copy_from_slice(&row[row_bytes - take..]); } &tile[..n * image] } @@ -486,15 +494,22 @@ mod tests { /// The tree over `num_leaves` equal byte leaves, each hashed as [`primitives::hash::hash`] of its bytes. fn merkle_tree(data: &[u8], num_leaves: usize) -> Vec { let leaf_bytes = data.len() / num_leaves; - let builder = MerkleBuilder::with_bytes(num_leaves, leaf_bytes, leaf_bytes, 0); + let builder = MerkleBuilder::with_bytes(num_leaves, leaf_bytes, leaf_bytes, leaf_bytes, 0); builder.absorb_all(data); builder.finish().nodes } - /// The tree over `num_leaves` rows of `row_words`, each hashed as `zeros(leaf_words - row_words) ‖ row`. - fn merkle_tree_padded_rows(data: &[F64], num_leaves: usize, row_words: usize, leaf_words: usize) -> Vec { + /// The tree over `num_leaves` rows of `row_words`, all but the last `data_words` of each zero, each hashed as + /// `zeros(leaf_words - row_words) ‖ row`. + fn merkle_tree_padded_rows( + data: &[F64], + num_leaves: usize, + row_words: usize, + data_words: usize, + leaf_words: usize, + ) -> Vec { assert_eq!(data.len(), row_words * num_leaves); - let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * leaf_words, 0); + let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * data_words, 8 * leaf_words, 0); builder.absorb_all(cast_slice(data)); builder.finish().nodes } @@ -549,26 +564,37 @@ mod tests { #[test] fn padded_rows_tree_matches_full_width() { // Invariant: each leaf hashes as its zero-extended row, however the prefix splits. - for (num_leaves, row_words, leaf_words) in [ + for (num_leaves, row_words, data_words, leaf_words) in [ // Nothing to pad. - (8usize, 64usize, 64usize), - (1024, 8, 8), + (8usize, 64usize, 64usize, 64usize), + (1024, 8, 8, 8), // A shared prefix plus a partial block, staged. - (2048, 37, 64), - (256, 27, 64), - (64, 1, 64), + (2048, 37, 37, 64), + (256, 27, 27, 64), + (64, 1, 1, 64), // A prefix of whole blocks: the rows are hashed where they lie. - (256, 30, 64), - (1 << 14, 47, 64), + (256, 30, 30, 64), + (1 << 14, 47, 47, 64), // A prefix shorter than one block: nothing shared, still staged. - (32, 5, 8), - (256, 61, 64), + (32, 5, 5, 8), + (256, 61, 61, 64), // Leaves that are not a whole number of blocks. - (16, 3, 5), - (64, 7, 40), + (16, 3, 3, 5), + (64, 7, 7, 40), + // Rows that lead with zeros: a shared prefix that reaches into them, or stops short of them. + (1 << 10, 48, 46, 64), + (256, 32, 30, 64), + (256, 44, 42, 64), + (64, 8, 5, 8), ] { let data: Vec = (0..row_words * num_leaves) - .map(|i| F64(i.wrapping_mul(0x9E37_79B9_7F4A_7C15) as u64 | 1)) + .map(|i| { + if i % row_words < row_words - data_words { + F64::ZERO + } else { + F64(i.wrapping_mul(0x9E37_79B9_7F4A_7C15) as u64 | 1) + } + }) .collect(); let want = merkle_tree_sequential( data.chunks(row_words).map(|row| { @@ -579,9 +605,9 @@ mod tests { num_leaves, ); assert_eq!( - &merkle_tree_padded_rows(&data, num_leaves, row_words, leaf_words)[..], + &merkle_tree_padded_rows(&data, num_leaves, row_words, data_words, leaf_words)[..], &want[..], - "num_leaves={num_leaves} row_words={row_words} leaf_words={leaf_words}" + "num_leaves={num_leaves} row_words={row_words} data_words={data_words} leaf_words={leaf_words}" ); } } @@ -589,13 +615,24 @@ mod tests { #[test] fn cut_tree_opens_like_the_full_tree() { // Invariant: opening through the cut tree gives the paths of the full tree. - for (num_leaves, row_words, leaf_words) in [(256usize, 8usize, 16usize), (1 << 10, 24, 24), (32, 8, 8)] { + for (num_leaves, row_words, data_words, leaf_words) in [ + (256usize, 8usize, 8usize, 16usize), + (1 << 10, 24, 24, 24), + (32, 8, 8, 8), + (256, 48, 45, 64), + ] { let data: Vec = (0..row_words * num_leaves) - .map(|i| F64(i as u64 * 0x9E37_79B9 + 1)) + .map(|i| { + if i % row_words < row_words - data_words { + F64::ZERO + } else { + F64(i as u64 * 0x9E37_79B9 + 1) + } + }) .collect(); - let full = merkle_tree_padded_rows(&data, num_leaves, row_words, leaf_words); + let full = merkle_tree_padded_rows(&data, num_leaves, row_words, data_words, leaf_words); let row = |q: usize| &data[q * row_words..][..row_words]; - let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * leaf_words, CUT); + let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * data_words, 8 * leaf_words, CUT); builder.absorb_all(cast_slice(&data)); let cut = builder.finish(); assert_eq!(cut.root(), *full.last().unwrap()); @@ -606,7 +643,7 @@ mod tests { ] { let group = cut.group_positions(&positions); let rows: Vec = group.iter().flat_map(|&q| row(q)).copied().collect(); - let got = cut.open(&positions, row_words, leaf_words, &rows, row); + let got = cut.open(&positions, row_words, data_words, leaf_words, &rows, row); let level_start = |level: usize| 2 * num_leaves - ((2 * num_leaves) >> level); let node = |level: usize, i: usize| full[level_start(level) + i]; let row = |q| row(q).iter().map(|word| word.0); @@ -623,9 +660,9 @@ mod tests { // Invariant: the tree does not depend on the block size the encode picks. let (num_leaves, row_words) = (1usize << 12, 6usize); let data: Vec = (0..row_words * num_leaves).map(|i| F64(i as u64 * 7 + 1)).collect(); - let want = merkle_tree_padded_rows(&data, num_leaves, row_words, row_words); + let want = merkle_tree_padded_rows(&data, num_leaves, row_words, row_words, row_words); for log_block in [0usize, 3, 5, 6, 9, 12] { - let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * row_words, 0); + let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * row_words, 8 * row_words, 0); // Blocks arrive out of order, as the encode's tasks finish. for b in (0..num_leaves >> log_block).rev() { let rows = row_words << log_block; @@ -635,7 +672,7 @@ mod tests { } // A cut tree is the full tree without its lowest levels, whatever the block size. for log_block in [4usize, 5, 9, 12] { - let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * row_words, CUT); + let builder = MerkleBuilder::with_bytes(num_leaves, 8 * row_words, 8 * row_words, 8 * row_words, CUT); for b in (0..num_leaves >> log_block).rev() { let rows = row_words << log_block; builder.absorb(b << log_block, &data[b * rows..][..rows]); @@ -652,7 +689,7 @@ mod tests { #[should_panic(expected = "every leaf absorbed")] fn a_missing_block_is_rejected() { let data = vec![F64::ZERO; 8 * 16]; - let builder = MerkleBuilder::new(16, 8, 8); + let builder = MerkleBuilder::new(16, 8, 8, 8); builder.absorb(0, &data[..64]); let _ = builder.finish(); } @@ -661,7 +698,7 @@ mod tests { #[should_panic(expected = "a block absorbed twice")] fn a_repeated_block_is_rejected() { let data = vec![F64::ZERO; 8 * 16]; - let builder = MerkleBuilder::new(16, 8, 8); + let builder = MerkleBuilder::new(16, 8, 8, 8); builder.absorb(8, &data[..64]); builder.absorb(8, &data[..64]); } diff --git a/crates/experimental_prover/pcs/src/whir.rs b/crates/experimental_prover/pcs/src/whir.rs index 66844d663a..5f6217f4bb 100644 --- a/crates/experimental_prover/pcs/src/whir.rs +++ b/crates/experimental_prover/pcs/src/whir.rs @@ -132,8 +132,8 @@ pub fn commit(message: &[F64], log_n: usize, log_batch_size: usize, log_inv_rate let n_positions = 1usize << k_code; let row_words = n_lanes.next_multiple_of(ROW_LANES).min(1usize << log_batch_size); - // Leaves are hashed as the encode finishes each block of rows. - let tree = merkle::MerkleBuilder::new(n_positions, row_words, 1usize << log_batch_size); + // Leaves are hashed as the encode finishes each block of rows, from their data lanes on. + let tree = merkle::MerkleBuilder::new(n_positions, row_words, n_lanes, 1usize << log_batch_size); let ntt = AdditiveNttF64::standard(k_code); let span = tracing::info_span!( "NTT", @@ -197,7 +197,9 @@ impl LigeroWitness { .iter() .flat_map(|&q| row(q).as_chunks::<2>().0.iter().map(|w| F128::new(w[0].0, w[1].0))), ); - let paths = self.tree.open(queries, row_words, row_words, &group_rows, row); + let paths = self + .tree + .open(queries, row_words, row_words, row_words, &group_rows, row); (rows, paths) } @@ -226,7 +228,7 @@ pub(crate) fn ligero_commit_ext( // One leaf per row, its F128s as K words: hashed as the encode finishes each block. let row_words = 2 * num_interleaved; - let builder = merkle::MerkleBuilder::new(block_len, row_words, row_words); + let builder = merkle::MerkleBuilder::new(block_len, row_words, row_words, row_words); let codeword = tracing::info_span!( "NTT", kind = "extension encode", @@ -892,7 +894,8 @@ pub(crate) fn recursive_prover_with_prepared_basis( let zeros = std::iter::repeat_n(F128::ZERO, num_interleaved_0 - n_lanes); rows.extend(image(q).iter().rev().map(|&k| F128::from(k)).chain(zeros)); } - ps.hint_merkle(l0_tree.open(&queries, row_words, num_interleaved_0, &group_rows, image)); + let paths = l0_tree.open(&queries, row_words, n_lanes, num_interleaved_0, &group_rows, image); + ps.hint_merkle(paths); // Nothing reads the L0 codeword, its stack or its tree again. drop((prover_data, std::mem::take(&mut stack))); rows