Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 10 additions & 65 deletions crates/experimental_prover/keccak_gkr/src/kernels.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
//! The bit-level kernels of the layer prover, over one block position at a time: the 32 packs of
//! one bit position of one block.
//!
//! Each has a portable form, an AVX-512 one (AVX-512BW, VPCLMULQDQ, GFNI), an AVX2 one
//! (PCLMULQDQ, and VPCLMULQDQ where it is) and a NEON one (PMULL), which agree bit for bit. The
//! vector forms are compiled only where their features are. With AVX2 and GFNI a block's
//! [`skip_row`]s are summed on residues instead (`skip_gfni`).
//! Each has a portable form and, where it pays, an AVX-512 one (AVX-512BW, VPCLMULQDQ, GFNI), an
//! AVX2 one (PCLMULQDQ, and VPCLMULQDQ where it is) and a NEON one (PMULL), which agree bit for bit.
//! The vector forms are compiled only where their features are. With AVX2 and GFNI a block's
//! [`skip_row`]s are summed on residues instead (`skip_gfni`), and on x86 with AVX2 the table rows
//! come from byte maps (`table_maps`), so [`fold_groups`] keeps only its portable and NEON forms.

use primitives::field::gf2_64::mul_wide;

Expand Down Expand Up @@ -333,7 +334,10 @@ mod x86 {
use core::arch::x86_64::*;

use super::*;
pub use super::{embed_extend_portable as embed_extend, products_clmul as products, skip_row_portable as skip_row};
pub use super::{
embed_extend_portable as embed_extend, fold_groups_portable as fold_groups, products_clmul as products,
skip_row_portable as skip_row,
};

/// The matrices sending a pack's low and high byte to each byte of its embedding. An affine
/// byte map reads output bit `t` off matrix byte `7 - t`.
Expand Down Expand Up @@ -396,12 +400,6 @@ mod x86 {
}
}

/// Eight packs, a word each.
#[inline(always)]
unsafe fn load(p: *const u32) -> __m512i {
unsafe { _mm512_cvtepu32_epi64(_mm256_loadu_si256(p.cast())) }
}

/// Byte `k` of the embedding of each pack of `v`, in the low byte of the pack's 16 bits.
#[inline(always)]
unsafe fn embedding_byte(v: __m512i, k: usize) -> __m512i {
Expand All @@ -414,41 +412,6 @@ mod x86 {
)
}
}

/// The four sums are summed across their lanes together: two registers' lanes interleave into
/// one, which takes two shuffles where each register alone would leave the vector unit.
#[inline(always)]
pub fn fold_groups(e: &[u32; PACKS], weights: &[[u64; GROUP]; 2]) -> [[u128; 2]; 2] {
unsafe {
// `acc[2 * half + k]`, four lanes to sum.
let mut acc = [_mm512_setzero_si512(); 4];
for g in 0..PACKS / 8 {
let ze = load(e.as_ptr().add(8 * g));
for k in 0..2 {
let zw = _mm512_loadu_si512(weights[k].as_ptr().add(8 * (g % 2)).cast());
let sum = &mut acc[2 * (g / 2) + k];
*sum = _mm512_xor_si512(*sum, _mm512_clmulepi64_epi128(ze, zw, 0x00));
*sum = _mm512_xor_si512(*sum, _mm512_clmulepi64_epi128(ze, zw, 0x11));
}
}
// Lanes `a0 + a2, a1 + a3, b0 + b2, b1 + b3`.
let pair = |a: __m512i, b: __m512i| {
_mm512_xor_si512(
_mm512_shuffle_i64x2::<0b01_00_01_00>(a, b),
_mm512_shuffle_i64x2::<0b11_10_11_10>(a, b),
)
};
// Of two pairs, the four sums, in the order of `acc`.
let (t, u) = (pair(acc[0], acc[1]), pair(acc[2], acc[3]));
let four = _mm512_xor_si512(
_mm512_shuffle_i64x2::<0b10_00_10_00>(t, u),
_mm512_shuffle_i64x2::<0b11_01_11_01>(t, u),
);
let mut a = [0u128; 4];
_mm512_storeu_si512(a.as_mut_ptr().cast(), four);
[[a[0], a[1]], [a[2], a[3]]]
}
}
}

/// Compiled only where its features are, so its kernels are safe to call. The embedding is
Expand All @@ -468,8 +431,8 @@ mod x86 {
mod avx2 {
use core::arch::x86_64::*;

pub use super::products_clmul as products;
use super::*;
pub use super::{fold_groups_portable as fold_groups, products_clmul as products};
use crate::clmul::{V, Vector};

/// The 32 packs of a block position, embedded, four to a register.
Expand Down Expand Up @@ -614,24 +577,6 @@ mod avx2 {
}
}
}

#[inline(always)]
pub fn fold_groups(e: &[u32; PACKS], weights: &[[u64; GROUP]; 2]) -> [[u128; 2]; 2] {
std::array::from_fn(|half| {
let mut acc = [V::zero(); 2];
for q in (0..GROUP).step_by(2 * V::N) {
// SAFETY: `2 * N` packs and weights at `q`.
unsafe {
let ze = V::load_packs(e.as_ptr().add(GROUP * half + q));
for (acc, weights) in acc.iter_mut().zip(weights) {
let zw = V::load(weights.as_ptr().add(q));
*acc = acc.xor(ze.clmul::<0x00>(zw)).xor(ze.clmul::<0x11>(zw));
}
}
}
acc.map(V::sum)
})
}
}

/// Compiled only where its feature is, so its kernels are safe to call.
Expand Down
19 changes: 9 additions & 10 deletions crates/experimental_prover/keccak_gkr/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,14 +82,18 @@
//! arithmetic in the power basis of `s`, where a pack's value is its embedding's bits and a
//! weighted sum is a carry-less product (`basis`, `power`).
//!
//! `U`'s weighted bit sums are outer products: with GFNI (x86-64), one affine byte map per byte
//! of eight positions' weights applied to bit-transposed products (`products_gfni`); elsewhere,
//! the positions grouped by each byte of their weight (`prover`'s digit buckets).
//! `U`'s weighted bit sums are outer products: with AVX2 and GFNI, one affine byte map per byte of eight
//! positions' weights applied to the bit-transposed values of the products at six points of GF(2^8), which
//! the inverse evaluation map turns into coefficients once a layer (`products_gfni`); elsewhere, the
//! positions grouped by each byte of their weight (`prover`'s digit buckets).
//!
//! The four rounds' sums are binary polynomials of degree at most 173. With AVX2 and GFNI they
//! are taken on residues modulo 24 irreducibles of degree 8, where a product is a byte product
//! (`skip_gfni`).
//!
//! With AVX2, the third walk weights each byte of a block's bits by a byte map, a GFNI affine map or two nibble
//! lookups, into its field element (`table_maps`).
//!
//! The rounds that remain fold a table of one field element per half block and position. A pass over it
//! binds two rounds' variables: over each quad of the rows it writes it sums the next round and, as a
//! quadratic in that round's challenge, the round after, so the table is read once per two rounds (`prover`).
Expand All @@ -116,13 +120,8 @@ mod prover;
pub mod rmfe;
#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))]
mod skip_gfni;
#[cfg(all(
target_arch = "x86_64",
target_feature = "avx512f",
target_feature = "avx512bw",
target_feature = "gfni"
))]
mod table_gfni;
#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))]
mod table_maps;
pub mod witness;

#[cfg(test)]
Expand Down
Loading
Loading