Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
407 changes: 320 additions & 87 deletions crates/keccak_gkr/src/kernels.rs

Large diffs are not rendered by default.

125 changes: 124 additions & 1 deletion crates/keccak_gkr/src/power.rs
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,58 @@ impl PowerField {
row_kernel!(width, dot_pair(a, b, width), dot_pair_portable(a, b, width))
}

/// `sum of B[j] * b[j]` and `sum of (B[j] + B'[j]) * (b[j] + b'[j])`, unreduced, where
/// `B[j] = weights[j] * a[j] + constant[j]` and `B'` likewise of `a'`: a round's values over
/// a pair of rows. `scratch` holds two rows where the products are reduced before the sums.
pub fn round_dot(
&self,
weights: &[u64],
constant: &[u64],
a: [&[u64]; 2],
b: [&[u64]; 2],
scratch: &mut [u64],
width: usize,
) -> ([u64; 9], [u64; 9]) {
assert!(
[weights, constant]
.iter()
.chain(&a)
.chain(&b)
.all(|row| row.len() >= 3 * width)
);
cfg_select! {
all(target_arch = "x86_64", target_feature = "avx2", target_feature = "pclmulqdq") => {
if width.is_multiple_of(x86::STEP) {
let _ = scratch;
// SAFETY: the features are enabled at compile time, and every row is bounded above.
unsafe { x86::round_dot(weights, constant, a, b, width) }
} else {
self.round_dot_rows(weights, constant, a, b, scratch, width)
}
}
_ => self.round_dot_rows(weights, constant, a, b, scratch, width),
}
}

/// [`Self::round_dot`] by reducing `B` and `B'` into `scratch` first: the other arms' form,
/// and the x86 kernel's test oracle, whose sums are congruent to these.
pub fn round_dot_rows(
&self,
weights: &[u64],
constant: &[u64],
a: [&[u64]; 2],
b: [&[u64]; 2],
scratch: &mut [u64],
width: usize,
) -> ([u64; 9], [u64; 9]) {
let (b0, b1) = scratch[..6 * width].split_at_mut(3 * width);
self.mul_row(b0, weights, a[0], constant, width);
self.mul_row(b1, weights, a[1], constant, width);
let (zero, infinity) = self.dot_pair([b0, b1], b, width);
let pad = |x: [u64; 6]| std::array::from_fn(|i| if i < 6 { x[i] } else { 0 });
(pad(zero), pad(infinity))
}

/// `dst[j] = lo[j] + challenge * (lo[j] + hi[j])`.
pub fn fold_row(&self, dst: &mut [u64], lo: &[u64], hi: &[u64], challenge: &Element, width: usize) {
assert!(dst.len() >= 3 * width && lo.len() >= 3 * width && hi.len() >= 3 * width);
Expand Down Expand Up @@ -342,6 +394,62 @@ mod x86 {
(finish(&acc[0]), finish(&acc[1]))
}

/// [`PowerField::round_dot`] without reducing `B`: its six words meet `b` as two halves of
/// three, Karatsuba's twelve products where reducing and multiplying take eighteen. `B + B'`
/// is `weights * (a + a')`, the constants cancelling.
pub unsafe fn round_dot(
weights: &[u64],
constant: &[u64],
a: [&[u64]; 2],
b: [&[u64]; 2],
width: usize,
) -> ([u64; 9], [u64; 9]) {
// Per sum, the 128-bit partial sums by word offset.
let mut acc = [[V::zero(); 8]; 2];
// The six words of products given by their even and odd entries' partial sums.
let words = |p: [[V; 5]; 2]| -> [V; 6] {
let [even, odd] = p.map(|half| std::array::from_fn::<V, 6, _>(|w| word(&half, w)));
std::array::from_fn(|w| even[w].unpacklo(odd[w]))
};
for j in (0..width).step_by(STEP) {
let (w, c, a0, a1, b0, b1) = unsafe {
(
load3(weights, width, j),
load3(constant, width, j),
load3(a[0], width, j),
load3(a[1], width, j),
load3(b[0], width, j),
load3(b[1], width, j),
)
};
let sw = sums(&w);
let mut first = words(karatsuba(&w, &sw, &a0, &sums(&a0)));
for (f, c) in first.iter_mut().zip(c) {
*f = f.xor(c);
}
let da = xor3(&a0, &a1);
let delta = words(karatsuba(&w, &sw, &da, &sums(&da)));
for (acc, (f, b)) in acc.iter_mut().zip([(first, b0), (delta, xor3(&b0, &b1))]) {
for offset in [0, 3] {
let p = partial(&[f[offset], f[offset + 1], f[offset + 2]], &b);
for s in 0..5 {
acc[offset + s] = acc[offset + s].xor(p[0][s].xor(p[1][s]));
}
}
}
}
let finish = |acc: &[V; 8]| {
let mut out = [0u64; 9];
for (s, sum) in acc.iter().enumerate() {
let sum = sum.sum();
out[s] ^= sum as u64;
out[s + 1] ^= (sum >> 64) as u64;
}
out
};
(finish(&acc[0]), finish(&acc[1]))
}

pub unsafe fn reduce_row(field: &PowerField, dst: &mut [u64], wide: &[u64], width: usize) {
for j in (0..width).step_by(STEP) {
let w: [V; 4] = std::array::from_fn(|k| unsafe { V::load(wide.as_ptr().add(k * width + j)) });
Expand Down Expand Up @@ -541,7 +649,8 @@ mod tests {
#[test]
fn arithmetic_matches_the_standard_basis() {
let mut rng = Rng::new(41);
let field = PowerField::new(rng.ext()).unwrap();
let s = rng.ext();
let field = PowerField::new(s).unwrap();
let width = 16;
let elements = |rng: &mut Rng| -> (Vec<F192>, Vec<u64>) {
let values = rng.ext_vec(width);
Expand Down Expand Up @@ -590,5 +699,19 @@ mod tests {
for j in 0..width {
assert_eq!(field.to_field(&get(&fast, width, j)), a0[j] * b0[j] + b1[j]);
}

// The sums are congruent, not equal: compare their values at `s`.
let eval = PolyEval::new(s, 72);
let mut scratch = vec![0u64; 6 * width];
let (zero, infinity) = field.round_dot(&ra0, &rb0, [&ra1, &rb1], [&rb0, &ra0], &mut scratch, width);
let (zero_rows, infinity_rows) =
field.round_dot_rows(&ra0, &rb0, [&ra1, &rb1], [&rb0, &ra0], &mut scratch, width);
assert_eq!(eval.eval(&zero), eval.eval(&zero_rows));
assert_eq!(eval.eval(&infinity), eval.eval(&infinity_rows));
let b = |j: usize, a: &[F192]| a0[j] * a[j] + b0[j];
let expected: F192 = (0..width).map(|j| b(j, &a1) * b0[j]).sum();
assert_eq!(eval.eval(&zero), expected);
let expected: F192 = (0..width).map(|j| (b(j, &a1) + b(j, &b1)) * (b0[j] + a0[j])).sum();
assert_eq!(eval.eval(&infinity), expected);
}
}
60 changes: 33 additions & 27 deletions crates/keccak_gkr/src/prover.rs
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ use primitives::multilinear::{eq_table, poly_eval};

use crate::basis::{PolyEval, add_mul, carry};
use crate::keccak::{BITS, LOCAL_VARS, along, linear_transpose, row_positions};
use crate::kernels::{GROUP, MIXED, PACKS, SKIP, SkipBlock, TERNARY, embed, mixed_points, products, slot};
use crate::kernels::{GROUP, MIXED, PACKS, SKIP, SkipBlock, SkipWeights, TERNARY, embed, mixed_points, products, slot};
use crate::open::fold;
use crate::power::{Element, PowerField, get, set};
use crate::protocol::{Claim, chi_weights, mu_at};
Expand Down Expand Up @@ -193,6 +193,8 @@ struct Layer<'a> {

/// A row of a table in the power basis: three words per position, laid out by word.
const ROW: usize = 3 * BITS;
/// The scratch [`Round::add_pair`] takes: four shifted rows, and two for `round_dot`.
const PAIR_SCRATCH: usize = 6 * ROW;

impl Layer<'_> {
/// The blocks a segment's tasks walk: those written, then the padding block once, standing
Expand Down Expand Up @@ -349,7 +351,7 @@ impl Layer<'_> {
/// here. A binary one is a pack, whose sum `binary` already holds. The packs of the padding
/// block are all equal, so their extension vanishes there.
fn skip_sums(&self, claim: &Claim, field: &PowerField, s: F192, binary: &[F192; GROUP]) -> Vec<F192> {
let weights: Vec<Element> = claim.weights.iter().map(|&w| field.to_poly(w)).collect();
let weights = SkipWeights::new(field, &claim.weights);
let groups: Vec<Element> = eq_table(&claim.point[SKIP..])
.iter()
.map(|&e| field.to_poly(e))
Expand Down Expand Up @@ -431,7 +433,7 @@ impl Layer<'_> {
target_feature = "gfni"
));
(
vec![0u64; 4 * ROW],
vec![0u64; PAIR_SCRATCH],
vec![0u64; if gfni { 0 } else { 8 * BITS }],
vec![0u64; 2 * ROW],
)
Expand Down Expand Up @@ -483,11 +485,11 @@ impl Layer<'_> {

/// The sums a round accumulates, unreduced: its values at `0` and at infinity.
#[derive(Clone, Copy, Default)]
struct RoundSums([u64; 9], [u64; 9]);
struct RoundSums([u64; 12], [u64; 12]);

impl RoundSums {
fn merge(mut self, other: Self) -> Self {
for i in 0..9 {
for i in 0..12 {
self.0[i] ^= other.0[i];
self.1[i] ^= other.1[i];
}
Expand Down Expand Up @@ -519,45 +521,49 @@ impl<'a> Round<'a> {
}
Self {
field,
eval: PolyEval::new(s, 72),
eval: PolyEval::new(s, 96),
kappa: field.to_poly(kappa),
weights,
constant,
}
}

/// `B[j] = weights[j] * (kappa + T[j + 1]) + kappa * weights[j + 2]` and `T[j + 2]`, for a
/// row `T`.
fn factors(&self, t: &[u64], b: &mut [u64], shifted: &mut [u64]) {
// `T[j + 1] + kappa`, then `T[j + 2]`: a lane is 64 positions, five lanes to a row.
/// `kappa + T[j + 1]` and `T[j + 2]`, for a row `T`: a lane is 64 positions, five lanes to a
/// row.
fn shifted(&self, t: &[u64], next: &mut [u64], after: &mut [u64]) {
for k in 0..3 {
let (src, next) = (&t[k * BITS..][..BITS], &mut shifted[k * BITS..][..BITS]);
let (src, next, after) = (
&t[k * BITS..][..BITS],
&mut next[k * BITS..][..BITS],
&mut after[k * BITS..][..BITS],
);
for lane in 0..BITS / 64 {
let at = along(64 * lane, 1);
for z in 0..64 {
next[64 * lane + z] = src[at + z] ^ self.kappa[k];
}
}
}
self.field.mul_row(b, &self.weights, shifted, &self.constant, BITS);
for k in 0..3 {
let (src, dst) = (&t[k * BITS..][..BITS], &mut shifted[k * BITS..][..BITS]);
for lane in 0..BITS / 64 {
dst[64 * lane..][..64].copy_from_slice(&src[along(64 * lane, 2)..][..64]);
after[64 * lane..][..64].copy_from_slice(&src[along(64 * lane, 2)..][..64]);
}
}
}

/// Add one pair of rows to a round's sums: the values at `0` and at infinity of
/// `sum over positions of B[j] * T[j + 2]`, weighted by `eq`.
/// `sum over positions of B[j] * T[j + 2]`, weighted by `eq`, where
/// `B[j] = weights[j] * (kappa + T[j + 1]) + kappa * weights[j + 2]`.
fn add_pair(&self, sums: &mut RoundSums, scratch: &mut [u64], low: &[u64], high: &[u64], eq: F192) {
let (b, shifted) = scratch.split_at_mut(2 * ROW);
let (shifted, rest) = scratch.split_at_mut(4 * ROW);
let (next, after) = shifted.split_at_mut(2 * ROW);
for (half, t) in [low, high].into_iter().enumerate() {
self.factors(t, &mut b[half * ROW..][..ROW], &mut shifted[half * ROW..][..ROW]);
self.shifted(t, &mut next[half * ROW..][..ROW], &mut after[half * ROW..][..ROW]);
}
let (zero, infinity) = self
.field
.dot_pair([&b[..ROW], &b[ROW..]], [&shifted[..ROW], &shifted[ROW..]], BITS);
let (zero, infinity) = self.field.round_dot(
&self.weights,
&self.constant,
[&next[..ROW], &next[ROW..]],
[&after[..ROW], &after[ROW..]],
rest,
BITS,
);
let weight = self.field.to_poly(eq);
add_mul(&mut sums.0, &zero, &weight);
add_mul(&mut sums.1, &infinity, &weight);
Expand Down Expand Up @@ -600,7 +606,7 @@ impl<'a> Round<'a> {
fold(out, segment, 0);
}
let mut sums = RoundSums::default();
let mut scratch = vec![0u64; 4 * ROW];
let mut scratch = vec![0u64; PAIR_SCRATCH];
for (pair, &eq) in dst[..to.len() * ROW].as_chunks::<{ 2 * ROW }>().0.iter().zip(eq) {
self.add_pair(&mut sums, &mut scratch, &pair[..ROW], &pair[ROW..], eq);
}
Expand All @@ -610,7 +616,7 @@ impl<'a> Round<'a> {
let (pairs, half) = (to.live.div_ceil(2), to.m / 2);
let mut sums = parallel::map_reduce_with_state(
to.segments * pairs,
|| (vec![0u64; 4 * ROW], vec![0u64; 2 * ROW]),
|| (vec![0u64; PAIR_SCRATCH], vec![0u64; 2 * ROW]),
RoundSums::default,
|(scratch, rows), sums, task| {
let (segment, t) = (task / pairs, task % pairs);
Expand Down Expand Up @@ -643,7 +649,7 @@ impl<'a> Round<'a> {
if first == half {
return;
}
let mut scratch = vec![0u64; 4 * ROW];
let mut scratch = vec![0u64; PAIR_SCRATCH];
for (pad, eq) in pads.as_chunks::<ROW>().0.iter().zip(eq.chunks_exact(half)) {
self.add_pair(sums, &mut scratch, pad, pad, eq[first..].iter().sum());
}
Expand Down
Loading
Loading