diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index fd3bc66b73..e8b7bdcb9b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -161,6 +161,52 @@ jobs: - name: Check formatting run: cargo fmt --all -- --check + # Verifies annotated kernel copies once per ISA configuration, with explicit trusted intrinsic contracts. + # Differential tests detect some copy drift and check supported intrinsic contracts on x86 and ARM64. + # They do not prove source equivalence, compiler correctness or hardware semantics. + verus: + name: Verus proofs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: verification/verus -> target + # Installs the pinned Verus release under ~/.verus, with the Rust toolchain it is built against. + - name: Verify + run: verification/verus/verify.sh + - name: The verified copies match production (AVX2) + working-directory: verification/verus + run: cargo test --release + - name: The verified copies match production, and the intrinsics their specifications (this CPU) + working-directory: verification/verus + env: + RUSTFLAGS: -C target-cpu=native + CARGO_TARGET_DIR: target/native + run: cargo test --release + - name: The verified copies match production (portable) + working-directory: verification/verus + env: + RUSTFLAGS: -C target-cpu=x86-64 + CARGO_TARGET_DIR: target/portable + run: cargo test --release + + verus-arm: + name: Verus proofs (ARM64 tests) + runs-on: ubuntu-24.04-arm + env: + RUSTFLAGS: -C target-cpu=neoverse-n2 -C target-feature=+aes,+sha3 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + workspaces: verification/verus -> target + - name: The verified copies match production, and the intrinsics their specifications + working-directory: verification/verus + run: cargo test --release + msrv: name: MSRV runs-on: ubuntu-latest diff --git a/crates/flock/src/zerocheck/skip_domain.rs b/crates/flock/src/zerocheck/skip_domain.rs index d859e677c5..4973873466 100644 --- a/crates/flock/src/zerocheck/skip_domain.rs +++ b/crates/flock/src/zerocheck/skip_domain.rs @@ -98,9 +98,11 @@ impl SkipDomain { a.mul(scale, sum) } - /// The first round's message, known on `Lambda` and zero on `S`, interpolated at `z` over the window `S + Lambda`. + /// The first round's message, known on `Lambda` and zero on `S`, interpolated at `z` over the window `S ∪ Lambda`. /// /// Its value is `D_2l · V_S(z) · V_Lambda(z) · sum_i values_i / (z + lambda_i)`, `l` the domain's size, with `V_Lambda(z) = V_S(z) + V_S(phi_8(l))`. + /// + /// This barycentric form returns zero on both `S` and `Lambda`. That is the required value on `S`, but at a node of `Lambda` it need not equal the message value, since `1 / 0` is taken as zero. The interpolation guarantee excludes `Lambda`. For a uniformly distributed challenge in `E`, hitting `Lambda` has probability at most `128 / 2^192` and can cause rejection of an honest proof. pub(crate) fn first_round_at(self, a: &mut A, z: A::E, vanishing: A::E, values: &[A::E]) -> A::E { let size = self.size(); let lambda = &PHI_8_TABLE_192[size..2 * size]; @@ -118,6 +120,8 @@ impl SkipDomain { } /// `sum_i L_i(z) values_i` over `S`, `L_i` its Lagrange basis: `D_l · V_S(z) · sum_i values_i / (z + s_i)`. + /// + /// This barycentric form is guaranteed to be the interpolant only for `z` off `S`: at a node `V_S(z)` is zero and `1 / 0` is taken as zero, so it gives 0, which need not equal the node's value. For a uniformly distributed challenge in `E`, hitting `S` has probability at most `128 / 2^192` and can cause rejection of an honest proof. pub fn lagrange_at(self, a: &mut A, z: A::E, vanishing: A::E, values: &[A::E]) -> A::E { let scaled = self.lagrange_scale(a, vanishing); let inverses = self.inverses(a, z); diff --git a/crates/pcs/src/ntt/additive_ntt_f64.rs b/crates/pcs/src/ntt/additive_ntt_f64.rs index e2e1f657bf..fb6ea7d369 100644 --- a/crates/pcs/src/ntt/additive_ntt_f64.rs +++ b/crates/pcs/src/ntt/additive_ntt_f64.rs @@ -82,9 +82,9 @@ impl AdditiveNttF64 { } } - /// Standard NTT with basis `{1, x, …, x^(dim-1)}`. Requires `dim ≤ 63` so - /// the evaluation domain (and the twiddles) stay inside F_{2^64} without - /// wrap; far beyond any codeword size in use. + /// Standard NTT with basis `{1, x, …, x^(dim-1)}`. Requires `1 <= dim <= 63` so + /// the evaluation domain (and the twiddles) stay inside F_{2^64} without wrap. + /// A zero dimension is unsupported: generating its empty first row panics. pub fn standard(dim: usize) -> Self { assert!(dim <= 63, "standard NTT requires dim ≤ 63"); let basis: Vec = (0..dim).map(|i| F64(1u64 << i)).collect(); diff --git a/crates/primitives/src/field/gf2_64x3.rs b/crates/primitives/src/field/gf2_64x3.rs index 9f2fb35e18..14a48ec1f5 100644 --- a/crates/primitives/src/field/gf2_64x3.rs +++ b/crates/primitives/src/field/gf2_64x3.rs @@ -877,6 +877,51 @@ mod tests { } } + /// The planar AVX-512 kernels: eight lane-wise sums and products, and an unreduced sum of products. + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] + #[test] + fn planar_products_match_software() { + use core::arch::x86_64::__m512i; + use core::mem::transmute; + use x86_64::{F192x8, F192x8Sum}; + // Qword `l` of plane `k` is coefficient `k` of element `l`. + let planes = |e: [F192; 8]| { + let plane = |k: fn(&F192) -> u64| { + // SAFETY: eight words and a 512-bit register have the same size and no invalid values. + unsafe { transmute::<[u64; 8], __m512i>(e.each_ref().map(k)) } + }; + F192x8([plane(|e| e.c0), plane(|e| e.c1), plane(|e| e.c2)]) + }; + let elements = |p: F192x8| { + // SAFETY: as above. + let [c0, c1, c2] = p.0.map(|plane| unsafe { transmute::<__m512i, [u64; 8]>(plane) }); + std::array::from_fn::(|l| F192::new(c0[l], c1[l], c2[l])) + }; + let pairs = operand_pairs(13); + // Padded to whole batches, so the last corner pairs are tested too. + let pairs = [ + pairs.as_slice(), + &pairs[..pairs.len().next_multiple_of(8) - pairs.len()], + ] + .concat(); + for batch in pairs.as_chunks::<8>().0 { + let (a, b) = (batch.map(|(a, _)| a), batch.map(|(_, b)| b)); + let want: [F192; 8] = std::array::from_fn(|i| software::mul(a[i], b[i])); + let (a8, b8) = (planes(a), planes(b)); + // SAFETY: the kernels' target features are enabled at compile time. + let (product, sum, total) = unsafe { + let mut sum = F192x8Sum::zero(); + sum.mul_add(a8, b8); + sum.mul_add(b8, a8.add(b8)); + (a8.mul(b8), a8.add(b8), sum.total()) + }; + assert_eq!(elements(product), want); + assert_eq!(elements(sum), std::array::from_fn(|i| a[i] + b[i])); + let want_total = (0..8).fold(F192::ZERO, |s, i| s + want[i] + software::mul(b[i], a[i] + b[i])); + assert_eq!(total.reduce(), want_total); + } + } + /// `frobenius` is the shuffle form of `self^(2^64)`, which `inv` relies on. #[test] fn frobenius_is_the_64th_squaring() { diff --git a/crates/primitives/src/field/gf2_8.rs b/crates/primitives/src/field/gf2_8.rs index 10f0009148..87976be0d6 100644 --- a/crates/primitives/src/field/gf2_8.rs +++ b/crates/primitives/src/field/gf2_8.rs @@ -252,6 +252,8 @@ mod tests { use super::*; #[cfg(target_arch = "aarch64")] use crate::test_util::Rng; + #[cfg(target_arch = "aarch64")] + use core::arch::aarch64::uint8x16_t; #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] use core::arch::x86_64::*; #[cfg(target_arch = "aarch64")] @@ -321,6 +323,26 @@ mod tests { } } + /// The 16-lane reduction alone, on every 16-bit polynomial: callers reduce sums of shifted + /// products, which are not all products of two bytes. + #[cfg(target_arch = "aarch64")] + #[test] + fn neon_gf8_reduce_vec16_matches_scalar() { + for first in (0..=u16::MAX).step_by(16) { + let p: [u16; 16] = std::array::from_fn(|i| first + i as u16); + let (lo, hi): ([u16; 8], [u16; 8]) = (p[..8].try_into().unwrap(), p[8..].try_into().unwrap()); + // SAFETY: baseline NEON on registers; eight `u16`s are sixteen plain bytes, each low byte first, + // which is the interleaved layout the reduction reads. + let got: [u8; 16] = unsafe { + transmute(neon::gf8_reduce_vec16( + transmute::<[u16; 8], uint8x16_t>(lo), + transmute::<[u16; 8], uint8x16_t>(hi), + )) + }; + assert_eq!(got, p.map(gf8_reduce), "first={first:#06x}"); + } + } + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] #[test] fn avx2_gf8_mul_vec32_matches_scalar() { diff --git a/verification/verus/.gitignore b/verification/verus/.gitignore new file mode 100644 index 0000000000..ea8c4bf7f3 --- /dev/null +++ b/verification/verus/.gitignore @@ -0,0 +1 @@ +/target diff --git a/verification/verus/Cargo.lock b/verification/verus/Cargo.lock new file mode 100644 index 0000000000..ab566fd873 --- /dev/null +++ b/verification/verus/Cargo.lock @@ -0,0 +1,420 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "fiat_shamir" +version = "0.1.0" +dependencies = [ + "bincode", + "parallel", + "primitives", + "serde", + "thiserror", +] + +[[package]] +name = "flock" +version = "0.1.0" +dependencies = [ + "fiat_shamir", + "parallel", + "pcs", + "primitives", + "thiserror", + "tracing", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "leanvm-verus" +version = "0.1.0" +dependencies = [ + "fiat_shamir", + "flock", + "parallel", + "pcs", + "primitives", + "vstd", +] + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parallel" +version = "0.1.0" +dependencies = [ + "libc", +] + +[[package]] +name = "pcs" +version = "0.1.0" +dependencies = [ + "fiat_shamir", + "parallel", + "primitives", + "thiserror", + "tracing", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "primitives" +version = "0.1.0" +dependencies = [ + "parallel", + "rand", + "serde", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "verus_builtin" +version = "0.0.0-2026-09-16-0054" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e4a9e69d45ddbe303075756d3fc2fcc965eb9dc70c80651d0e4458a6af1ed83" + +[[package]] +name = "verus_builtin_macros" +version = "0.0.0-2026-09-20-0158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213ca3e8a80925aec251d3f1952a213d6090156b58ad81dccc3a2f1fed77c99f" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", + "verus_prettyplease", + "verus_syn", +] + +[[package]] +name = "verus_prettyplease" +version = "0.0.0-2026-09-06-0133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71956c0dcf55f57fd5a5335fff16981f7e0912d2e887914518c5a64fe36c063c" +dependencies = [ + "proc-macro2", + "verus_syn", +] + +[[package]] +name = "verus_state_machines_macros" +version = "0.0.0-2026-09-06-0133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f78ea16629331605a440e9e0a259dd3492a78212186ba82253ad0988127407f" +dependencies = [ + "indexmap", + "proc-macro2", + "quote", + "verus_syn", +] + +[[package]] +name = "verus_syn" +version = "0.0.0-2026-09-06-0133" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8b53574934e1f608dfccd809ce17f3733d6eb99c14ce8220326121929e1637" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "vstd" +version = "0.0.0-2026-10-04-0306" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f22e1cad7b5f35642a5a020b25e9b49915066d836bab77f8431bbc80228290a7" +dependencies = [ + "verus_builtin", + "verus_builtin_macros", + "verus_state_machines_macros", +] + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879fb705ce98c32e41ebdb970fbe1204f8492423b314c6ab0354c3e7b5542866" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "708882a28301d604fa039cc7727607a98d04c4b86dc76ec9cc683f805d709759" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] diff --git a/verification/verus/Cargo.toml b/verification/verus/Cargo.toml new file mode 100644 index 0000000000..be2e5ef0cb --- /dev/null +++ b/verification/verus/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "leanvm-verus" +version = "0.1.0" +edition = "2021" +publish = false +description = "Verus proofs of leanVM arithmetic and kernel models" + +# Its own workspace: not a member of the leanVM one, so `cargo build` and the main CI never see it. +[workspace] + +[dependencies] +vstd = "=0.0.0-2026-10-04-0306" +# The pool, which the trusted dispatch specification of `src/parallel.rs` runs. +parallel = { path = "../../crates/parallel" } + +[dev-dependencies] +primitives = { path = "../../crates/primitives", features = ["test-util"] } +pcs = { path = "../../crates/pcs" } +fiat_shamir = { path = "../../crates/fiat_shamir" } +flock = { path = "../../crates/flock" } + +[package.metadata.verus] +verify = true + +[lints.rust] +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(verus_only)', 'cfg(verus_keep_ghost)'] } diff --git a/verification/verus/README.md b/verification/verus/README.md new file mode 100644 index 0000000000..1d6c4a34be --- /dev/null +++ b/verification/verus/README.md @@ -0,0 +1,297 @@ +# Verus proofs of leanVM arithmetic, kernel models and BLAKE2s + +This standalone crate verifies annotated executable copies and mathematical models of leanVM arithmetic using [Verus](https://verus-lang.github.io/verus/guide/overview.html). Coverage includes portable field arithmetic, selected SIMD field and reduction kernels, bit transposes, the embedding `φ₈`, bit-fold linear maps, equality tables and multilinear evaluation, additive NTT kernels and evaluation theorems, the `GF(2^8)` NTT used by flock, skip-domain interpolation, Fiat-Shamir block encoding, and portable scalar/batched BLAKE2s. Each theorem applies to its stated preconditions and the checked copy, not directly to production source. + +It is a workspace of its own, outside the leanVM one: `cargo build`, `cargo testall` and the other CI jobs never +see it, and the production crates do not depend on Verus. + +## How it is tied to the production code + +Verus checks functions inside `verus! { }`. Production does not depend on `vstd`: modules here copy production operations and attach specifications. Source adaptations are documented at each function: iterator chains, array maps and patterns become loops or explicit arrays; argument assertions become preconditions; debug assertions become proved assertions. SIMD copies call real intrinsics with trusted lane specifications and trusted memory wrappers where pointer operations cannot be expressed directly. + +`tests/equivalence/` compares executable copies with production on exhaustive small domains and deterministic edge and random inputs. Private functions use reachable public callers or source modules compiled into the test binary, with exceptions recorded below. These tests can detect drift, but do not prove source equivalence or guarantee that future divergence will be caught. A verified model or copy is not verified production source. + +The proof matrix covers portable x86, Haswell, AVX2 with VPCLMULQDQ (with and without GFNI), AVX-512, and AArch64 with and without SHA3. Native tests exercise available hardware; AArch64 tests under QEMU exercise an emulator rather than physical ARM hardware. CI additionally runs the equivalence suite on an ARM64 runner. Runtime feature checks skip unavailable x86 instructions; a passing skipped test is not hardware evidence for that instruction. + +## What is proven + +Polynomials over GF(2) are machine words, bit `i` the coefficient of `x^i`. The specifications are: + +- `clmul(a, b)`: the carry-less product, defined as the XOR over the set bits `i` of `a` of `b << i` + (`src/clmul.rs`). +- `is_remainder(p, r)`: `p = q * M + r` for some polynomial `q`, with `deg r < 64` (`M = x^64 + x^4 + x^3 + x + 1`); + `k_mod(p)` is that remainder and `k_mul(a, b) = k_mod(clmul(a, b))`, the product in `K`. The same for + `GF(2^8)` with `M = x^8 + x^4 + x^3 + x + 1` (`f8_mul`). +- `e_mul(a, b)`: the product of `a0 + a1 y + a2 y^2` and `b0 + b1 y + b2 y^2` as polynomials in `y` over `K`, + folded by `y^3 = y + 1` and `y^4 = y^2 + y`. +- `k_pow`, `e_pow`, `f8_pow`: repeated products. + +### `K = GF(2^64)` (`src/gf2_64.rs`) + +- `software::clmul` computes `clmul`; so do `mul_wide` and `square_wide` (portable arms). +- `reduce(p)` returns the remainder of `p` modulo `M`, for every 128-bit `p`, and the remainder is unique. +- `F64 * F64` is `k_mul`, `F64::square` is `k_mul(a, a)`; `+` is XOR. +- `K` is a commutative ring: `k_mul` is commutative, associative, distributes over XOR, and has one and zero. +- `F64::inv(a)` is `a^(2^64 - 2)` (the Itoh-Tsujii addition chain is checked step by step), and + `a * inv(a) = 1` for every nonzero `a`, `inv(0) = 0`. +- Fermat: `a^(2^64) = a` for every `a`. +- So `K` is a field, with no assumption on `M`: the proof shows the only idempotents (`e^2 = e`) are 0 and 1, + one bit-vector query on the squaring map, then `a^(2^64 - 1)` is an idempotent that is not 0 when `a` is not. +- SIMD copies of `x86_64::mul` and `aarch64::mul_shift_tail` compute `k_mul`; x86 `clmul` and NEON `pmull`/`pmull_hi` compute the specified carry-less product. `aarch64::reduce_pair_pmull4` returns the two `k_mod` remainders lane by lane, for arbitrary 128-bit inputs. BMI2 `spread` equals the carry-less square by depositing each input bit into an even output position. The dispatched `F64` operations retain their field contracts in each configuration. +- These proofs use `lemma_clmul_fold_reduction` for the two high-word folds by `0x1B` and the explicit intrinsic assumptions below. The empty NEON assembly barrier is represented by the trusted identity wrapper `hide_lanes`, not verified assembly. + +### `E = GF(2^192)` (`src/gf2_64x3.rs`) + +- `software::mul_unreduced`, `F192::mul_unreduced`, `mul_base_unreduced` and `From` build an + `F192Unreduced` whose three 128-bit coefficients, each reduced modulo `M`, are the product (`e_value`). +- `F192Unreduced::reduce` returns that element; `F192 * F192 = e_mul`; `mul_base(k) = e_mul(a, k)`; + `square(a) = e_mul(a, a)` (the cross terms cancel). +- Lazy reduction: `e_value(u ^ v) = e_value(u) + e_value(v)`, and for any sequence of unreduced values, + reducing their XOR once equals summing their reductions (`lemma_lazy_reduction`). This is what the + accumulating kernels rely on. +- `mul2`, `mul4`, `mul_unreduced4`, `mul_base8` (portable arms) are lane-wise products; `Weights8::new` then + `get(i)` returns weight `i`; `dot_base` returns `sum_i w_i * k_i`. +- `E` is a commutative ring (associativity by expansion into the 27 triple products). +- `frobenius(a)` is `a^(2^64)` (it is the 64th squaring, by Fermat in `K`). +- `F192::inv(a)` is `a^(2^192 - 2)`, and `a * inv(a) = 1` for every nonzero `a`; the norm `a * φ(a) * φ²(a)` + lies in `K` (production's `debug_assert!`). So `E` is a field too, again with no assumption. +- The x86 Karatsuba product (six PCLMULQDQs and `fold`) gives the same three unreduced coefficients as the + schoolbook product, given that PCLMULQDQ computes `clmul` (`lemma_karatsuba_fold`). + +#### x86 GF(2^192) SIMD copies + +`src/gf2_64x3_x86.rs`, `src/gf2_64x3_x86_avx2.rs`, and `src/gf2_64x3_x86_wrappers.rs` cover the PCLMULQDQ scalar-register kernels, AVX2/VPCLMULQDQ two-product and lane-major kernels, and AVX-512 four-product, lane-major, mixed-product and planar kernels. These are proofs of annotated copies, not proofs obtained by verifying production source directly. + +- `pair`, `karatsuba`, `fold`, `mul_unreduced`, `mul_base_unreduced`, `F192x1` and `F192x1Unreduced` specify the exact coefficient words or held field element. `reduce_lanes128/256/512` reduce arbitrary low/high word pairs, not only multiplication outputs. +- `karatsuba_vec2/4`, `mul_vec2/4`, `mul_unreduced_vec2/4`, `mul_by_pairs` and `mul_by_pairs256` specify every product lane. The configured `F192` and batched dispatch copies call these kernels. Squaring uses the existing base-field square/reduce copies, as production's x86 arm does. +- `Lanes4` holds `[c0,c1]` and `[c2,c2]` in each 128-bit lane. AVX-512 has four such lanes per register; AVX2 has two registers per plane, with elements `2h` and `2h+1` in half `h`. Loads, stores, XOR, multiplication, reduction, and transpose preserve this representation. Transpose moves both words of each plane: output lane `j` of row `i` is input lane `i` of row `j`. Multiplication and AVX2 stores require the duplicated `c2` invariant; the safe wrapper's type invariant enforces it. +- `F192x4` and its unreduced wrapper prove lane-wise arithmetic, memory round trips and transpose; sums use the lazy-reduction identity. `MixedSums8` has eight independently specified sums. `mul_base8_add` updates each by `t * k[i]`, and `mul_base8_reduce` reduces each once. +- AVX-512 `F192x8` stores coefficient `c` of element `i` in qword `i` of plane `c`. Its products are lane-wise `e_mul`; `F192x8Sum::mul_add` adds all eight products to the held sum. `dot_base` is the mixed inner product for `k.len() == 8 * w.len()`. The proof has this precondition; production rejects mismatched lengths. No correctness claim is made for violating a precondition or running code without its required ISA. + +Trust inventory added by these copies: + +- Existing `intrinsics::x86` register views, layout axioms and intrinsic specifications remain assumptions, including carry-less multiplication, XOR, unpacking, shifts, permutation, broadcast and ternary logic. `intrinsics::x86_bits` retains its own registrations; no intrinsic is specified a second time by the GF192 extension. +- `intrinsics::x86_gfx86` adds specifications for `_mm_setzero_si128`, `_mm_set1_epi64x`, `_mm_slli_epi64`, `_mm_srli_epi64`, `_mm_shuffle_epi32`, `_mm256_shuffle_epi32`, `_mm512_shuffle_epi32`, `_mm256_blend_epi32`, `_mm256_permute2x128_si256`, `_mm256_permutevar_pd`, `_mm256_castpd_si256`, `_mm256_castsi256_si128`, `_mm256_extracti128_si256`, `_mm512_setzero_si512`, `_mm512_broadcast_i32x4`, `_mm512_castsi512_si256`, `_mm512_castsi256_si512`, and `_mm512_extracti64x4_epi64`. The widening cast constrains only its low half; unspecified upper words are never used as initialized data. The `__m256d` bit view and the all-zero register-array layout axioms for `core::mem::zeroed` are also trusted. +- Raw pointer helpers are `external_body`: `load_c01`, `load_c2`, `store_c01_c2`, `load_k8`, `load_k_at`, `load_weights`, `load_head8`, `load_tail4`, `store_head_tail`, and AVX2 `load_half`, `store_half`, `load_k4`. Their contracts assume the documented `repr(C)` field layout, transparent `F64` layout, 64-byte `Weights8` alignment and in-bounds reads/writes. Arithmetic, permutations and accumulation outside these helpers are proved, not assumed. + +`tests/equivalence/intrinsics_x86_gfx86.rs` checks each added intrinsic assumption against hardware, including shift boundaries, lane selectors, casts and zeroed arrays. The executable shuffle model takes even word slices of at most eight words, matching the register widths and bounding its index arithmetic. `tests/equivalence/gf2_64x3.rs` checks the copies against production through scalar and batched dispatch, register operators, loads/stores, arbitrary wide reductions, transposes, mixed accumulation, dot products and planar products/sums. Private helpers are exercised through those callers. These deterministic edge/random comparisons are evidence for the tested inputs only; they neither prove the assumptions nor prevent future source drift. + +### `GF(2^8)` (`src/gf2_8.rs`) + +- `clmul8_software` (and the portable arm of `clmul8`) computes the carry-less product of two bytes. +- `gf8_reduce(p)` is the remainder modulo `x^8 + x^4 + x^3 + x + 1` for every 16-bit `p`, not only the + documented 15-bit ones, and the remainder is unique. +- `F8 * F8` is that product, a commutative ring as for `K`; `F8::inv(a)` is `a^254` and `a * inv(a) = 1` for + every nonzero `a`. + +### SIMD kernels of `E` on aarch64 and of `GF(2^8)` (`src/gf2_64x3/aarch64.rs`, `src/gf2_8.rs`) + +Copies of `crates/primitives/src/field/gf2_64x3/aarch64.rs` (all of it), of the aarch64 dispatch arms of `gf2_64x3.rs`, and of the SIMD arms of `gf2_8.rs`, proven against the portable specifications given the intrinsic specifications of `src/intrinsics/aarch64_gfneon.rs` and `src/intrinsics/x86_gfneon.rs` (and the shared `aarch64.rs`, `x86.rs`). They verify in the configurations that compile them: the `E` kernels and the NEON helpers in `neon` and `neon-no-sha3` (EOR3 and its two-XOR fallback), `gf8_mul_vec32` in `haswell`, `avx2-vpclmulqdq`, `avx2-gfni` and `avx512`. A register holding a 128-bit coefficient is read as one polynomial (`v128`, its two lanes). + +- `E` products: `aarch64::mul(a, b) = e_mul(a, b)`; `mul_unreduced` and `mul_base_unreduced` return an `F192Unreduced` whose value (`e_value`) is `e_mul(a, b)`, `e_mul(a, k)`; `mul_base(a, k) = e_mul(a, k)`; `square(a) = e_mul(a, a)`; `reduce(u) = e_value(u)`. The nine PMULL products of `products`, y-folded, are the three coefficients the portable `software::mul_unreduced` builds (`lemma_products_fold`, `folded`), whatever the operand words. `reduce_lane` (two PMULL2 by `0x1B` and a three-way XOR) is `k_mod` of the coefficient (`lemma_clmul_fold_reduction`). +- With those arms, `F192 * F192`, `F192::mul_unreduced`, `mul_base`, `mul_base_unreduced`, `square` and `F192Unreduced::reduce` keep their specifications on aarch64 with `aes`, so everything proven above on top of them (`inv`, the batched portable arms, `dot_base`, ...) holds there too. +- Register-resident values: `F192x1` holds an element (`value()`), with the type invariant that both lanes of its `c22` register hold `c2` (the high-lane products read it); `new`, `load`, `store` (which initializes the `MaybeUninit`), `+` and `*` are `e_add` and `e_mul` of the values, `mul_unreduced` and `mul_base_unreduced` give an `F192x1Unreduced` whose value reduces to the product; `F192x1Unreduced` XOR and `^=` are `u_xor`, `zero()` is `F192Unreduced::ZERO`, `reduce` is `e_value`, and the conversions to `F192` and `F192Unreduced` return the values. So an XOR-accumulated sum of register products reduces to the sum of the products (`lemma_lazy_reduction`). +- `GF(2^8)`: `clmul8_neon` (PMULL on bytes) and the `clmul8` dispatch compute `clmul`. `neon::gf8_reduce_vec16(c0, c1)` returns, in lane `i`, the remainder modulo `M` of polynomial lane `i` (low byte `2 (i % 8)`, high byte `2 (i % 8) + 1` of `c0` for `i < 8`, of `c1` after), for every 16-bit lane, not only products of two bytes: its Barrett quotient (the high byte of `hi * 0x8d * x`) is exact for every high byte (`lemma_reduce_vec16_lane`, one bit-vector query). `neon::gf8_mul_vec16(a, b)` is `f8_mul` lane by lane. `avx2::gf8_mul_vec32(a, b)` is `f8_mul` on each of the 32 bytes: after step `s` of the Horner loop each byte is `a * (b >> (8 - s))` (`lemma_horner_step`, from `a * (x c) = x (a * c)` and linearity). + +### Bit transposes (`src/bits.rs`) + +- `transpose_8x8_bits`: bit `8r + c` of the result is bit `8c + r` of the input; it is an involution. +- `transpose_64x64`: afterwards bit `r` of word `c` is bit `c` of word `r` before, for all `r, c < 64`; + transposing twice restores the matrix. Each masked-swap round exchanges `(r, c)` and `(r ^ J, c ^ J)` + exactly when bit `J` of `r` and `c` differ. +- `bit_transpose_64bytes_portable`: bit `x` of `output[8b + t]` is bit `t` of `input[8x + b]`. + +### Bit transposes, SIMD arms (`src/bits.rs`) + +`bit_transpose_64bytes` and every SIMD arm it dispatches to are copied under production's `cfg(target_feature)` gates, and each proves the portable arm's theorem, `is_bit_transpose_64bytes(input, output)`: bit `x` of `output[8b + t]` is bit `t` of `input[8x + b]`, for all `b, t, x < 8`. `verify.sh` checks each arm in the configurations that compile it: `avx512` (all three x86 arms), `avx2-gfni` (the two AVX2 ones), `haswell` and `avx2-vpclmulqdq` (the AVX2 one), `neon` and `neon-no-sha3`, and `portable` (the dispatch to the portable arm). + +- `bit_transpose_64bytes_gfni` (AVX-512 VBMI with GFNI): after `vpermb` with the written-out `IDX`, word `b` of the register is byte column `b` with the rows reversed, its byte `y` being `input[8 (7 - y) + b]` (`is_reversed_column`, from `lemma_idx`); GF2P8AFFINEQB with the constant `0x8040_2010_0804_0201` then makes bit `x` of result byte `t` bit `t` of the matrix word's byte `7 - x` (`lemma_affine_unit`, one bit-vector query on the instruction's closed form), which undoes the reversal (`lemma_affine_column`). +- `gather_columns_avx2`: byte `k` of result register `w` is input byte `gather_src(w, 16 (k / 16) + order[k % 16])` for every `order` entry below 16: the two unpack rounds put input byte `unpacked_src` at each byte (`lemma_unpack_rounds`), the `0b11_01_10_00` permute moves it to `gather_src` (`lemma_permuted`), and the shuffle by the broadcast `order` picks within each lane. +- `bit_transpose_64bytes_gfni_avx2`: with `REVERSED` each word of the gathered pair is a reversed column (`lemma_reversed`), and the affine map of the AVX-512 arm turns it into its output row (`lemma_gfni_avx2_rows`). +- `bit_transpose_64bytes_avx2` and its `swap`: with `NATURAL` each word is a column in row order (`lemma_natural`); `swap::(x, mask)` is the masked swap `swap_word` on each word, and the three of them are `transpose_8x8_formula` (`lemma_three_swaps`), so each stored word is the output row (`lemma_avx2_rows`, through `lemma_transposed_column`). +- `bit_transpose_64bytes_neon`: `vqtbl4q_u8` with `IDX0..IDX3` makes the two words of register `j` columns `2j` and `2j + 1` (`lemma_neon_idx`, `lemma_neon_columns`); the three rounds are the same three masked swaps; the stores make them rows `2j`, `2j + 1` (`lemma_neon_rows`). + +The copies differ from production where Verus requires it, each noted at the function: the function-local `const` tables are module-level (`IDX` written out instead of built by a `while` loop), `swap` is moved out of `bit_transpose_64bytes_avx2`, `array::map` and `into_iter().enumerate()` become two calls and an index loop, and the loads and stores through `as_ptr()` are the memory helpers below. The copied arms are `pub` (production's are private), so `tests/equivalence/bits.rs` (`every_arm_matches_reference`) runs each one this build compiles against the definition and production's dispatched function. + +### Additive NTT (`src/ntt.rs`) + +Following annex `d` of the leanVM document: + +- Butterflies: the forward butterfly evaluates the line `u + v X` at `X = t` and `X = t + 1`; the inverse + butterfly of `inverse_transform` and the forward one undo each other for every twiddle; the transposed + butterfly undoes the forward one with the rows swapped. +- Twiddles: `span_get(b, idx)` is the subset sum of `b` selected by the bits of `idx`, GF(2)-linear in `idx`; + `twiddle(layer, block)` is `Ŵ_i(sum_j bit_j(block) b_(i+1+j))` with `i = L - layer - 1`; `twiddles_radix8` + returns exactly the seven twiddles of the three layers it fuses. +- Twiddle table: entry `k` of row `i` of `generate_evals_from_subspace(b)` is `s_i(b_(i+k)) * s_i(b_i)^(-1)`, + with `s_0(x) = x`, `s_i(x) = s_(i-1)(x) (s_(i-1)(x) + s_(i-1)(b_(i-1)))` the subspace polynomials; each `s_i` + is GF(2)-linear and vanishes on the span of `b_0 .. b_(i-1)`. +- Layers: `radix8_butterflies` and the radix-4 group of `butterfly_interleaved_fused_2layer` equal three and + two successive single layers. The layer-by-layer forward transform (`forward_scalar_from_layer`, the + production tests' reference, which they compare with the parallel `transform`) followed by the inverse layers + is the identity, for every number of interleaved lanes and every size up to the table's; with one lane the + inverse layers are `inverse_transform`. +- Evaluation: for the table `AdditiveNttF64::standard(dim)` builds (`1 <= dim <= 63`), output word `v` of the + forward transform on `2^dim` words is `P(v)`, where `P(x) = sum_j a_j X_j(x)` with novel basis + `X_j = prod_i Ŵ_i(x)^(bit_i(j))` and the input `a` read as its coefficients; the domain point of index `v` is + the field element `v` itself (no bit reversal). Encoding at rate `2^-r` (layers `r..dim` on `2^r` copies of the + message) gives the evaluations of the polynomial whose coefficients are the message, zero-padded: the + Reed-Solomon codeword (`lemma_standard_forward_evaluates`, `lemma_standard_encode_evaluates`; the sum is + `novel_sum`, equal to the annex's even-odd recurrence `novel_eval` by `lemma_novel_eval_flat`). The proof + needs every row of the table to start with one, `Ŵ_i(b_i) = 1`; it is proven for the standard basis, from + `K` being a field and `s_i` vanishing only on the span of `b_0 .. b_(i-1)`. For an arbitrary basis the same + theorems (`lemma_forward_evaluates`, `lemma_encode_evaluates`) take that as a hypothesis. + +### Parallel NTT driver refinement (`src/ntt_driver.rs`, `src/parallel.rs`) + +- `run_layers` proves the fused radix-8/radix-4/single-layer sweeps equal `sub_layers` for every well-formed table, positive lane count and valid sub-block. `lemma_gather_layer` and `lemma_gather_sub_layers` prove that gathering rows paired by a band of layers commutes with it. +- `group` proves gather/transform/scatter with permissions for every accessed word. Its input can come from the codeword, its first replica, or a separate read-only message, under the explicit permission and source-content preconditions. This is a single-group theorem, not a proof that production's fused-message scheduler establishes those preconditions. +- `gathered_pass` partitions words by `(block, residue)`, dispatches groups through the real pool, and proves equality to the corresponding global layers. `deep_pass` proves the same for the remaining layers on disjoint contiguous sub-blocks. `transform` composes them into `forward_layers`, the verified layer reference, for any positive gathered width and `start <= deep_start <= log_d <= table.len()`, with `2^log_d` rows and any positive lane count whose buffer length fits `usize`. The zero-layer domain is included with a well-formed table. +- These are executable refinements, not production-source proofs. The top-level refinement takes the cache plan as arguments and expects replicas already populated. It does not implement production's cache planner, fused message replication, row callbacks, or streaming-store fences. Gathered work is dispatched per group rather than borrowing scratch once per claimed range; the deep pass dispatches individual sub-blocks instead of explicitly batching adjacent ones. Streaming stores become ordinary copies. Function comments document the other Rust rewrites. +- `PointsTo` maps are split by task owner and returned with their postconditions. Different owners have disjoint keys and only receive their own permissions. Bounds and pointer addresses are checked before constructing row slices. `for_each` and `chunks_mut` are proved adapters over the trusted `for_each_chunk` contract: disjoint claims cover every item exactly once and join before return. The actual pool's atomic claim counter and synchronization are not verified; the conditional permission proof is not a formal safety proof of production's pool. +- `tests/equivalence/ntt_driver.rs` executes the refinement, layer reference and production public encoder word for word on seeded random inputs, rates, lane counts, domain/table sizes, gathered widths and deep splits. Separate processes configure the real pool for 1, 2 and 4 workers. These finite differential checks neither prove production equivalence nor prevent future drift. + +### BLAKE2s (`src/blake2s.rs`, `src/blake2s_batch.rs`) + +- The model transcribes [RFC 7693](https://www.rfc-editor.org/rfc/rfc7693), sections 2 and 3: 32-bit modular addition, G with rotations 16, 12, 8, 7, IV, SIGMA, ten rounds, counter injection, final-block flag and feed-forward. The message model is unkeyed BLAKE2s-256, including parameter initialization, little-endian encoding, zero padding, and the empty message's single final block. +- The scalar executable copies prove `compress = f_spec`, `hash = blake2s_spec`, and streaming `Hasher::finalize = blake2s_spec` of all bytes absorbed. The streaming invariant retains a full last block until more input arrives. Message lengths and continued counters must remain below `2^64`. `hash_from_state` proves whole-block continuation from arbitrary states; `lemma_zero_prefix_continuation` connects the zero-prefix state to the full message hash. +- `compress_groups` proves RFC compression separately for every group and every lane of a transposed state, for all group counts. Both the single-group inline rounds and interleaved multi-group rounds are covered. The theorem is generic over the `Lanes32` arithmetic contracts; `Scalar8` proves those contracts. G and group loops are factored into inline helpers to isolate their proof contexts. +- Safe array models `transpose_words` and `store_digests` prove the state/message transpose and little-endian digest scatter. `compress_rows` composes these with compression and proves each lane's output is the RFC compression digest of that input lane. These layout models replace raw-pointer operations; they do not prove production's pointer manipulation or batched driver. +- Equivalence tests compare scalar compression, one-shot and streaming hashes, prefix states and continuation with production. The RFC Appendix B `"abc"` known answer checks both copies and production. Batch tests compare groups 1, 2 and 4 through production's public batched hash and compare arbitrary transposed compression states lane by lane, including both counter halves and final flags. +- These are functional-correctness results relative to the RFC model, not proofs of collision resistance, preimage resistance, constant-time execution, or cryptographic security. Keyed and variable-output BLAKE2 modes are not modeled. + +### SIMD butterflies of the additive NTT (`src/ntt_simd.rs`) + +The kernels of `crates/pcs/src/ntt/additive_ntt_f64.rs` that `lane_butterflies` dispatches to, `butterfly_lanes_avx512` (VPCLMULQDQ with AVX-512F), `butterfly_lanes_avx2` (VPCLMULQDQ with AVX2, no AVX-512F), `butterfly_lanes_neon_8` and `butterfly_lane_pair_neon` (AES, so PMULL; EOR3 or its two-EOR fallback in `reduce_pair_pmull4`), each for `TRANSPOSED` false and true, and `lane_butterflies` with production's `cfg` arms. Each `cfg` arm is checked in the `verify.sh` configuration that compiles it: `avx512`, `avx2-vpclmulqdq` and `avx2-gfni`, `neon` and `neon-no-sha3`; the others (`portable`, `haswell`) check the scalar arm. + +- Each kernel at offset `at` of width `w` (8, 4, 8, 2) leaves word `j` of the rows, for `at <= j < at + w`, as `butterfly_spec(TRANSPOSED, u_j, v_j, t)` (forward `u' = u + t v, v' = v + u'`, transposed `u' = u + v, v' = v + t u'`, products by `k_mul`, the specification of the portable `butterfly_one`), and every other word unchanged (`butterflied`). +- The products: PCLMULQDQ `0x00` and `0x11` unpacked low with high give word `i`'s 128-bit product in lane order (`lemma_products_in_lane_order`); PMULL and PMULL2 read back as `uint64x2_t` give it as the register's two lanes (`lemma_pmull_products`, `lemma_reduced_products`), reduced by the already verified `reduce_pair_pmull4`. +- The x86 reduction `lo ^ g(hi ^ spill)`, `g(x) = x ^ x<<1 ^ x<<3 ^ x<<4`, `spill = hi>>63 ^ hi>>61 ^ hi>>60`, is `k_mod` of the product (`lemma_shift_reduction`, from `lemma_k_mod`). On AVX-512 the XORs are `vpternlogq 0x96`, the three-way XOR (`lemma_ternlog_xor3`); on AVX2 the shifts are doublings by addition (`lemma_avx2_lane`) and the spill is one byte shuffle of a 16-entry table by the top nibbles, proven word by word from the shuffle's byte semantics and the register layout (`lemma_spill_shuffle`, `lemma_spill_table`, `lemma_word_of_low_byte`). +- `lane_butterflies` (and so `butterfly_lanes`, `transposed_butterfly_lanes`) keeps its specification in every configuration: the kernel blocks compose (`lemma_butterflied_extend`), then the NEON pair tail and the scalar tail finish the row. +- Rewrites, noted at each function: the kernels take the rows and the offset their pointers point to instead of `*mut F64`, and load and store through the helpers below (Verus cannot obtain pointer permissions from the `&mut [F64]` borrows `lane_butterflies` holds); production's function-local constants `XOR3` and `SPILL` are module constants, `SPILL` listed rather than computed by a `while` loop in a `const` block; the AVX2 kernel names the nibble shift and the table load (`nibbles`, `t128`) so the proof can refer to them; the NEON pair loop tests `top.len() - lane >= 2` instead of `lane + 2 <= top.len()`. +- Tests (`tests/equivalence/ntt_simd.rs`): each kernel, both directions, every offset of short rows, edge rows and twiddles (0, 1, all ones, the top bit, every top nibble) and random ones, against production's butterfly in the production field; `butterfly_lanes` and `transposed_butterfly_lanes` on every row length up to 40; and production's own kernels, which are private, through `encode_interleaved_in_place` on 8 to 31 lanes, reproduced by a layer-by-layer driver built from the verified `butterfly_lanes`. Run natively (AVX-512), with the AVX2 flags, and under qemu with and without SHA3. + +### The embedding `φ₈` (`src/phi8_tower.rs`) + +`φ₈` (`phi8`) is the GF(2)-linear map sending the byte `x^i` to `PHI_8_BASIS[i]` in `K` (`phi8_basis`); `phi8_e(a)` is `φ₈(a)` in `E`. Annex `c` calls it the subfield embedding of `GF(2^8)` in `E`, and flock's univariate skip domain is `φ₈(0..64)`. + +- `build_phi8_table_192` fills entry `v` with `φ₈(v)` embedded in `E`; so does the static `PHI_8_TABLE_192`, and `phi8_192(a)` returns `phi8_e(a)`. +- `φ₈` is GF(2)-linear (`lemma_phi8_xor`), so `φ₈(0..64)` is the span of `φ₈(1), φ₈(2), .., φ₈(32)`, and zero only at zero (`lemma_phi8_nonzero`, from injectivity). +- `φ₈` is multiplicative into `K`: `φ₈(a * b) = φ₈(a) φ₈(b)` for all bytes, `a * b` the product of `GF(2^8)` (`f8_mul`) and the right side `k_mul` (`lemma_phi8_mul`). The proof: `φ₈(x a) = φ₈(x) φ₈(a)` (`lemma_phi8_mulx`), which by linearity needs only the eight monomials, eight products of concrete constants in `K` checked by bit-vector evaluation of the closed-form carry-less product (`clmul64_closed`, `lemma_k_mul_closed`); then `φ₈(x^i b) = φ₈(x^i) φ₈(b)` by induction on `i` and associativity in `K`, and the general product by linearity in the left factor. +- So `φ₈` is a ring homomorphism into `E`, `φ₈(1) = 1`, `φ₈(0) = 0`, and it is injective (the eight basis images are GF(2)-independent), with image in `K` (`c1 = c2 = 0`) (`lemma_phi8_e_homomorphism`). Its image is therefore a subfield of `E` with 256 elements. That it is the only one (production's doc comment) is not proven: it would need that a polynomial of degree 256 has at most 256 roots. + +### Bit folds and linear maps of `E` (`src/bit_fold.rs`) + +The portable arm of `crates/primitives/src/bit_fold.rs` (`bit_fold/portable.rs`) and the wrappers `BitFold`, `F192Map`, `Sliced`. Bit `s` of a row of bytes is bit `s % 8` of byte `s / 8`; coordinate bit `b` of an `E` value is bit `b % 64` of coefficient `b / 64`. + +- Tables: `lookup_tables(w)` returns one table per whole chunk of 8 weights, entry `[j][v]` the sum of `w[8j + i]` over the set bits `i` of `v` (`byte_sum`); the lowest-set-bit recurrence is proven to build exactly that (`lemma_byte_sum_clear`). The weight of bit `s` is recovered as table `s / 8` at the single bit `s % 8` (`lemma_byte_sum_monomial`), so `Imp::new(w)` holds the weights `w`, truncated to whole chunks. +- Fold: `BitFold::new(w)` (for 8, 16, 32, 64 or 128 bytes per row, production's `assert!` being the `requires`) then `fold_block(rows, out)` sets `out[p] = sum_{s : bit s of rows[p]} w_s` (`fold_spec`) for every `p < rows.len()`, the definition the module doc states, and leaves `out[rows.len()..]` unchanged. The table lookups add up to that sum byte by byte (`lemma_tables_fold`). +- Linear maps: `F192Map::new(w)` then `apply_add(xs, out)` (and `apply_sliced_add` on `Sliced::new(xs)`) adds `map(w, xs[p]) = sum_{b : coordinate bit b of xs[p]} w_b` (`map_spec`) to `out[p]` for every `p < out.len()`: the 24 little-endian bytes of a value fold to its map (`lemma_row_sum_le_row`). The map is GF(2)-linear (`lemma_coord_sum_add`, `lemma_coord_sum_zero`) and every element is the sum of the coordinate vectors of its set bits (`lemma_units_sum`), so a GF(2)-linear `Φ: E -> E` is the `F192Map` of the weights `Φ(unit(b))`, which is how ring switching uses it. +- `after_mul(c)` returns the map with weights `map(w, unit(b) * c)` and, for every `x`, `after_mul(c)(x) = map(w, x * c)` (`lemma_after_mul`, from the linearity of the map and the distributivity `(a + b) c = a c + b c` in `E`, `lemma_e_mul_add_left`). This is the identity production's `composed_map_is_the_map_after_the_product` tests. +### Lane-interleaved NTT (`src/ntt_lanes.rs`) + +A buffer of `m` interleaved lanes holds word `v` of lane `l` at word `m v + l` (`lane(data, m, l)`). + +- One layer commutes with taking a lane: lane `l` of a forward or inverse layer's output is the same layer applied to lane `l` alone (`lemma_lane_layer`), so the forward layers `start..end` and the inverse layers transform every lane independently (`lemma_lane_forward_layers`, `lemma_lane_inverse_layers`). This holds for any table and any lane count, and `forward_scalar_from_layer` computes `forward_layers` for any lane count. +- Evaluation: for `AdditiveNttF64::standard(dim)` on `m` lanes of `2^dim` rows, output word `m v + l` is `P_l(v) = sum_j a_(m j + l) X_j(v)`, lane `l`'s novel-basis polynomial at the domain point `v` (`lemma_standard_lanes_forward_evaluates`). +- Encoding: the encoder at rate `2^-r` on `m` lanes (layers `r..dim` on `2^r` copies of an `m`-lane message of `2^(dim - r)` rows, as `encode_interleaved_in_place` lays it out) gives at word `m v + l` the evaluation at `v` of the polynomial whose coefficients are lane `l` of the message, zero-padded: every lane is Reed-Solomon encoded (`lemma_standard_lanes_encode_evaluates`). +- No new executable copy: the lanes theorems are about `forward_scalar_from_layer`, already checked against `encode_interleaved_in_place` for 1, 2, 3, 5 and 8 lanes, at rate 1 and at several rates, in `tests/equivalence/ntt.rs`. + +### `GF(2^8)` additive NTT of flock's zerocheck (`src/flock_ntt.rs`) + +Over the field of `src/gf2_8.rs`, with the standard basis `b_i = x^i` (the byte with bit `i` alone), the subspace polynomials `s_0(x) = x`, `s_i(x) = s_(i-1)(x) (s_(i-1)(x) + s_(i-1)(b_(i-1)))` and `Ŵ_i(x) = s_i(b_i)^(-1) s_i(x)`. `novel8(m, a, x)` is the novel-basis polynomial `sum_(j < 2^m) a_j X_j(x)`, `X_j = prod_(i < m) Ŵ_i(x)^(bit_i(j))`, written by its split on the top bit of `j`; `lemma_novel8_flat` proves it equal to the flat sum `novel8_sum`. `fft_spec` and `ifft_spec` are the recursions of `fft_rec` and `ifft_rec`. + +- Subspace polynomials: each `s_i` and `Ŵ_i` is GF(2)-linear, `s_i` vanishes exactly on `{0, .., 2^i - 1}` (`lemma_subspace_poly8_vanishes`, `lemma_subspace_poly8_roots`, from `GF(2^8)` having no zero divisors), so `Ŵ_i(b_i) = 1` for every `i < 8` (`lemma_normalized_poly8_at_basis`). +- Twiddles: `compute_twiddles(k, β)` (any `k <= 8`, any offset `β`) returns `2^k - 1` entries, entry `2^d - 1 + j` (depth `d < k`, block `j < 2^d`) being `Ŵ_(k-1-d)(β + j 2^(k-d))`, `Ŵ` of the first point of the block, as its documentation's layout says (`twiddles_of`, the postcondition). +- Butterflies and recursions: `fft_butterfly`, `ifft_butterfly`, `fft_rec` and `ifft_rec` compute `fft_spec` and `ifft_spec` on every power-of-two length; `AdditiveNttGf8::forward` and `inverse` are `fft_rec` and `ifft_rec` from the root. +- Evaluation: with the table of `new(k, β)`, output word `u` of `forward` on `2^k` coefficients is `P(β + u)` (the point `β ⊕ u`, no bit reversal), the novel-basis polynomial of the input at the `u`-th point of the domain `β + span{1, 2, .., 2^(k-1)}`, as the struct's documentation claims (`lemma_fft_evaluates`). +- Inverse: `inverse` undoes `forward` and `forward` undoes `inverse`, for any table and any buffer (`lemma_ifft_after_fft`, `lemma_fft_after_ifft`), so `inverse` interpolates: on the evaluations of `P` it returns `P`'s coefficients (`lemma_ifft_interpolates`). +- The extension matrix: let `M = forward_Λ ∘ inverse_S` for any two `2^k`-point NTTs (offsets `β_s`, `β_l`), column `t` being the image of `e_t` (`lde_column`). Then `M[i][j] = M[i ⊕ j][0]` (`lemma_lde_shift`): the interpolant of `e_j` on `S` is the interpolant of `e_0` translated by `j`, because a translate of a novel-basis polynomial is again one (`translate`, `lemma_translate`) and the transform is injective. This is the "XOR-shift relation" `inv_table.rs` relies on. +- The table: `InvNttTableByteSingleGf8::new(ntt_s, ntt_l)` (`3 <= k <= 7`) holds, at row `w`, the XOR of the columns `t < 8` of `M` over the set bits `t` of `w`, as its field's documentation claims (`table_row`, the postcondition), for any two NTTs of the same `k`. +- `apply_scalar` applies `M`: its output word `i` on a row of `ell / 8` bytes is `sum_b T[bytes[b]][i ⊕ 8b]` (the postcondition), which equals `sum_j x_j M[i][j]`, `x_(8b+t)` being bit `t` of byte `b`, for the table of two NTTs built by `new` (`lemma_table_applies_lde`). +- Copies: `compute_twiddles` and `new` take `k <= 8` as a precondition (the domain lies in `GF(2^8)`; production does not check it), `fft_rec` and `ifft_rec` take a power-of-two length and in-table twiddle reads (production's only caller guarantees both), the `assert!`s become `requires`, iterator loops and `copy_from_slice` become index loops, the `continue` of `new`'s last loop becomes an `if` (Verus's `for` has no `continue`), and `w.trailing_zeros()` is taken on `w as u64` (vstd specifies it for `u64`, not `usize`). + +### Fiat-Shamir step block (`src/fiat_shamir.rs`) + +- `step_block(scalars, tag)` and `builder_step_message`, a value-level model of the message the circuit's `Builder::step` hashes, put the last scalar in words 4 to 6, the preceding scalar (if present) in words 0 to 2, the count in word 3 and the tag in word 7, with zero elsewhere (`block_word`). For equal scalar/tag inputs these models produce the same block. This does not prove the circuit constraints bind its wires to those inputs. +- On the domain production takes (at most `MAX_PENDING = 2` scalars, longer slices panic) the block names its scalars and its tag (`lemma_step_block_decodes`), so two steps with the same block absorb the same scalars, so the same count, under the same tag (`lemma_step_block_injective`). This holds for every tag word; the four `DS_*` tags are pairwise distinct (`lemma_tags_distinct`). +- This is injectivity of the current two-scalar step encoding, not a security theorem for the Fiat-Shamir transform or a proof about any replacement transcript or duplex design. + +### The equality polynomial and multilinear evaluation (`src/multilinear.rs`) + +Over `E`, `eq(r, x) = prod_i (r_i x_i + (1 + r_i)(1 + x_i))` (`eq_poly`, `eq_factor`; `1 - a = 1 + a` in characteristic 2), and a table over `n` variables holds at index `x` the value at the cube point whose coordinate `i` is bit `i` of `x` (`cube_point`, `eq_at`). `is_eq_table(t, r, seed)` says `t[x] = seed * eq(r, x)` for all `2^n` indices. + +- `eq_eval(r, x)`, the product of `1 + r_i + x_i`, is `eq(r, x)` for every `x` in `E^n`, not only on the cube (`lemma_eq_factor_sum`). +- `eq_table`, `eq_table_seeded` and `fill_eq_table_uninit` build `seed * eq(r, .)` entry by entry, LSB first, for `n < 64`: the doubling build below `2^16` entries (each level writes `v r_i` to the high child and `v (1 + r_i)` to the low one, `lemma_eq_at_high`) and the tensor build above (`eq(r, (h << L) | l) = eq(r[..L], l) eq(r[L..], h)`, `lemma_eq_at_tensor`). +- `shrink_eq_low` and `shrink_eq_high` sum the low pairs, or the two halves, of any table; on `seed * eq(r, .)` the result is `seed * eq(r[1..], .)`, or `seed * eq(r[..n-1], .)` (`lemma_shrink_low_eq`, `lemma_shrink_high_eq`). +- `SplitEq::with_low_vars`, `with_high_vars`: the two tables are `eq` over `r[..L]` and `r[L..]`; `at(x)` is `eq(r, x)`. +- `interp(lo, hi, t) = (1 + t) lo + t hi`, and `interp_k` the same on `K` endpoints. +- `mle_eval(table, point)` is the multilinear extension `sum_x eq(point, x) table[x]` (`mle`) of the `K`-valued table, on both paths: folding the lowest variable (`fold_low_k`, then `fold_ladder`, by `lemma_mle_fold_low`), and from three variables the packed low `eq` table, one `dot_base` per row, then the ladder over the rows (`lemma_mle_blocks`). +- `window_denominator(2^log)` (and the table `DENOMINATORS` it reads, computed at compile time) is `(prod_{k=1}^{2^log - 1} φ₈(k))^(2^64 - 2)`, the inverse in `K` of the product of the window's nonzero nodes (`lemma_denominator_inverts`; the product is nonzero since the nodes are and `K` has no zero divisors). + +### The univariate-skip domain (`src/skip_domain.rs`) + +`SkipDomain` of `crates/flock/src/zerocheck/skip_domain.rs` is generic over `fiat_shamir::arith::Arith`; the copy is instantiated at `Native` (its trait methods and the defaults it inherits copied as inherent methods). The domain of `l = 2^k` nodes (`k < 8`) is `S = {s_i = φ₈(i)}`; `V_l(z) = prod_i (z + s_i)` (`vanishing_spec`) and `L_i(z) = prod_{k != i} (z + s_k) / prod_{k != i} (s_i + s_k)` (`lagrange_basis`), the textbook Lagrange basis, with `L_i(s_j) = [i = j]` (`lemma_lagrange_basis_at_nodes`). + +- `vanishing_coefficients` returns the coefficients of `V_l` as a linearized polynomial, `V_l(x) = sum_j c_j x^(2^j)` for every `x`, and it is monic (production's `debug_assert!`); adding a basis element `a` takes `V` to `V(x)^2 + V(a) V(x)` (`lemma_lin_step`, `lemma_vanishing_double`). `vanishing(z)` is `V_l(z)`. +- Every node of a window sees the same `prod_{k != i} (s_i + s_k)`: `k -> i ^ k` permutes the window (`lemma_prod_xor`), so it is the product of the nonzero nodes, which `window_denominator` inverts (`lemma_weight_inverts`). +- `lagrange_at(z, V_l(z), values)` (through `lagrange_scale`, `inverses`, `lagrange_with`) is `sum_i values_i L_i(z)` (`lagrange_sum`), and `first_round_at(z, V_l(z), values)` is `sum_i values_i L_(l+i)(z)` over the window of `2l` nodes (`window_sum`): the interpolant of `values` on the coset `{s_l, .., s_(2l-1)}` and of zero on `S`, for every `z` off the respective theorem's excluded nodes. At a node production returns zero because its vanishing factor is zero and `1 / 0 = 0`. This need not equal the supplied value on `S` for `lagrange_at` or on `Lambda` for `first_round_at`; zero is the correct first-round value on `S`. Under a uniform challenge in `E`, the exceptional set for each operation has at most `128 / 2^192` probability and can cause rejection of an honest proof. The proofs do not establish transcript challenge uniformity. + +## Trust base and assumptions + +- Kernel theorems do not use `assume` or `admit` to discharge their obligations. The trust boundary explicitly includes `assume_specification`, external vector type declarations, `axiom fn` layout facts, and `#[verifier::external_body]` memory helpers in the intrinsic modules. These are assumptions, not proved ISA or pointer-safety results. +- The driver has explicit trusted adapters in `src/parallel.rs`: slice/permission borrowing, raw slice construction, core pointer/integer specifications, worker count, and exactly-once joined dispatch. `with_scratch` trusts the thread-local scratch borrow and callback contract. Their `external_body`/`assume_specification` annotations are trust boundaries, not solved proof obligations. +- `src/blake2s.rs` retains an explicit `assume_specification` for `u32::rotate_right`, absent from the pinned `vstd`: for `0 < n < 32` it equals `(x >> n) ^ (x << (32 - n))`. A deterministic test checks samples and edge cases against Rust's implementation, but does not prove this trusted library contract. +- Verus and Z3 are trusted: Verus's encoding of Rust (machine integers, the truncating casts and shifts the code + uses, arrays, `Vec`) and Z3's answers, in both its integer and its bit-vector modes. +- `vstd`'s specifications of what the copies call are trusted: integer `From`, the operator traits, `Vec` and + slice indexing (by index and by range), `slice_to_vec`, `vec![x; n]`, `Vec::clone`, `Vec::truncate`, + `Vec::as_mut_slice`, `split_at_mut`, `u64::trailing_zeros`, and its proven `pow2` and division lemmas. +- `src/ntt.rs` declares `global size_of usize == 8`: the NTT proofs are for 64-bit targets, which Verus checks + when it compiles the crate. +- Copy-to-production correspondence is tested, not proved. `builder_step_message` has no public production counterpart (`Builder` is in `leanvm_core`'s private `rec` module); production's `rec::transcript::tests::the_circuit_replays_the_native_transcript` checks the circuit/native transcript relationship separately. `SkipDomain::first_round_at` and `SkipDomain::new` are compared with the references used by production's own tests. +- `PHI_8_TABLE_192`, `DENOMINATORS` and `SkipDomain::FLOCK` are written in Verus's `exec static` / `exec const` form, which states what the initializer returns; Verus checks the initializer like a function body. The parallel pass of `fill_eq_table_uninit` is copied as a loop over the same rows in order. +- `src/intrinsics/mod.rs` trusts `core::mem::transmute` through the uninterpreted `transmuted` view. Per-architecture layout axioms relate byte, word and polynomial views in little-endian lane order. Intrinsic specifications and memory contracts are compared with their executable models, not proved from ISA semantics, compiler lowering or hardware. +- Shared x86 contracts in `src/intrinsics/x86.rs`, exercised by `tests/equivalence/intrinsics_x86.rs`, specify scalar/register conversion, word constructors, XOR/AND, wrapping 64-bit addition, PCLMULQDQ and VPCLMULQDQ per 128-bit lane (immediate bits 0 and 4 choose operands), BMI2 bit deposit, low/high word unpacking, lane shifts, lane-local byte shuffles, broadcast, 64-bit permutations, two-source permutations, 128-bit-block shuffles, and ternary Boolean logic. Layout axioms are `axiom_m128_bytes`, `axiom_m256_bytes`, `axiom_m512_bytes`, `axiom_m128_as_u128`, `axiom_m256_as_pairs`, `axiom_m512_as_pairs`, and `axiom_m512_from_words`. Their source declarations enumerate each intrinsic and immediate domain. +- Shared AArch64 contracts in `src/intrinsics/aarch64.rs`, exercised by `tests/equivalence/intrinsics_aarch64.rs`, specify `vmull_p64`, `vmull_high_p64`, `vdupq_n_u64`, `veorq_u64`, `veor3q_u64`, `vzip1q_u64`, and `vgetq_lane_u64`. Layout axioms are `axiom_u128_as_u64x2`, `axiom_u64x2_as_u128`, and `axiom_u64x2_as_p64x2`. `gf2_64::aarch64::hide_lanes` trusts the empty inline-assembly barrier to preserve its registers; reduction tests exercise that wrapper through `reduce_pair_pmull4`. +- `src/bit_fold.rs` relies on `vstd`'s specification of `u8::trailing_zeros` (with its proven `axiom_u8_trailing_zeros`) and of `Vec::push`, `Vec::as_slice` and `Vec::as_mut_slice`. +- `src/intrinsics/aarch64_gfneon.rs` and `src/intrinsics/x86_gfneon.rs` (the `E` aarch64 kernels and the `GF(2^8)` SIMD arms): `assume_specification`s of `vreinterpretq_p64_u64`, `vcreate_u64`, `vcombine_u64`, `vextq_u64`, `vdupq_laneq_u64`, `vdup_n_p8`, `vmull_p8`, `vreinterpretq_u16_p16`, `vreinterpretq_u8_u16`, `vgetq_lane_u16`, `vshlq_n_u16`, `vget_low_u8`, `vget_high_u8`, `vuzp1q_u8`, `vuzp2q_u8`, `veorq_u8`, `_mm256_setzero_si256`, `_mm256_set1_epi8`, `_mm256_add_epi8`, `_mm256_cmpgt_epi8`; the layout axioms `axiom_u8x8_as_p8x8`, `axiom_u64_as_p8x8`, `axiom_words_as_u64x2`; and the `external_body` memory helpers `gf2_64x3::aarch64::load_f192` and `store_f192`, whose bodies are production's loads and stores. Each is compared with the hardware in `tests/equivalence/intrinsics_aarch64_gfneon.rs` and `intrinsics_x86_gfneon.rs` (under `qemu-aarch64-static` for NEON), through the executable twins `model_vmull_p8_lane`, `model_uzp_u8_lane`, `model_ext_u64_lane`, `model_u16_byte`, `model_cmpgt_epi8_lane` where the lane semantics is more than a move. `vstd`'s `MaybeUninit` specification (`as_option`) states what `store` writes. +- The SIMD arms of the bit transposes assume the specifications of the intrinsics they call, each tied to the hardware by a differential test that runs the real intrinsic on edge and random operands and compares every lane with the specification's executable twin: + - `src/intrinsics/x86_bits.rs`, tested by `tests/equivalence/intrinsics_x86_bits.rs`: `_mm256_unpacklo_epi8`, `_mm256_unpackhi_epi8` (`unpacklo_epi8_lane`, `unpackhi_epi8_lane`), `_mm512_permutexvar_epi8` (`permutexvar_epi8_lane`: byte `idx[k] & 63`, tested with the top index bits set), and `_mm256_gf2p8affine_epi64_epi8`, `_mm512_gf2p8affine_epi64_epi8` (`gf2p8affine_lane`: per byte, bit `i` is the parity of `A.byte[7 - i] & x` XOR bit `i` of the immediate, `A` the 64-bit word holding the byte; tested on random and edge matrices with the immediates `0, 1, 0x63, 0x80, 0xAA, 0xFF`, and the twin `model_affine_byte` against Intel's pseudocode written with `count_ones`). The memory helpers `load128_bytes`, `load256_bytes_at`, `store256_bytes_at`, `load512_bytes`, `store512_bytes` (`external_body`, each body the production load or store) are tested at several offsets, the stores also for leaving the other bytes alone. + - `src/intrinsics/aarch64_bits.rs`, tested by `tests/equivalence/intrinsics_aarch64_bits.rs` (under `qemu-aarch64-static` from x86): `vqtbl4q_u8` (`tbl4_byte`: the 64-byte table, zero from index 64 on), `vreinterpretq_u64_u8`, `vreinterpretq_u8_u64` (the little-endian layout, `u64x2_byte`), `vandq_u64`, `vshrq_n_u64` (zero at 64), `vshlq_n_u64`, and the helpers `vld1q_u8_16`, `vld1q_u8_at`, `vst1q_u8_at`. The view `u8x16` of `uint8x16_t` is the `transmute` to `[u8; 16]`; `uint8x16x4_t` is a transparent external type (its four public fields). + - The shared ones of `src/intrinsics/x86.rs` (`_mm256_shuffle_epi8`, `_mm256_permute4x64_epi64`, `_mm256_broadcastsi128_si256`, `_mm256_set1_epi64x`, `_mm256_and_si256`, `_mm256_xor_si256`, the 64-bit lane shifts, `_mm512_set1_epi64`, and the byte layout axioms `axiom_m128_bytes`, `axiom_m256_bytes`, `axiom_m512_bytes`) and `src/intrinsics/aarch64.rs` (`vdupq_n_u64`, `veorq_u64`). +- `src/ntt_simd.rs` relies on the intrinsic specifications of `src/intrinsics/x86.rs` (`_mm512_set1_epi64`, `_mm512_xor_si512`, `_mm512_clmulepi64_epi128`, `_mm512_unpacklo_epi64`, `_mm512_unpackhi_epi64`, `_mm512_srli_epi64`, `_mm512_slli_epi64`, `_mm512_ternarylogic_epi64`; `_mm256_set1_epi64x`, `_mm256_xor_si256`, `_mm256_clmulepi64_epi128`, `_mm256_unpacklo_epi64`, `_mm256_unpackhi_epi64`, `_mm256_broadcastsi128_si256`, `_mm256_srli_epi64`, `_mm256_shuffle_epi8`, `_mm256_add_epi64`, with the byte layout axioms `axiom_m128_bytes`, `axiom_m256_bytes`) and `src/intrinsics/aarch64.rs` (`vdupq_n_u64`, `veorq_u64`, `vgetq_lane_u64`, `vmull_p64`, `vmull_high_p64`, with `axiom_u128_as_u64x2`, `axiom_u64x2_as_p64x2`, and those `reduce_pair_pmull4` uses), and on the trusted memory helpers of `src/intrinsics/x86_nttsimd.rs` (`loadu512_at`, `storeu512_at`, `loadu256_at`, `storeu256_at`, `loadu128_bytes`) and `src/intrinsics/aarch64_nttsimd.rs` (`vld1q_u64_at`, `vst1q_u64_at`): `external_body` functions whose bodies are production's load or store at the offset and whose specification is the words they read or write. `tests/equivalence/intrinsics_x86_nttsimd.rs` and `intrinsics_aarch64_nttsimd.rs` check each helper at every offset of random rows. + +## Not covered + +- Production-source equivalence, compiler correctness, CPU correctness and absence of undefined behavior in trusted memory helpers are not established by these proofs. Production differential coverage includes: + - `K`: `field::gf2_64::tests::mul_and_square_match_the_reference` (PCLMULQDQ and PMULL products, `pdep` square), + `neon_variants_match_software`. + - `E`: the AArch64 and x86 SIMD copies are verified as described above. Production also checks them with `field::gf2_64x3::tests::products_match_software`, `batched_products_match_software`, `lane_products_match_software`, `register_products_match_software`, `batched_mixed_products_match_scalar`, `mixed_sums_match_software`, and `planar_products_match_software`. + - `GF(2^8)`: the SIMD arms are verified (above); production also tests them with `software_matches_neon`, `neon_gf8_mul_vec16_matches_scalar`, `avx2_gf8_mul_vec32_matches_scalar`, `neon_gf8_reduce_vec16_matches_scalar`. + - Bit transposes: `bits::tests::every_arm_matches_reference`; the SIMD arms of `bit_transpose_64bytes` are also verified (see above). + - Bit folds and `F192Map`: `bit_fold::tests::fold_block_matches_definition`, `f192_map_matches_definition`, `composed_map_is_the_map_after_the_product`, `avx2_products_match_definition` (every AVX2 product, also on GFNI machines). The equivalence tests of this crate compare the verified portable copies with whatever arm the machine dispatches. + - NTT butterflies: verified (`src/ntt_simd.rs`, above); production's driver tests `ntt::additive_ntt_f64::tests::interleaved_parallel_matches_scalar` (forward) and `whir::induce::tests::blocked_and_gathered_transposes_match_layer_by_layer` (transposed) run them too. + - flock's LDE table (`apply_v128` on NEON and SSE2, `apply_avx2`, `apply_avx512`/`apply_zmm`): `zerocheck::ntt::inv_table::tests::apply_simd_matches_apply_scalar`, and `tests/equivalence/flock_ntt.rs`, which compares the dispatched `apply` with the verified `apply_scalar` on every single-byte row and on random rows. The round-1 kernels that read the table through `data_ptr` and `apply_zmm` (`zerocheck/round1.rs`) are not covered. +- Production's NTT cache planner, fused-message scheduling/replication, row callbacks, streaming store ordering, and `transpose_lane_major` are not proved. The parallel refinement above proves populated-replica computation with explicit plan inputs. The inverse reference in `ntt.rs` remains covered; the separate transposed driver is outside this refinement. +- BLAKE2s's raw-pointer batch driver, pointer load/store safety, dispatch and SIMD implementations are not verified. Batch comparisons exercise the backend selected by the test build. +- Bit folds (`crates/primitives/src/bit_fold.rs`): `BitFold::at_level`, which builds its weights with `multilinear::eq_table` (copied in `src/multilinear.rs`) and then calls `BitFold::new`, and `BitFold::fold_quads` (AVX-512 with GFNI only) are not copied. Only `out[..rows.len()]` of `fold_block` is documented and compared: past it the portable arm leaves `out` as it was (proven) and the SIMD arms store the fold of a zero row. +- Of `multilinear.rs`, the parallel `mle_eval_par`, `SplitEq::weighted_sum` and its SIMD variants, the high folds (`fold_high_k`, `fold_high_inplace`, `interp_into`), `barycentric_sum`, `skip_lagrange_weights`, `poly_eval` and the inner products are not copied. +- The circuit's constraints (that the hash row's wires carry the message `builder_step_message` computes) are not modeled; only the message is. +- `AdditiveNttF64::standard(0)` panics while building an empty first table row. The supported constructor domain is `1 <= dim <= 63`, now stated in its production documentation; zero-dimensional construction is not proved or fixed here. + +## Reproduce + +Verus release `0.2026.10.04.426d8b0` (commit `426d8b01e7ffb910a36c15e1f870ff985bb61eef`, with its bundled Z3 +4.16.0, built against Rust 1.98.1), and `vstd = "=0.0.0-2026-10-04-0306"` from crates.io, the `vstd` of that +release. From the repository root: + +```bash +verification/verus/verify.sh # every proof; installs the pinned Verus under ~/.verus if missing +verification/verus/verify.sh gf2_64 # one module +(cd verification/verus && cargo test --release) # the verified copies against production +``` + +`verify.sh` downloads the pinned Linux x86-64 release if needed, installs its Rust toolchain with `rustup`, and checks every configuration listed above. Set `VERUS_HOME` to install elsewhere, `VERUS_THREADS` to control solver parallelism (default 4), and `VERUS_CONFIGS` to select configurations. Solver and equivalence counts belong in the PR verification report, not a performance claim. + +The script isolates whole-crate and per-module proof caches, because changing the forwarded module argument must not reuse a different module's cached result. `CARGO_TARGET_DIR` selects the cache root; configuration and proof-selection directories are appended. diff --git a/verification/verus/src/bit_fold.rs b/verification/verus/src/bit_fold.rs new file mode 100644 index 0000000000..55e032fefd --- /dev/null +++ b/verification/verus/src/bit_fold.rs @@ -0,0 +1,1014 @@ +//! Weighted sums of packed bit rows: the fold that turns witness bits into `E` values, and the GF(2)-linear +//! maps `E -> E` that ring switching applies. +//! +//! The executable functions are the portable paths of `crates/primitives/src/bit_fold.rs` and +//! `bit_fold/portable.rs`, copied with the same bodies where Verus accepts them; +//! `tests/equivalence/bit_fold.rs` checks the two agree. +//! +//! Specification: bit `s` of a row of bytes is bit `s % 8` of byte `s / 8`, and the fold of the row with +//! weights `w` is ([`fold_spec`]) +//! +//! ```text +//! fold(row) = sum_{s : bit s of row is set} w_s, +//! ``` +//! +//! the sum in `E` ([`e_add`]). An `E` value has 192 coordinate bits, bit `b` being bit `b % 64` of coefficient +//! `b / 64`, and the map with weights `w` sends `x` to the sum of the weights of its set coordinate bits +//! ([`map_spec`]): every GF(2)-linear map `E -> E` is of this form, `w_b` being the image of the `b`-th +//! coordinate vector ([`unit`]). +use crate::bits::*; +use crate::gf2_64::*; +use crate::gf2_64x3::*; +use vstd::prelude::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// `w` if `set`, else zero. +pub open spec fn pick(set: bool, w: F192) -> F192 { + if set { + w + } else { + F192::ZERO + } +} + +/// `sum_{i < n : bit i of v} w[i]`: the subset sum of the weights of a byte. +pub open spec fn byte_sum(w: Seq, v: u8, n: nat) -> F192 + decreases n, +{ + if n == 0 { + F192::ZERO + } else { + e_add(byte_sum(w, v, (n - 1) as nat), pick(bit8(v, n - 1), w[n - 1])) + } +} + +/// `sum_{s < n : bit s of row} w[s]`, bit `s` of the row being bit `s % 8` of byte `s / 8`. +pub open spec fn row_sum(w: Seq, row: Seq, n: nat) -> F192 + decreases n, +{ + if n == 0 { + F192::ZERO + } else { + e_add(row_sum(w, row, (n - 1) as nat), pick(bit8(row[(n - 1) / 8], (n - 1) % 8), w[n - 1])) + } +} + +/// The fold of a row: the sum of the weights of its set bits. +pub open spec fn fold_spec(w: Seq, row: Seq) -> F192 { + row_sum(w, row, 8 * row.len()) +} + +/// Coefficient `q < 3` of `x`. +pub open spec fn coeff(x: F192, q: int) -> u64 { + if q == 0 { + x.c0 + } else if q == 1 { + x.c1 + } else { + x.c2 + } +} + +/// Coordinate bit `b < 192` of `x`: bit `b % 64` of coefficient `b / 64`. +pub open spec fn coord(x: F192, b: int) -> bool { + bit64(coeff(x, b / 64), b % 64) +} + +/// `sum_{b < n : coordinate bit b of x} w[b]`. +pub open spec fn coord_sum(w: Seq, x: F192, n: nat) -> F192 + decreases n, +{ + if n == 0 { + F192::ZERO + } else { + e_add(coord_sum(w, x, (n - 1) as nat), pick(coord(x, n - 1), w[n - 1])) + } +} + +/// The GF(2)-linear map `E -> E` sending coordinate bit `b` to `w[b]`. +pub open spec fn map_spec(w: Seq, x: F192) -> F192 { + coord_sum(w, x, 192) +} + +/// The coordinate vector `b`: coordinate bit `b` set, every other clear. +pub open spec fn unit(b: int) -> F192 { + let one = 1u64 << ((b % 64) as u64); + F192 { + c0: if b / 64 == 0 { one } else { 0 }, + c1: if b / 64 == 1 { one } else { 0 }, + c2: if b / 64 == 2 { one } else { 0 }, + } +} + +/// The 24 bytes of `x`: the little-endian bytes of `c0`, then `c1`, then `c2`. +pub open spec fn le_row(x: F192) -> Seq { + Seq::new(24, |k: int| (coeff(x, k / 8) >> ((8 * (k % 8)) as u64)) as u8) +} + +/// `sum_{j < n} tables[j][row[j]]`: what the byte tables add up. +pub open spec fn tables_fold(tables: Seq<[F192; 256]>, row: Seq, n: nat) -> F192 + decreases n, +{ + if n == 0 { + F192::ZERO + } else { + e_add(tables_fold(tables, row, (n - 1) as nat), tables[n - 1][row[n - 1] as int]) + } +} + +// --------------------------------------------------------------------------------------------- +// Sums in E +// --------------------------------------------------------------------------------------------- +proof fn lemma_xor_laws(x: u64, y: u64, z: u64) + ensures + x ^ y == y ^ x, + (x ^ y) ^ z == x ^ (y ^ z), + x ^ 0 == x, + 0 ^ x == x, + x ^ x == 0, +{ + assert(x ^ y == y ^ x && (x ^ y) ^ z == x ^ (y ^ z) && x ^ 0 == x && 0 ^ x == x && x ^ x == 0) by (bit_vector); +} + +/// `E` under `+` is an abelian group of exponent 2. +pub proof fn lemma_e_add_laws(a: F192, b: F192, c: F192) + ensures + e_add(a, b) == e_add(b, a), + e_add(e_add(a, b), c) == e_add(a, e_add(b, c)), + e_add(a, F192::ZERO) == a, + e_add(F192::ZERO, a) == a, + e_add(a, a) == F192::ZERO, +{ + lemma_xor_laws(a.c0, b.c0, c.c0); + lemma_xor_laws(a.c1, b.c1, c.c1); + lemma_xor_laws(a.c2, b.c2, c.c2); +} + +/// `(a + b) + (c + d) = (a + c) + (b + d)`. +pub proof fn lemma_e_add_swap(a: F192, b: F192, c: F192, d: F192) + ensures + e_add(e_add(a, b), e_add(c, d)) == e_add(e_add(a, c), e_add(b, d)), +{ + assert(forall|x: u64, y: u64, z: u64, t: u64| #[trigger] ((x ^ y) ^ (z ^ t)) == (x ^ z) ^ (y ^ t)) + by (bit_vector); +} + +/// `pick(p xor q, w) = pick(p, w) + pick(q, w)`. +proof fn lemma_pick_xor(p: bool, q: bool, w: F192) + ensures + pick(p != q, w) == e_add(pick(p, w), pick(q, w)), +{ + lemma_e_add_laws(w, F192::ZERO, F192::ZERO); + lemma_e_add_laws(F192::ZERO, F192::ZERO, F192::ZERO); +} + +// --------------------------------------------------------------------------------------------- +// The byte tables +// --------------------------------------------------------------------------------------------- +/// Clearing a set bit `t` of `v` removes `w[t]` from its subset sum. +pub proof fn lemma_byte_sum_clear(w: Seq, v: u8, t: u8, n: nat) + requires + t < 8, + n <= 8, + bit8(v, t as int), + ensures + byte_sum(w, v, n) == e_add(byte_sum(w, v ^ (1u8 << t), n), pick(t < n, w[t as int])), + decreases n, +{ + let v2 = v ^ (1u8 << t); + if n == 0 { + lemma_e_add_laws(F192::ZERO, F192::ZERO, F192::ZERO); + } else { + let i = (n - 1) as nat; + lemma_byte_sum_clear(w, v, t, i); + let iu = i as u8; + assert(t < 8 && iu < 8 ==> ((((v ^ (1u8 << t)) >> iu) & 1 == 1) <==> (((v >> iu) & 1 == 1) != (iu + == t)))) by (bit_vector); + let (s, s2) = (byte_sum(w, v, i), byte_sum(w, v2, i)); + let q = pick(t < i, w[t as int]); + if i == t { + // s + w_t = (s2 + 0) + w_t. + lemma_e_add_laws(s2, F192::ZERO, w[t as int]); + lemma_e_add_laws(s2, w[t as int], F192::ZERO); + } else { + let p = pick(bit8(v, i as int), w[i as int]); + // (s2 + q) + p = (s2 + p) + q. + lemma_e_add_laws(s2, q, p); + lemma_e_add_laws(q, p, F192::ZERO); + lemma_e_add_laws(s2, p, q); + } + } +} + +/// The subset sum of the single bit `t` is `w[t]`. +pub proof fn lemma_byte_sum_monomial(w: Seq, t: u8, n: nat) + requires + t < 8, + n <= 8, + ensures + byte_sum(w, 1u8 << t, n) == pick(t < n, w[t as int]), + decreases n, +{ + if n == 0 { + } else { + let i = (n - 1) as nat; + lemma_byte_sum_monomial(w, t, i); + let iu = i as u8; + assert(t < 8 && iu < 8 ==> ((((1u8 << t) >> iu) & 1 == 1) <==> (iu == t))) by (bit_vector); + lemma_e_add_laws(pick(t < i, w[t as int]), F192::ZERO, F192::ZERO); + lemma_e_add_laws(w[t as int], F192::ZERO, F192::ZERO); + } +} + +/// The subset sum depends only on the first `n` weights. +proof fn lemma_byte_sum_ext(w1: Seq, w2: Seq, v: u8, n: nat) + requires + forall|i: int| 0 <= i < n ==> w1[i] == w2[i], + ensures + byte_sum(w1, v, n) == byte_sum(w2, v, n), + decreases n, +{ + if n > 0 { + lemma_byte_sum_ext(w1, w2, v, (n - 1) as nat); + } +} + +// --------------------------------------------------------------------------------------------- +// Folds by byte +// --------------------------------------------------------------------------------------------- +/// The bits `8j .. 8j + i` of a row are bits `0 .. i` of byte `j`. +proof fn lemma_row_sum_byte(w: Seq, row: Seq, j: int, i: nat) + requires + 0 <= j < row.len(), + 8 * j + 8 <= w.len(), + i <= 8, + ensures + row_sum(w, row, (8 * j + i) as nat) == e_add( + row_sum(w, row, (8 * j) as nat), + byte_sum(w.subrange(8 * j, 8 * j + 8), row[j], i), + ), + decreases i, +{ + let base = row_sum(w, row, (8 * j) as nat); + if i == 0 { + lemma_e_add_laws(base, F192::ZERO, F192::ZERO); + } else { + let k = (i - 1) as nat; + lemma_row_sum_byte(w, row, j, k); + let s = 8 * j + k; + assert(s / 8 == j && s % 8 == k) by (nonlinear_arith) + requires + s == 8 * j + k, + 0 <= k < 8, + 0 <= j, + ; + let p = pick(bit8(row[j], k as int), w[s]); + assert(w.subrange(8 * j, 8 * j + 8)[k as int] == w[s]); + lemma_e_add_laws(base, byte_sum(w.subrange(8 * j, 8 * j + 8), row[j], k), p); + } +} + +/// The byte tables of the weights `w` fold a row as [`fold_spec`] does. +pub proof fn lemma_tables_fold(tables: Seq<[F192; 256]>, w: Seq, row: Seq, n: nat) + requires + n <= row.len(), + n <= tables.len(), + 8 * row.len() <= w.len(), + forall|j: int, v: int| + 0 <= j < tables.len() && 0 <= v < 256 ==> #[trigger] tables[j][v] == byte_sum( + w.subrange(8 * j, 8 * j + 8), + v as u8, + 8, + ), + ensures + tables_fold(tables, row, n) == row_sum(w, row, 8 * n), + decreases n, +{ + if n > 0 { + let j = n - 1; + lemma_tables_fold(tables, w, row, (n - 1) as nat); + lemma_row_sum_byte(w, row, j, 8); + assert(tables[j][row[j] as int] == byte_sum(w.subrange(8 * j, 8 * j + 8), row[j], 8)); + } +} + +// --------------------------------------------------------------------------------------------- +// The map on E +// --------------------------------------------------------------------------------------------- +/// Index arithmetic: bit `s % 8` of byte `s / 8` of [`le_row`] is coordinate bit `s`. +proof fn lemma_le_row_bit(x: F192, s: int) + requires + 0 <= s < 192, + ensures + bit8(le_row(x)[s / 8], s % 8) == coord(x, s), +{ + let su = s as u64; + assert(su < 192 ==> (su / 8) / 8 == su / 64 && su % 64 == 8 * ((su / 8) % 8) + su % 8) by (bit_vector); + let (r, i) = (((s / 8) % 8) as u64, (s % 8) as u64); + let c = coeff(x, s / 64); + assert(r < 8 && i < 8 ==> ((((c >> (8 * r)) as u8) >> (i as u8)) & 1 == 1 <==> (c >> (8 * r + i)) & 1 == 1)) + by (bit_vector); +} + +/// The fold of the bytes of `x` is the map of `x`. +pub proof fn lemma_row_sum_le_row(w: Seq, x: F192, n: nat) + requires + n <= 192, + ensures + row_sum(w, le_row(x), n) == coord_sum(w, x, n), + decreases n, +{ + if n > 0 { + lemma_row_sum_le_row(w, x, (n - 1) as nat); + lemma_le_row_bit(x, n - 1); + } +} + +proof fn lemma_coeff_add(x: F192, y: F192, q: int) + ensures + coeff(e_add(x, y), q) == coeff(x, q) ^ coeff(y, q), +{ +} + +/// The map is GF(2)-linear: `map(x + y) = map(x) + map(y)`. +pub proof fn lemma_coord_sum_add(w: Seq, x: F192, y: F192, n: nat) + ensures + coord_sum(w, e_add(x, y), n) == e_add(coord_sum(w, x, n), coord_sum(w, y, n)), + decreases n, +{ + if n == 0 { + lemma_e_add_laws(F192::ZERO, F192::ZERO, F192::ZERO); + } else { + let b = n - 1; + lemma_coord_sum_add(w, x, y, (n - 1) as nat); + let (cx, cy) = (coeff(x, b / 64), coeff(y, b / 64)); + lemma_coeff_add(x, y, b / 64); + let r = (b % 64) as u64; + assert(((cx ^ cy) >> r) & 1 == 1 <==> (((cx >> r) & 1 == 1) != ((cy >> r) & 1 == 1))) by (bit_vector); + lemma_pick_xor(coord(x, b), coord(y, b), w[b]); + lemma_e_add_swap( + coord_sum(w, x, (n - 1) as nat), + coord_sum(w, y, (n - 1) as nat), + pick(coord(x, b), w[b]), + pick(coord(y, b), w[b]), + ); + } +} + +/// The map sends zero to zero. +pub proof fn lemma_coord_sum_zero(w: Seq, n: nat) + ensures + coord_sum(w, F192::ZERO, n) == F192::ZERO, + decreases n, +{ + if n > 0 { + lemma_coord_sum_zero(w, (n - 1) as nat); + let r = ((n - 1) % 64) as u64; + assert((0u64 >> r) & 1 != 1) by (bit_vector); + lemma_e_add_laws(F192::ZERO, F192::ZERO, F192::ZERO); + } +} + +/// The coordinate bits below `n` of `x`, as an element. +pub open spec fn low_part(x: F192, n: nat) -> F192 { + F192 { + c0: crate::clmul::low(x.c0, if n < 64 { n } else { 64 }), + c1: crate::clmul::low(x.c1, if n < 64 { 0 } else if n < 128 { (n - 64) as nat } else { 64 }), + c2: crate::clmul::low(x.c2, if n < 128 { 0 } else { (n - 128) as nat }), + } +} + +/// `sum_{b < n : coordinate bit b of x} unit(b)` is the coordinate bits of `x` below `n`. +proof fn lemma_units_low_part(x: F192, n: nat) + requires + n <= 192, + ensures + coord_sum(Seq::new(192, |b: int| unit(b)), x, n) == low_part(x, n), + decreases n, +{ + let units = Seq::new(192, |b: int| unit(b)); + assert(forall|a: u64| #[trigger] (a & ((1u64 << 0u64) - 1) as u64) == 0) by (bit_vector); + if n == 0 { + assert(crate::clmul::low(x.c0, 0) == 0 && crate::clmul::low(x.c1, 0) == 0 && crate::clmul::low(x.c2, 0) + == 0); + } else { + let b = n - 1; + lemma_units_low_part(x, b as nat); + let (q, r) = (b / 64, b % 64); + let prev = low_part(x, b as nat); + crate::clmul::lemma_low_step(coeff(x, q), r as nat); + lemma_xor_laws(prev.c0, 0, 0); + lemma_xor_laws(prev.c1, 0, 0); + lemma_xor_laws(prev.c2, 0, 0); + assert(units[b] == unit(b)); + assert(e_add(prev, pick(coord(x, b), unit(b))) == low_part(x, n)); + } +} + +/// Every element is the sum of the coordinate vectors of its set bits. +pub proof fn lemma_units_sum(x: F192) + ensures + map_spec(Seq::new(192, |b: int| unit(b)), x) == x, +{ + lemma_units_low_part(x, 192); +} + +/// `(a + b) c = a c + b c` in `E`. +pub proof fn lemma_e_mul_add_left(a: F192, b: F192, c: F192) + ensures + e_mul(e_add(a, b), c) == e_add(e_mul(a, c), e_mul(b, c)), +{ + lemma_k_mul_xor_left(a.c0, b.c0, c.c0); + lemma_k_mul_xor_left(a.c0, b.c0, c.c1); + lemma_k_mul_xor_left(a.c0, b.c0, c.c2); + lemma_k_mul_xor_left(a.c1, b.c1, c.c0); + lemma_k_mul_xor_left(a.c1, b.c1, c.c1); + lemma_k_mul_xor_left(a.c1, b.c1, c.c2); + lemma_k_mul_xor_left(a.c2, b.c2, c.c0); + lemma_k_mul_xor_left(a.c2, b.c2, c.c1); + lemma_k_mul_xor_left(a.c2, b.c2, c.c2); + let (a00, a01, a02) = (k_mul(a.c0, c.c0), k_mul(a.c0, c.c1), k_mul(a.c0, c.c2)); + let (a10, a11, a12) = (k_mul(a.c1, c.c0), k_mul(a.c1, c.c1), k_mul(a.c1, c.c2)); + let (a20, a21, a22) = (k_mul(a.c2, c.c0), k_mul(a.c2, c.c1), k_mul(a.c2, c.c2)); + let (b00, b01, b02) = (k_mul(b.c0, c.c0), k_mul(b.c0, c.c1), k_mul(b.c0, c.c2)); + let (b10, b11, b12) = (k_mul(b.c1, c.c0), k_mul(b.c1, c.c1), k_mul(b.c1, c.c2)); + let (b20, b21, b22) = (k_mul(b.c2, c.c0), k_mul(b.c2, c.c1), k_mul(b.c2, c.c2)); + // d0 + d3, d1 + d3 + d4, d2 + d4 of the sum are those of `a` plus those of `b`. + assert((a00 ^ b00) ^ ((a12 ^ b12) ^ (a21 ^ b21)) == (a00 ^ (a12 ^ a21)) ^ (b00 ^ (b12 ^ b21))) by (bit_vector); + assert(((a01 ^ b01) ^ (a10 ^ b10)) ^ ((a12 ^ b12) ^ (a21 ^ b21)) ^ (a22 ^ b22) == ((a01 ^ a10) ^ (a12 ^ a21) + ^ a22) ^ ((b01 ^ b10) ^ (b12 ^ b21) ^ b22)) by (bit_vector); + assert(((a02 ^ b02) ^ (a11 ^ b11) ^ (a20 ^ b20)) ^ (a22 ^ b22) == ((a02 ^ a11 ^ a20) ^ a22) ^ ((b02 ^ b11 + ^ b20) ^ b22)) by (bit_vector); +} + +/// `0 c = 0` in `E`. +pub proof fn lemma_e_mul_zero_left(c: F192) + ensures + e_mul(F192::ZERO, c) == F192::ZERO, +{ + lemma_k_mul_zero(c.c0); + lemma_k_mul_zero(c.c1); + lemma_k_mul_zero(c.c2); + assert(0u64 ^ 0u64 == 0u64 && 0u64 ^ 0u64 ^ 0u64 == 0u64) by (bit_vector); +} + +/// The map with weights `w'_b = map(unit(b) c)`, on the coordinate bits below `n` of `x`, is `map` of +/// `low_part(x, n) c`. +proof fn lemma_after_mul_upto(w: Seq, c: F192, x: F192, n: nat) + requires + n <= 192, + ensures + coord_sum(Seq::new(192, |b: int| map_spec(w, e_mul(unit(b), c))), x, n) == map_spec( + w, + e_mul(coord_sum(Seq::new(192, |b: int| unit(b)), x, n), c), + ), + decreases n, +{ + let w2 = Seq::new(192, |b: int| map_spec(w, e_mul(unit(b), c))); + let units = Seq::new(192, |b: int| unit(b)); + if n == 0 { + lemma_e_mul_zero_left(c); + lemma_coord_sum_zero(w, 192); + } else { + let b = n - 1; + lemma_after_mul_upto(w, c, x, b as nat); + let s = coord_sum(units, x, b as nat); + let p = pick(coord(x, b), unit(b)); + assert(units[b] == unit(b)); + assert(w2[b] == map_spec(w, e_mul(unit(b), c))); + lemma_e_mul_add_left(s, p, c); + lemma_coord_sum_add(w, e_mul(s, c), e_mul(p, c), 192); + if !coord(x, b) { + lemma_e_mul_zero_left(c); + lemma_coord_sum_zero(w, 192); + } + } +} + +/// The map whose weights are `map(unit(b) c)` is `x -> map(x c)`: what `F192Map::after_mul` builds. +pub proof fn lemma_after_mul(w: Seq, c: F192, x: F192) + ensures + map_spec(Seq::new(192, |b: int| map_spec(w, e_mul(unit(b), c))), x) == map_spec(w, e_mul(x, c)), +{ + lemma_after_mul_upto(w, c, x, 192); + lemma_units_sum(x); +} + +// --------------------------------------------------------------------------------------------- +// Executable code: the portable arm, `crates/primitives/src/bit_fold/portable.rs` +// --------------------------------------------------------------------------------------------- +/// The fold of one row through the byte tables: one lookup and one XOR per byte. +/// +/// Rewritten from `tables.try_into().expect("one table per byte")` and +/// `row.iter().zip(tables).fold(F192::ZERO, |acc, (&v, sums)| acc + sums[usize::from(v)])`: an index loop, the +/// length the `expect` checks a `requires`. +#[inline(always)] +fn fold_row_lookup(tables: &[[F192; 256]], row: &[u8; CHUNKS]) -> (r: F192) + requires + tables@.len() == CHUNKS, + ensures + r == tables_fold(tables@, row@, CHUNKS as nat), +{ + let mut acc = F192::ZERO; + let mut j = 0; + while j < CHUNKS + invariant + j <= CHUNKS, + tables@.len() == CHUNKS, + acc == tables_fold(tables@, row@, j as nat), + decreases CHUNKS - j, + { + acc = acc + tables[j][usize::from(row[j])]; + j += 1; + } + acc +} + +/// One 256-entry subset-sum table per byte of a row: entry `[j][v]` sums the weights of the set bits of `v` at byte `j`. +/// +/// Rewritten from `weights.as_chunks::<8>().0.iter().map(..).collect()`: a loop over the whole chunks, pushing +/// each table. `v.isolate_lowest_one()` and `low.trailing_zeros()` (on `usize`, which `vstd` does not specify) +/// become `(v as u8).trailing_zeros()` and `1 << tz`, the same bit since `v < 256`. +fn lookup_tables(weights: &[F192]) -> (r: Vec<[F192; 256]>) + ensures + r@.len() == weights@.len() / 8, + forall|j: int, v: int| + 0 <= j < r@.len() && 0 <= v < 256 ==> #[trigger] r@[j][v] == byte_sum( + weights@.subrange(8 * j, 8 * j + 8), + v as u8, + 8, + ), +{ + let mut out: Vec<[F192; 256]> = Vec::new(); + let n = weights.len() / 8; + let mut j = 0; + while j < n + invariant + n == weights@.len() / 8, + weights@.len() <= usize::MAX, + j <= n, + out@.len() == j, + forall|j2: int, v: int| + 0 <= j2 < j && 0 <= v < 256 ==> #[trigger] out@[j2][v] == byte_sum( + weights@.subrange(8 * j2, 8 * j2 + 8), + v as u8, + 8, + ), + decreases n - j, + { + let ghost w = weights@.subrange(8 * j, 8 * j + 8); + let mut sums = [F192::ZERO; 256]; + assert(sums[0] == byte_sum(w, 0, 8)) by { + assert((0u8 >> 0u8) & 1 != 1 && (0u8 >> 1u8) & 1 != 1 && (0u8 >> 2u8) & 1 != 1 && (0u8 >> 3u8) & 1 != 1 + && (0u8 >> 4u8) & 1 != 1 && (0u8 >> 5u8) & 1 != 1 && (0u8 >> 6u8) & 1 != 1 && (0u8 >> 7u8) & 1 != 1) + by (bit_vector); + reveal_with_fuel(byte_sum, 9); + lemma_e_add_laws(F192::ZERO, F192::ZERO, F192::ZERO); + } + // Each entry adds its lowest set bit's weight to an entry already built. + for v in 1..256usize + invariant + n == weights@.len() / 8, + weights@.len() <= usize::MAX, + j < n, + w == weights@.subrange(8 * j, 8 * j + 8), + forall|u: int| 0 <= u < v ==> #[trigger] sums[u] == byte_sum(w, u as u8, 8), + { + let tz = (v as u8).trailing_zeros() as usize; + let low = 1usize << tz; + proof { + vstd::std_specs::bits::axiom_u8_trailing_zeros(v as u8); + let (vb, t) = (v as u8, tz as u8); + assert(vb != 0); + assert(tz < 8 && t == tz as u8 && (vb >> t) & 1 == 1); + assert(1 <= v < 256 && tz < 8 && t == tz as u8 && vb == v as u8 && (vb >> t) & 1 == 1 ==> (v ^ (1usize + << tz)) < v && (v ^ (1usize << tz)) as u8 == vb ^ (1u8 << t)) by (bit_vector); + lemma_byte_sum_clear(w, vb, t, 8); + assert(8 * j + 8 <= weights@.len()); + assert(w[t as int] == weights@[8 * j + t]); + } + sums[v] = sums[v ^ low] + weights[8 * j + tz]; + } + out.push(sums); + j += 1; + } + out +} + +/// The byte tables. +#[derive(Clone, Debug)] +#[verifier::allow(autoderive_clone_without_spec)] +pub struct Imp { + /// One subset-sum table per byte of a row. + tables: Vec<[F192; 256]>, +} + +impl Imp { + /// The weight of bit `s`: the entry of the single bit `s % 8` in table `s / 8`. + pub closed spec fn weights(&self) -> Seq { + Seq::new((8 * self.tables@.len()) as nat, |s: int| self.tables@[s / 8][(1u8 << ((s % 8) as u8)) as int]) + } + + /// Every table holds the subset sums of its weights. + pub closed spec fn wf(&self) -> bool { + forall|j: int, v: int| + 0 <= j < self.tables@.len() && 0 <= v < 256 ==> #[trigger] self.tables@[j][v] == byte_sum( + self.weights().subrange(8 * j, 8 * j + 8), + v as u8, + 8, + ) + } + + /// Bytes per row. + pub closed spec fn n_tables(&self) -> nat { + self.tables@.len() + } + + pub fn new(weights: &[F192]) -> (r: Self) + ensures + r.wf(), + r.n_tables() == weights@.len() / 8, + r.weights() == weights@.subrange(0, 8 * (weights@.len() / 8) as int), + { + let r = Self { tables: lookup_tables(weights) }; + proof { + let n = weights@.len() / 8; + let w = weights@.subrange(0, 8 * n as int); + assert forall|s: int| 0 <= s < 8 * n implies #[trigger] r.weights()[s] == w[s] by { + let (j, i) = (s / 8, s % 8); + lemma_byte_sum_monomial(weights@.subrange(8 * j, 8 * j + 8), i as u8, 8); + } + assert(r.weights() =~= w); + assert forall|j: int, v: int| 0 <= j < n && 0 <= v < 256 implies #[trigger] r.tables@[j][v] == byte_sum( + r.weights().subrange(8 * j, 8 * j + 8), + v as u8, + 8, + ) by { + assert(r.weights().subrange(8 * j, 8 * j + 8) =~= weights@.subrange(8 * j, 8 * j + 8)); + } + } + r + } + + pub fn new_f192(weights: &[F192]) -> (r: Self) + ensures + r.wf(), + r.n_tables() == weights@.len() / 8, + r.weights() == weights@.subrange(0, 8 * (weights@.len() / 8) as int), + { + Self::new(weights) + } + + /// Rewritten from `for (o, row) in out.iter_mut().zip(rows)`: an index loop to the shorter length. + #[inline] + pub fn fold_block(&self, rows: &[[u8; CHUNKS]], out: &mut [F192; BLOCK]) + requires + self.wf(), + self.n_tables() == CHUNKS, + ensures + forall|p: int| + 0 <= p < BLOCK && p < rows@.len() ==> #[trigger] final(out)[p] == fold_spec(self.weights(), rows@[p]@), + forall|p: int| rows@.len() <= p < BLOCK ==> #[trigger] final(out)[p] == old(out)[p], + { + let mut p = 0; + while p < BLOCK && p < rows.len() + invariant + self.wf(), + self.n_tables() == CHUNKS, + p <= BLOCK, + p <= rows@.len(), + forall|q: int| 0 <= q < p ==> #[trigger] out[q] == fold_spec(self.weights(), rows@[q]@), + forall|q: int| p <= q < BLOCK ==> #[trigger] out[q] == old(out)[q], + decreases BLOCK - p, + { + let o = fold_row_lookup(self.tables.as_slice(), &rows[p]); + proof { + lemma_tables_fold(self.tables@, self.weights(), rows@[p as int]@, CHUNKS as nat); + } + out[p] = o; + p += 1; + } + } + + pub const fn slice(xs: &[F192; BLOCK]) -> (r: ImpSliced) + ensures + r == *xs, + { + *xs + } + + #[inline] + pub fn apply_sliced_add(&self, xs: &ImpSliced, out: &mut [F192]) + requires + self.wf(), + self.n_tables() == 24, + ensures + final(out).len() == old(out).len(), + forall|p: int| + 0 <= p < old(out).len() && p < BLOCK ==> #[trigger] final(out)[p] == e_add( + old(out)[p], + map_spec(self.weights(), xs[p]), + ), + forall|p: int| BLOCK <= p < old(out).len() ==> #[trigger] final(out)[p] == old(out)[p], + { + self.apply_add_f192(xs, out); + } + + /// Rewritten from `for (o, x) in out.iter_mut().zip(xs)`: an index loop to the shorter length; and from + /// `row[..8].copy_from_slice(&x.c0.to_le_bytes())` (the same for `c1`, `c2`): a loop storing byte `k` of each + /// coefficient, `vstd` specifying neither. + #[inline] + pub fn apply_add_f192(&self, xs: &[F192; BLOCK], out: &mut [F192]) + requires + self.wf(), + self.n_tables() == 24, + ensures + final(out).len() == old(out).len(), + forall|p: int| + 0 <= p < old(out).len() && p < BLOCK ==> #[trigger] final(out)[p] == e_add( + old(out)[p], + map_spec(self.weights(), xs[p]), + ), + forall|p: int| BLOCK <= p < old(out).len() ==> #[trigger] final(out)[p] == old(out)[p], + { + let mut p = 0; + while p < BLOCK && p < out.len() + invariant + self.wf(), + self.n_tables() == 24, + p <= BLOCK, + p <= out.len(), + out.len() == old(out).len(), + forall|q: int| + 0 <= q < p ==> #[trigger] out[q] == e_add(old(out)[q], map_spec(self.weights(), xs[q])), + forall|q: int| p <= q < out.len() ==> #[trigger] out[q] == old(out)[q], + decreases BLOCK - p, + { + let x = xs[p]; + let mut row = [0u8; 24]; + for k in 0..8 + invariant + forall|k2: int| 0 <= k2 < 24 && k2 % 8 < k ==> #[trigger] row[k2] == le_row(x)[k2], + { + row[k] = (x.c0 >> (8 * k as u64)) as u8; + row[8 + k] = (x.c1 >> (8 * k as u64)) as u8; + row[16 + k] = (x.c2 >> (8 * k as u64)) as u8; + } + assert(row@ =~= le_row(x)); + let image = fold_row_lookup(self.tables.as_slice(), &row); + proof { + lemma_tables_fold(self.tables@, self.weights(), row@, 24); + lemma_row_sum_le_row(self.weights(), x, 192); + } + out[p] += image; + p += 1; + } + } +} + +/// A block of values, as they are: production's `portable::Sliced`, renamed since the two modules are one here. +pub type ImpSliced = [F192; BLOCK]; + +// --------------------------------------------------------------------------------------------- +// Executable code: `crates/primitives/src/bit_fold.rs`, on the portable arm +// --------------------------------------------------------------------------------------------- +/// Rows folded per call. +pub const BLOCK: usize = 64; + +/// Production's row sizes: `n` bytes, `n` a power of two in `8..=128`. +pub open spec fn is_row_bytes(n: int) -> bool { + n == 8 || n == 16 || n == 32 || n == 64 || n == 128 +} + +/// The weights of every bit of a row, prepared for folding. +#[derive(Clone, Debug)] +#[verifier::allow(autoderive_clone_without_spec)] +pub struct BitFold { + /// Bytes per row. + n_chunks: usize, + /// The target's fold kernel data. + imp: Imp, +} + +impl BitFold { + pub closed spec fn wf(&self) -> bool { + self.imp.wf() && self.imp.n_tables() == self.n_chunks + } + + /// The weight of every bit of a row. + pub closed spec fn weights(&self) -> Seq { + self.imp.weights() + } + + pub closed spec fn spec_n_chunks(&self) -> usize { + self.n_chunks + } + + /// Prepare `weights`, one per bit of a row. + /// + /// Production's `assert!` that a row is 8 to 128 bytes, a power of two, is the `requires`: `vstd` specifies + /// neither `is_power_of_two` nor `RangeInclusive::contains`. + pub fn new(weights: &[F192]) -> (r: Self) + requires + weights@.len() % 8 == 0, + is_row_bytes((weights@.len() / 8) as int), + ensures + r.wf(), + r.weights() == weights@, + r.spec_n_chunks() == weights@.len() / 8, + { + let n_chunks = weights.len() / 8; + let r = Self { n_chunks, imp: Imp::new(weights) }; + assert(weights@.subrange(0, 8 * n_chunks as int) =~= weights@); + r + } + + /// Bytes per row. + pub const fn n_chunks(&self) -> (r: usize) + ensures + r == self.spec_n_chunks(), + { + self.n_chunks + } + + /// Fold up to 64 consecutive rows into `out[..rows.len()]`. + /// + /// Production's `debug_assert_eq!(CHUNKS, self.n_chunks)` and `assert!(rows.len() <= BLOCK)` are the + /// `requires`. + #[inline] + pub fn fold_block(&self, rows: &[[u8; CHUNKS]], out: &mut [F192; BLOCK]) + requires + self.wf(), + CHUNKS == self.spec_n_chunks(), + rows@.len() <= BLOCK, + ensures + forall|p: int| + 0 <= p < rows@.len() ==> #[trigger] final(out)[p] == fold_spec(self.weights(), rows@[p]@), + forall|p: int| rows@.len() <= p < BLOCK ==> #[trigger] final(out)[p] == old(out)[p], + { + self.imp.fold_block(rows, out); + } +} + +/// A GF(2)-linear map from F192 to F192, given by the image of each of its 192 coordinate bits. +#[derive(Clone, Debug)] +#[verifier::allow(autoderive_clone_without_spec)] +pub struct F192Map { + imp: Imp, +} + +impl F192Map { + pub closed spec fn wf(&self) -> bool { + self.imp.wf() && self.imp.n_tables() == 24 + } + + /// The image of each coordinate bit. + pub closed spec fn weights(&self) -> Seq { + self.imp.weights() + } + + /// The map sending coordinate bit `b` (bit `b % 64` of coefficient `b / 64`) to `weights[b]`. + /// + /// Production's `assert_eq!(weights.len(), 192)` is the `requires`. + pub fn new(weights: &[F192]) -> (r: Self) + requires + weights@.len() == 192, + ensures + r.wf(), + r.weights() == weights@, + { + let r = Self { imp: Imp::new_f192(weights) }; + assert(weights@.subrange(0, 192) =~= weights@); + r + } + + /// Add the image of each of `xs` to `out`. + /// + /// Production's `assert!(out.len() <= BLOCK)` is the `requires`. + #[inline] + pub fn apply_add(&self, xs: &[F192; BLOCK], out: &mut [F192]) + requires + self.wf(), + old(out).len() <= BLOCK, + ensures + final(out).len() == old(out).len(), + forall|p: int| + 0 <= p < old(out).len() ==> #[trigger] final(out)[p] == e_add( + old(out)[p], + map_spec(self.weights(), xs[p]), + ), + { + self.imp.apply_add_f192(xs, out); + } + + /// The map `x -> self(x * c)`, itself GF(2)-linear. + /// + /// Rewritten from `weights.chunks_mut(BLOCK).enumerate()` and `std::array::from_fn`: index loops; each + /// chunk's images are added into a zeroed `Vec` (as they are into the zeroed chunk) and copied to `weights`. + pub fn after_mul(&self, c: F192) -> (r: Self) + requires + self.wf(), + ensures + r.wf(), + r.weights() == Seq::new(192, |b: int| map_spec(self.weights(), e_mul(unit(b), c))), + forall|x: F192| #[trigger] map_spec(r.weights(), x) == map_spec(self.weights(), e_mul(x, c)), + { + let ghost target = Seq::new(192, |b: int| map_spec(self.weights(), e_mul(unit(b), c))); + let mut weights = [F192::ZERO; 192]; + let mut chunk = 0; + while chunk < 3 + invariant + self.wf(), + chunk <= 3, + target == Seq::new(192, |b: int| map_spec(self.weights(), e_mul(unit(b), c))), + forall|b: int| 0 <= b < BLOCK * chunk ==> #[trigger] weights[b] == target[b], + decreases 3 - chunk, + { + let mut xs = [F192::ZERO; BLOCK]; + for i in 0..BLOCK + invariant + chunk < 3, + forall|i2: int| 0 <= i2 < i ==> #[trigger] xs[i2] == e_mul(unit(BLOCK * chunk + i2), c), + { + let bit = BLOCK * chunk + i; + let mut words = [0u64; 3]; + words[bit / 64] = 1 << (bit % 64); + assert((F192 { c0: words[0], c1: words[1], c2: words[2] }) == unit(bit as int)); + xs[i] = F192::new(words[0], words[1], words[2]) * c; + } + let mut w: Vec = Vec::new(); + for i in 0..BLOCK + invariant + w@.len() == i, + forall|i2: int| 0 <= i2 < i ==> #[trigger] w@[i2] == F192::ZERO, + { + w.push(F192::ZERO); + } + self.apply_add(&xs, w.as_mut_slice()); + for i in 0..BLOCK + invariant + chunk < 3, + w@.len() == BLOCK, + target == Seq::new(192, |b: int| map_spec(self.weights(), e_mul(unit(b), c))), + forall|i2: int| + 0 <= i2 < BLOCK ==> #[trigger] w@[i2] == e_add(F192::ZERO, map_spec(self.weights(), xs[i2])), + forall|i2: int| 0 <= i2 < BLOCK ==> #[trigger] xs[i2] == e_mul(unit(BLOCK * chunk + i2), c), + forall|b: int| 0 <= b < BLOCK * chunk + i ==> #[trigger] weights[b] == target[b], + { + proof { + lemma_e_add_laws(map_spec(self.weights(), xs[i as int]), F192::ZERO, F192::ZERO); + } + weights[BLOCK * chunk + i] = w[i]; + } + chunk += 1; + } + let r = Self::new(&weights); + assert(r.weights() =~= target); + assert forall|x: F192| #[trigger] map_spec(r.weights(), x) == map_spec(self.weights(), e_mul(x, c)) by { + lemma_after_mul(self.weights(), c, x); + } + r + } + + /// Add the image of each value of `xs` to `out`. + /// + /// Production's `assert!(out.len() <= BLOCK)` is the `requires`. + #[inline] + pub fn apply_sliced_add(&self, xs: &Sliced, out: &mut [F192]) + requires + self.wf(), + old(out).len() <= BLOCK, + ensures + final(out).len() == old(out).len(), + forall|p: int| + 0 <= p < old(out).len() ==> #[trigger] final(out)[p] == e_add( + old(out)[p], + map_spec(self.weights(), xs.0[p]), + ), + { + self.imp.apply_sliced_add(&xs.0, out); + } +} + +/// A block of values in the layout the map reads, so that a block mapped many times is transposed once. +/// +/// Its field is public here (private in production) so that the specifications can name the block. +#[derive(Clone, Debug)] +#[verifier::allow(autoderive_clone_without_spec)] +pub struct Sliced(pub ImpSliced); + +impl Sliced { + /// The block `xs`. + pub fn new(xs: &[F192; BLOCK]) -> (r: Self) + ensures + r.0 == *xs, + { + Self(Imp::slice(xs)) + } +} + +} // verus! diff --git a/verification/verus/src/bits.rs b/verification/verus/src/bits.rs new file mode 100644 index 0000000000..6908d4f0bb --- /dev/null +++ b/verification/verus/src/bits.rs @@ -0,0 +1,1574 @@ +//! Bit transposes. +//! +//! The executable functions are copies of `crates/primitives/src/bits.rs`: the portable paths and the SIMD arms +//! of `bit_transpose_64bytes` (AVX-512 VBMI with GFNI, AVX2 with GFNI, AVX2, NEON), each under production's +//! `cfg`, so every build Verus checks proves the arms it compiles. `tests/equivalence/bits.rs` checks the copies +//! agree with production. Where Verus rejects the production form (`step_by`, const-generic inner functions, +//! `as_chunks_mut`, `array::from_fn`, `from_le_bytes`, `array::map`, function-local `const` items, loads and +//! stores through raw pointers), the copy spells out the same arithmetic in `while`/`for` loops or calls a +//! helper of `crate::intrinsics`; each such spot says what it replaces. +//! +//! Specification: a word is a row of bits, bit `i` of `x` being `(x >> i) & 1` ([`bit64`], [`bit8`]). Every +//! transpose is stated bit by bit: which input bit lands at which output bit. The arms of the 64-byte transpose +//! all prove [`is_bit_transpose_64bytes`], given the intrinsic specifications of `crate::intrinsics`. +#[cfg(all(target_arch = "aarch64", verus_keep_ghost))] +use crate::intrinsics::aarch64::*; +#[cfg(all(target_arch = "aarch64", verus_keep_ghost))] +use crate::intrinsics::aarch64_bits::*; +#[cfg(target_arch = "aarch64")] +use crate::intrinsics::aarch64_bits::{vld1q_u8_16, vld1q_u8_at, vst1q_u8_at}; +#[cfg(all(target_arch = "x86_64", verus_keep_ghost))] +use crate::intrinsics::x86::*; +#[cfg(all(target_arch = "x86_64", verus_keep_ghost))] +use crate::intrinsics::x86_bits::*; +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +use crate::intrinsics::x86_bits::{load128_bytes, load256_bytes_at, store256_bytes_at}; +#[cfg(all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"))] +use crate::intrinsics::x86_bits::{load512_bytes, store512_bytes}; +#[cfg(target_arch = "aarch64")] +use core::arch::aarch64::*; +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +use core::arch::x86_64::*; +use vstd::prelude::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// Bit `i` of a 64-bit word. +pub open spec fn bit64(x: u64, i: int) -> bool { + (x >> (i as u64)) & 1 == 1 +} + +/// Bit `i` of a byte. +pub open spec fn bit8(x: u8, i: int) -> bool { + (x >> (i as u8)) & 1 == 1 +} + +/// `after` is the transpose of the 64x64 bit matrix `before`, word `r` being row `r`. +pub open spec fn is_transpose_64x64(before: [u64; 64], after: [u64; 64]) -> bool { + forall|r: int, c: int| 0 <= r < 64 && 0 <= c < 64 ==> #[trigger] bit64(after[c], r) == bit64(before[r], c) +} + +/// Two words agreeing on bits `0..n` agree on their low `n` bits. +proof fn lemma_bit64_ext_upto(x: u64, y: u64, n: u64) + requires + n <= 64, + forall|i: int| 0 <= i < 64 ==> #[trigger] bit64(x, i) == bit64(y, i), + ensures + n < 64 ==> x & sub(1u64 << n, 1) == y & sub(1u64 << n, 1), + n == 64 ==> x == y, + decreases n, +{ + if n == 0 { + assert(x & sub(1u64 << 0u64, 1) == y & sub(1u64 << 0u64, 1)) by (bit_vector); + } else { + let m = sub(n, 1); + lemma_bit64_ext_upto(x, y, m); + assert(bit64(x, m as int) == bit64(y, m as int)); + assert(m < 64 && x & sub(1u64 << m, 1) == y & sub(1u64 << m, 1) && (((x >> m) & 1 == 1) == ((y >> m) & 1 + == 1)) ==> (add(m, 1) < 64 ==> x & sub(1u64 << add(m, 1), 1) == y & sub(1u64 << add(m, 1), 1)) && ( + add(m, 1) == 64 ==> x == y)) by (bit_vector); + } +} + +/// Two words with the same 64 bits are equal. +pub proof fn lemma_bit64_ext(x: u64, y: u64) + requires + forall|i: int| 0 <= i < 64 ==> #[trigger] bit64(x, i) == bit64(y, i), + ensures + x == y, +{ + lemma_bit64_ext_upto(x, y, 64); +} + +// --------------------------------------------------------------------------------------------- +// The 8x8 transpose +// --------------------------------------------------------------------------------------------- +/// The production `transpose_8x8_bits`, as a spec function. +pub open spec fn transpose_8x8_formula(x: u64) -> u64 { + let t = (x ^ (x >> 7u64)) & 0x00AA_00AA_00AA_00AAu64; + let x1 = x ^ (t ^ (t << 7u64)); + let t = (x1 ^ (x1 >> 14u64)) & 0x0000_CCCC_0000_CCCCu64; + let x2 = x1 ^ (t ^ (t << 14u64)); + let t = (x2 ^ (x2 >> 28u64)) & 0x0000_0000_F0F0_F0F0u64; + x2 ^ t ^ (t << 28u64) +} + +/// Bit `r * 8 + c` of the 8x8 transpose is bit `c * 8 + r` of its input. +pub proof fn lemma_transpose_8x8_bit(x: u64, r: u64, c: u64) + by (bit_vector) + requires + r < 8, + c < 8, + ensures + ((transpose_8x8_formula(x) >> add(mul(r, 8), c)) & 1 == 1) == ((x >> add(mul(c, 8), r)) & 1 == 1), +{ +} + +/// The 8x8 transpose is an involution. +pub proof fn lemma_transpose_8x8_involution(x: u64) + ensures + transpose_8x8_formula(transpose_8x8_formula(x)) == x, +{ + assert(transpose_8x8_formula(transpose_8x8_formula(x)) == x) by (bit_vector); +} + +// --------------------------------------------------------------------------------------------- +// The 64x64 transpose: one round +// --------------------------------------------------------------------------------------------- +/// The (distance, mask) pairs of the six rounds: the mask keeps the columns whose bit `j` is clear. +pub open spec fn is_round(j: u64, mask: u64) -> bool { + ||| j == 32 && mask == 0x0000_0000_FFFF_FFFFu64 + ||| j == 16 && mask == 0x0000_FFFF_0000_FFFFu64 + ||| j == 8 && mask == 0x00FF_00FF_00FF_00FFu64 + ||| j == 4 && mask == 0x0F0F_0F0F_0F0F_0F0Fu64 + ||| j == 2 && mask == 0x3333_3333_3333_3333u64 + ||| j == 1 && mask == 0x5555_5555_5555_5555u64 +} + +/// `base` is a multiple of `2j`. +pub open spec fn aligned(base: u64, j: u64) -> bool { + base & sub(add(j, j), 1) == 0 +} + +/// The row/column bits from `j` up: the bits a transpose has exchanged after the rounds `32, ..., j`. +pub open spec fn high_bits(j: u64) -> u64 { + 63u64 & !sub(j, 1) +} + +/// What one round writes to word `r`: the masked swap of the pair `(r & !j, r | j)` it belongs to. +pub open spec fn round_word(m0: [u64; 64], r: u64, j: u64, mask: u64) -> u64 { + let (lo, hi) = (r & !j, (r & !j) | j); + let t = ((m0[lo as int] >> j) ^ m0[hi as int]) & mask; + if r & j == 0 { + m0[r as int] ^ (t << j) + } else { + m0[r as int] ^ t + } +} + +/// One masked swap, bit by bit: the low word takes the high word's bits of the columns with bit `j` set, +/// shifted down by `j`, and the high word the low word's bits of the columns with bit `j` clear, shifted up. +proof fn lemma_round_pair(a: u64, b: u64, j: u64, mask: u64, c: u64) + by (bit_vector) + requires + is_round(j, mask), + c < 64, + ensures + ({ + let t = ((a >> j) ^ b) & mask; + &&& ((((a ^ (t << j)) >> c) & 1 == 1) == if c & j != 0 { + (b >> (c ^ j)) & 1 == 1 + } else { + (a >> c) & 1 == 1 + }) + &&& ((((b ^ t) >> c) & 1 == 1) == if c & j == 0 { + (a >> (c ^ j)) & 1 == 1 + } else { + (b >> c) & 1 == 1 + }) + }), +{ +} + +/// Index facts of [`round_word`], by bit-vector reasoning. +proof fn lemma_round_index(r: u64, c: u64, j: u64, mask: u64) + by (bit_vector) + requires + is_round(j, mask), + r < 64, + c < 64, + ensures + ({ + let (lo, hi, s) = (r & !j, (r & !j) | j, (r ^ c) & j); + &&& lo < 64 && hi < 64 && r ^ s < 64 && c ^ s < 64 && c ^ j < 64 + &&& r & j == 0 ==> lo == r && (if c & j != 0 { + r ^ s == hi && c ^ s == c ^ j + } else { + r ^ s == r && c ^ s == c + }) + &&& r & j != 0 ==> hi == r && (if c & j == 0 { + r ^ s == lo && c ^ s == c ^ j + } else { + r ^ s == r && c ^ s == c + }) + }), +{ +} + +/// [`round_word`] bit by bit: the round moves bit `(r ^ s, c ^ s)` to `(r, c)`, with `s = j` exactly when bit +/// `j` of the row and the column differ. +proof fn lemma_round_word_bit(m0: [u64; 64], r: u64, c: u64, j: u64, mask: u64) + requires + is_round(j, mask), + r < 64, + c < 64, + ensures + bit64(round_word(m0, r, j, mask), c as int) == bit64( + m0[(r ^ ((r ^ c) & j)) as int], + (c ^ ((r ^ c) & j)) as int, + ), +{ + lemma_round_index(r, c, j, mask); + let (lo, hi) = (r & !j, (r & !j) | j); + lemma_round_pair(m0[lo as int], m0[hi as int], j, mask, c); +} + +/// The first block is aligned, and a low half never exceeds its row. +proof fn lemma_round_start(j: u64, mask: u64, r: u64) + by (bit_vector) + requires + is_round(j, mask), + ensures + aligned(0, j), + r & !j <= r, +{ +} + +/// Index facts of the round's loops, by bit-vector reasoning. +proof fn lemma_round_loop(j: u64, mask: u64, base: u64, k: u64, r: u64) + by (bit_vector) + requires + is_round(j, mask), + aligned(base, j), + base < 64, + ensures + aligned(add(base, add(j, j)), j), + add(base, add(j, j)) <= 64, + // Within a block, the low half has bit `j` clear and its partner is `k + j`. + base <= k < add(base, j) ==> { + &&& k < 64 && k & j == 0 && add(k, j) < 64 && add(k, j) & j != 0 + &&& k & !j == k && add(k, j) & !j == k && (k & !j) | j == add(k, j) + }, + base <= k < add(base, j) ==> ((r & !j) < add(k, 1) <==> ((r & !j) < k || r == k || r == add(k, j))), + // No low half lies in the block's upper half. + (r & !j) < add(base, j) <==> (r & !j) < add(base, add(j, j)), +{ +} + +/// One round of [`transpose_64x64`]: for every block, swap the `j x j` blocks above and below the diagonal. +/// +/// The production's `for base in (0..64).step_by(2 * J)` is the `while` loop over `base` below; the +/// production nests this function inside `transpose_64x64`. +#[inline(always)] +fn round(m: &mut [u64; 64], mask: u64) + requires + is_round(J as u64, mask), + ensures + forall|r: int, c: int| + 0 <= r < 64 && 0 <= c < 64 ==> #[trigger] bit64(final(m)[r], c) == bit64( + old(m)[(r as u64 ^ ((r as u64 ^ c as u64) & J as u64)) as int], + (c as u64 ^ ((r as u64 ^ c as u64) & J as u64)) as int, + ), +{ + let ghost m0 = *m; + let ghost j = J as u64; + proof { + lemma_round_start(j, mask, 0); + } + let mut base: usize = 0; + while base < 64 + invariant + is_round(j, mask), + j == J as u64, + base <= 64, + base < 64 ==> aligned(base as u64, j), + forall|r: int| + 0 <= r < 64 ==> #[trigger] m[r] == if (r as u64 & !j) < base as u64 { + round_word(m0, r as u64, j, mask) + } else { + m0[r] + }, + decreases 64 - base, + { + for k in base..base + J + invariant + is_round(j, mask), + j == J as u64, + base < 64, + aligned(base as u64, j), + forall|r: int| + 0 <= r < 64 ==> #[trigger] m[r] == if (r as u64 & !j) < k as u64 { + round_word(m0, r as u64, j, mask) + } else { + m0[r] + }, + { + proof { + lemma_round_loop(j, mask, base as u64, k as u64, k as u64); + assert(m[k as int] == m0[k as int] && m[(k + J) as int] == m0[(k + J) as int]); + } + let ghost m1 = *m; + let t = ((m[k] >> J) ^ m[k + J]) & mask; + m[k] ^= t << J; + m[k + J] ^= t; + assert(m[k as int] == round_word(m0, k as u64, j, mask)); + assert(m[(k + J) as int] == round_word(m0, (k + J) as u64, j, mask)); + proof { + assert forall|r: int| 0 <= r < 64 implies #[trigger] m[r] == if (r as u64 & !j) < (k + 1) as u64 { + round_word(m0, r as u64, j, mask) + } else { + m0[r] + } by { + lemma_round_loop(j, mask, base as u64, k as u64, r as u64); + assert(m1[r] == if (r as u64 & !j) < k as u64 { + round_word(m0, r as u64, j, mask) + } else { + m0[r] + }); + } + } + } + proof { + assert forall|r: int| 0 <= r < 64 implies #[trigger] m[r] == if (r as u64 & !j) < (base + 2 * J) as u64 { + round_word(m0, r as u64, j, mask) + } else { + m0[r] + } by { + lemma_round_loop(j, mask, base as u64, 0, r as u64); + } + lemma_round_loop(j, mask, base as u64, 0, 0); + } + base += 2 * J; + } + proof { + assert forall|r: int, c: int| 0 <= r < 64 && 0 <= c < 64 implies #[trigger] bit64(m[r], c) == bit64( + m0[(r as u64 ^ ((r as u64 ^ c as u64) & j)) as int], + (c as u64 ^ ((r as u64 ^ c as u64) & j)) as int, + ) by { + lemma_round_start(j, mask, r as u64); + assert(m[r] == round_word(m0, r as u64, j, mask)); + lemma_round_word_bit(m0, r as u64, c as u64, j, mask); + } + } +} + +// --------------------------------------------------------------------------------------------- +// The 64x64 transpose: the six rounds +// --------------------------------------------------------------------------------------------- +/// After the rounds `32, ..., j`, bit `(r, c)` came from the position whose row and column bits from `j` up are +/// exchanged. +pub open spec fn stage(m0: [u64; 64], m: [u64; 64], j: u64) -> bool { + forall|r: int, c: int| + 0 <= r < 64 && 0 <= c < 64 ==> #[trigger] bit64(m[r], c) == bit64( + m0[(r as u64 ^ ((r as u64 ^ c as u64) & high_bits(j))) as int], + (c as u64 ^ ((r as u64 ^ c as u64) & high_bits(j))) as int, + ) +} + +/// Composing round `j` with the rounds before it exchanges bit `j` too. +proof fn lemma_stage_index(r: u64, c: u64, j: u64, mask: u64) + by (bit_vector) + requires + is_round(j, mask), + r < 64, + c < 64, + ensures + ({ + let s = (r ^ c) & j; + let (r1, c1) = (r ^ s, c ^ s); + let d1 = (r1 ^ c1) & high_bits(add(j, j)); + let d = (r ^ c) & high_bits(j); + &&& r1 < 64 && c1 < 64 && r ^ d < 64 && c ^ d < 64 + &&& r1 ^ d1 == r ^ d && c1 ^ d1 == c ^ d + }), +{ +} + +proof fn lemma_stage_step(m0: [u64; 64], m1: [u64; 64], m2: [u64; 64], j: u64, mask: u64) + requires + is_round(j, mask), + stage(m0, m1, add(j, j)), + forall|r: int, c: int| + 0 <= r < 64 && 0 <= c < 64 ==> #[trigger] bit64(m2[r], c) == bit64( + m1[(r as u64 ^ ((r as u64 ^ c as u64) & j)) as int], + (c as u64 ^ ((r as u64 ^ c as u64) & j)) as int, + ), + ensures + stage(m0, m2, j), +{ + assert forall|r: int, c: int| 0 <= r < 64 && 0 <= c < 64 implies #[trigger] bit64(m2[r], c) == bit64( + m0[(r as u64 ^ ((r as u64 ^ c as u64) & high_bits(j))) as int], + (c as u64 ^ ((r as u64 ^ c as u64) & high_bits(j))) as int, + ) by { + lemma_stage_index(r as u64, c as u64, j, mask); + let s = (r as u64 ^ c as u64) & j; + let (r1, c1) = ((r as u64 ^ s) as int, (c as u64 ^ s) as int); + assert(bit64(m1[r1], c1) == bit64( + m0[(r1 as u64 ^ ((r1 as u64 ^ c1 as u64) & high_bits(add(j, j)))) as int], + (c1 as u64 ^ ((r1 as u64 ^ c1 as u64) & high_bits(add(j, j)))) as int, + )); + } +} + +/// Transpose the 64x64 bit matrix whose word `r` is row `r`, in place. +/// +/// ```text +/// after: m[c] bit r = before: m[r] bit c +/// ``` +/// +/// Six rounds of masked swaps (Hacker's Delight, section 7-3), halving the block each round. +/// Round `j` swaps the upper-right and lower-left `j x j` blocks of every `2j x 2j` block on the diagonal. +/// The rounds with `j >= 8` pair whole runs of words, which the compiler vectorizes. +#[inline] +pub fn transpose_64x64(m: &mut [u64; 64]) + ensures + is_transpose_64x64(*old(m), *final(m)), +{ + let ghost m0 = *m; + proof { + assert forall|r: int, c: int| 0 <= r < 64 && 0 <= c < 64 implies #[trigger] bit64(m0[r], c) == bit64( + m0[(r as u64 ^ ((r as u64 ^ c as u64) & high_bits(64))) as int], + (c as u64 ^ ((r as u64 ^ c as u64) & high_bits(64))) as int, + ) by { + let (r, c) = (r as u64, c as u64); + assert(r < 64 && c < 64 ==> r ^ ((r ^ c) & high_bits(64)) == r && c ^ ((r ^ c) & high_bits(64)) == c) + by (bit_vector); + } + } + let ghost m1 = *m; + round::<32>(m, 0x0000_0000_FFFF_FFFF); + proof { + lemma_stage_step(m0, m1, *m, 32, 0x0000_0000_FFFF_FFFF); + } + let ghost m1 = *m; + round::<16>(m, 0x0000_FFFF_0000_FFFF); + proof { + lemma_stage_step(m0, m1, *m, 16, 0x0000_FFFF_0000_FFFF); + } + let ghost m1 = *m; + round::<8>(m, 0x00FF_00FF_00FF_00FF); + proof { + lemma_stage_step(m0, m1, *m, 8, 0x00FF_00FF_00FF_00FF); + } + let ghost m1 = *m; + round::<4>(m, 0x0F0F_0F0F_0F0F_0F0F); + proof { + lemma_stage_step(m0, m1, *m, 4, 0x0F0F_0F0F_0F0F_0F0F); + } + let ghost m1 = *m; + round::<2>(m, 0x3333_3333_3333_3333); + proof { + lemma_stage_step(m0, m1, *m, 2, 0x3333_3333_3333_3333); + } + let ghost m1 = *m; + round::<1>(m, 0x5555_5555_5555_5555); + proof { + lemma_stage_step(m0, m1, *m, 1, 0x5555_5555_5555_5555); + assert forall|r: int, c: int| 0 <= r < 64 && 0 <= c < 64 implies #[trigger] bit64(m[c], r) == bit64( + m0[r], + c, + ) by { + let (r, c) = (r as u64, c as u64); + assert(r < 64 && c < 64 ==> c ^ ((c ^ r) & high_bits(1)) == r && r ^ ((c ^ r) & high_bits(1)) == c) + by (bit_vector); + } + } +} + +/// Transposing twice gives back the matrix. +pub proof fn lemma_transpose_64x64_involution(a: [u64; 64], b: [u64; 64], c: [u64; 64]) + requires + is_transpose_64x64(a, b), + is_transpose_64x64(b, c), + ensures + c == a, +{ + assert forall|r: int| 0 <= r < 64 implies c[r] == a[r] by { + assert forall|i: int| 0 <= i < 64 implies #[trigger] bit64(c[r], i) == bit64(a[r], i) by { + assert(bit64(c[r], i) == bit64(b[i], r)); + assert(bit64(b[i], r) == bit64(a[r], i)); + } + lemma_bit64_ext(c[r], a[r]); + } + assert(c =~= a); +} + +// --------------------------------------------------------------------------------------------- +// The 64-byte transpose +// --------------------------------------------------------------------------------------------- +/// Transpose the 8x8 bit matrix whose byte `r` is row `r` (Hacker's Delight, section 7-3). +/// +/// Bit `r * 8 + c` moves to bit `c * 8 + r`. +#[inline(always)] +pub const fn transpose_8x8_bits(mut x: u64) -> (y: u64) + ensures + y == transpose_8x8_formula(x), + forall|r: int, c: int| 0 <= r < 8 && 0 <= c < 8 ==> #[trigger] bit64(y, r * 8 + c) == bit64(x, c * 8 + r), +{ + let ghost x0 = x; + let t = (x ^ (x >> 7)) & 0x00AA_00AA_00AA_00AA; + x ^= t ^ (t << 7); + let t = (x ^ (x >> 14)) & 0x0000_CCCC_0000_CCCC; + x ^= t ^ (t << 14); + let t = (x ^ (x >> 28)) & 0x0000_0000_F0F0_F0F0; + proof { + assert forall|r: int, c: int| 0 <= r < 8 && 0 <= c < 8 implies #[trigger] bit64( + transpose_8x8_formula(x0), + r * 8 + c, + ) == bit64(x0, c * 8 + r) by { + lemma_transpose_8x8_bit(x0, r as u64, c as u64); + } + } + x ^ t ^ (t << 28) +} + +/// Gathering byte `v` into byte `x` of a word whose bytes from `x` up are clear. +proof fn lemma_gather_byte(column: u64, v: u8, x: u64, x2: u64, t: u64) + by (bit_vector) + requires + x < 8, + t < 8, + column >> mul(8, x) == 0, + ensures + add(x, 1) < 8 ==> (column | ((v as u64) << mul(8, x))) >> mul(8, add(x, 1)) == 0, + x2 < x ==> ((((column | ((v as u64) << mul(8, x))) >> add(mul(x2, 8), t)) & 1 == 1) == ((column >> add( + mul(x2, 8), + t, + )) & 1 == 1)), + (((column | ((v as u64) << mul(8, x))) >> add(mul(x, 8), t)) & 1 == 1) == ((v >> (t as u8)) & 1 == 1), +{ +} + +/// Byte `t` of a word, bit `x`, is bit `t * 8 + x` of the word. +proof fn lemma_scatter_byte(row: u64, t: u64, x: u64) + by (bit_vector) + requires + t < 8, + x < 8, + ensures + ((((row >> mul(8, t)) as u8) >> (x as u8)) & 1 == 1) == ((row >> add(mul(t, 8), x)) & 1 == 1), +{ +} + +/// What every arm of `bit_transpose_64bytes` computes: bit `x` of `output[8b + t]` is bit `t` of `input[8x + b]`. +/// +/// ```text +/// output[b * 8 + t] bit x = input[x * 8 + b] bit t +/// ``` +pub open spec fn is_bit_transpose_64bytes(input: [u8; 64], output: [u8; 64]) -> bool { + forall|b: int, t: int, x: int| + 0 <= b < 8 && 0 <= t < 8 && 0 <= x < 8 ==> #[trigger] bit8(output[b * 8 + t], x) == bit8(input[x * 8 + b], t) +} + +/// Gather each byte column into a word, and transpose its bits. +/// +/// ```text +/// output[b * 8 + t] bit x = input[x * 8 + b] bit t +/// ``` +/// +/// The production iterates `output.as_chunks_mut::<8>()`, gathers the column with +/// `u64::from_le_bytes(std::array::from_fn(|x| input[8 * x + b]))` and stores the row with `to_le_bytes`; the +/// copy spells out the same gather and store as loops over the bytes. Production's function is private. +#[inline] +pub fn bit_transpose_64bytes_portable(input: &[u8; 64], output: &mut [u8; 64]) + ensures + is_bit_transpose_64bytes(*input, *final(output)), +{ + for b in 0..8 + invariant + forall|b2: int, t: int, x: int| + 0 <= b2 < b && 0 <= t < 8 && 0 <= x < 8 ==> #[trigger] bit8(output[b2 * 8 + t], x) == bit8( + input[x * 8 + b2], + t, + ), + { + // `u64::from_le_bytes(std::array::from_fn(|x| input[8 * x + b]))`: byte `x` of the column is + // `input[8 * x + b]`. + let mut column = 0u64; + assert(0u64 >> mul(8, 0u64) == 0) by (bit_vector); + for x in 0..8 + invariant + 0 <= b < 8, + x < 8 ==> column >> mul(8, x as u64) == 0, + forall|x2: int, t: int| + 0 <= x2 < x && 0 <= t < 8 ==> #[trigger] bit64(column, x2 * 8 + t) == bit8(input[x2 * 8 + b], t), + { + let ghost c0 = column; + column |= (input[8 * x + b] as u64) << (8 * x); + proof { + let v = input[8 * x + b]; + lemma_gather_byte(c0, v, x as u64, 0, 0); + assert forall|x2: int, t: int| 0 <= x2 < x + 1 && 0 <= t < 8 implies #[trigger] bit64( + column, + x2 * 8 + t, + ) == bit8(input[x2 * 8 + b], t) by { + lemma_gather_byte(c0, v, x as u64, x2 as u64, t as u64); + if x2 < x { + assert(bit64(c0, x2 * 8 + t) == bit8(input[x2 * 8 + b], t)); + } + } + } + } + let row = transpose_8x8_bits(column); + // `*row = transpose_8x8_bits(column).to_le_bytes()`: byte `t` of the row is `output[8 * b + t]`. + for t in 0..8 + invariant + 0 <= b < 8, + forall|r: int, c: int| 0 <= r < 8 && 0 <= c < 8 ==> #[trigger] bit64(row, r * 8 + c) == bit64(column, c * 8 + r), + forall|x2: int, t2: int| + 0 <= x2 < 8 && 0 <= t2 < 8 ==> #[trigger] bit64(column, x2 * 8 + t2) == bit8(input[x2 * 8 + b], t2), + forall|b2: int, t2: int, x: int| + 0 <= b2 < b && 0 <= t2 < 8 && 0 <= x < 8 ==> #[trigger] bit8(output[b2 * 8 + t2], x) == bit8( + input[x * 8 + b2], + t2, + ), + forall|t2: int, x: int| + 0 <= t2 < t && 0 <= x < 8 ==> #[trigger] bit8(output[b * 8 + t2], x) == bit8(input[x * 8 + b], t2), + { + output[8 * b + t] = (row >> (8 * t)) as u8; + proof { + assert forall|x: int| 0 <= x < 8 implies #[trigger] bit8(output[b * 8 + t], x) == bit8( + input[x * 8 + b], + t as int, + ) by { + lemma_scatter_byte(row, t as u64, x as u64); + assert(bit64(row, t * 8 + x) == bit64(column, x * 8 + t)); + assert(bit64(column, x * 8 + t) == bit8(input[x * 8 + b], t as int)); + } + } + } + proof { + assert forall|b2: int, t: int, x: int| 0 <= b2 < b + 1 && 0 <= t < 8 && 0 <= x < 8 implies #[trigger] bit8( + output[b2 * 8 + t], + x, + ) == bit8(input[x * 8 + b2], t) by { + if b2 == b { + assert(bit8(output[b * 8 + t], x) == bit8(input[x * 8 + b], t)); + } + } + } + } +} + +/// Transpose the eight 8x8 bit matrices of a 64-byte block, gathered across rows. +/// +/// ```text +/// output[b * 8 + t] bit x = input[x * 8 + b] bit t +/// ``` +/// +/// So byte column `b` of the eight input rows becomes output row `b`, bit-transposed. +/// +/// - AVX-512 VBMI with GFNI: one byte permute and one affine map. +/// - AVX2 with GFNI: a byte gather in shuffles and unpacks, then one affine map. +/// - AVX2: the same gather, then three masked-swap rounds. +/// - aarch64: a table lookup, then three masked-swap rounds. +/// - Elsewhere: the same rounds on a gathered word per column. +#[inline] +pub fn bit_transpose_64bytes(input: &[u8; 64], output: &mut [u8; 64]) + ensures + is_bit_transpose_64bytes(*input, *final(output)), +{ + #[cfg(all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"))] + // SAFETY: the features are enabled at compile time. + unsafe { + bit_transpose_64bytes_gfni(input, output); + } + #[cfg(all( + target_arch = "x86_64", + target_feature = "avx2", + target_feature = "gfni", + not(target_feature = "avx512vbmi") + ))] + // SAFETY: the features are enabled at compile time. + unsafe { + bit_transpose_64bytes_gfni_avx2(input, output); + } + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", not(target_feature = "gfni")))] + // SAFETY: the feature is enabled at compile time. + unsafe { + bit_transpose_64bytes_avx2(input, output); + } + #[cfg(target_arch = "aarch64")] + // SAFETY: aarch64 always has NEON. + unsafe { + bit_transpose_64bytes_neon(input, output); + } + #[cfg(not(any( + all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"), + all(target_arch = "x86_64", target_feature = "avx2"), + target_arch = "aarch64" + )))] + bit_transpose_64bytes_portable(input, output); +} + +// --------------------------------------------------------------------------------------------- +// The SIMD arms: shared specifications and lemmas +// --------------------------------------------------------------------------------------------- +/// Byte `t` of a word, little-endian. +pub open spec fn byte_of(w: u64, t: int) -> u8 { + (w >> ((8 * t) as u64)) as u8 +} + +/// `column` holds byte column `b` of the input in row order: its byte `x` is `input[8x + b]`. +pub open spec fn is_column(input: [u8; 64], b: int, column: u64) -> bool { + forall|x: int| 0 <= x < 8 ==> #[trigger] byte_of(column, x) == input[8 * x + b] +} + +/// `column` holds byte column `b` of the input in reversed row order: its byte `y` is `input[8 (7 - y) + b]`. +pub open spec fn is_reversed_column(input: [u8; 64], b: int, column: u64) -> bool { + forall|y: int| 0 <= y < 8 ==> #[trigger] byte_of(column, y) == input[8 * (7 - y) + b] +} + +proof fn lemma_cases_4(k: int) + requires + 0 <= k < 4, + ensures + k == 0 || k == 1 || k == 2 || k == 3, +{ +} + +proof fn lemma_cases_8(k: int) + requires + 0 <= k < 8, + ensures + k == 0 || k == 1 || k == 2 || k == 3 || k == 4 || k == 5 || k == 6 || k == 7, +{ +} + +proof fn lemma_cases_16(k: int) + requires + 0 <= k < 16, + ensures + k == 0 || k == 1 || k == 2 || k == 3 || k == 4 || k == 5 || k == 6 || k == 7 || k == 8 || k == 9 || k == 10 + || k == 11 || k == 12 || k == 13 || k == 14 || k == 15, +{ +} + +proof fn lemma_cases_32(k: int) + requires + 0 <= k < 32, + ensures + k == 0 || k == 1 || k == 2 || k == 3 || k == 4 || k == 5 || k == 6 || k == 7 || k == 8 || k == 9 || k == 10 + || k == 11 || k == 12 || k == 13 || k == 14 || k == 15 || k == 16 || k == 17 || k == 18 || k == 19 || k + == 20 || k == 21 || k == 22 || k == 23 || k == 24 || k == 25 || k == 26 || k == 27 || k == 28 || k == 29 + || k == 30 || k == 31, +{ +} + +/// Byte `8j + t` of a run of words is byte `t` of word `j`. +proof fn lemma_div8(j: int, t: int) + requires + 0 <= t < 8, + ensures + (8 * j + t) / 8 == j, + (8 * j + t) % 8 == t, +{ + vstd::arithmetic::div_mod::lemma_fundamental_div_mod_converse(8 * j + t, 8, j, t); +} + +/// The 8x8 transpose of column `b` is output row `b`: bit `x` of its byte `t` is bit `t` of `input[8x + b]`. +proof fn lemma_transposed_column(input: [u8; 64], b: int, column: u64, t: int) + requires + 0 <= b < 8, + 0 <= t < 8, + is_column(input, b, column), + ensures + forall|x: int| + 0 <= x < 8 ==> #[trigger] bit8(byte_of(transpose_8x8_formula(column), t), x) == bit8(input[x * 8 + b], t), +{ + let row = transpose_8x8_formula(column); + assert forall|x: int| 0 <= x < 8 implies #[trigger] bit8(byte_of(row, t), x) == bit8(input[x * 8 + b], t) by { + lemma_scatter_byte(row, t as u64, x as u64); + lemma_transpose_8x8_bit(column, t as u64, x as u64); + lemma_scatter_byte(column, x as u64, t as u64); + assert(byte_of(column, x) == input[8 * x + b]); + } +} + +/// The masked swap of one round of [`transpose_8x8_bits`] on a word: `t = (x ^ (x >> d)) & mask`, then +/// `x ^ (t ^ (t << d))`. +pub open spec fn swap_word(x: u64, d: u64, mask: u64) -> u64 { + let t = (x ^ (x >> d)) & mask; + x ^ (t ^ (t << d)) +} + +/// The three masked swaps of the AVX2 and NEON arms are [`transpose_8x8_formula`]. +pub proof fn lemma_three_swaps(x: u64) + ensures + swap_word(swap_word(swap_word(x, 7, 0x00AA_00AA_00AA_00AAu64), 14, 0x0000_CCCC_0000_CCCCu64), 28, 0x0000_0000_F0F0_F0F0u64) + == transpose_8x8_formula(x), +{ + assert(swap_word(swap_word(swap_word(x, 7, 0x00AA_00AA_00AA_00AAu64), 14, 0x0000_CCCC_0000_CCCCu64), 28, 0x0000_0000_F0F0_F0F0u64) + == transpose_8x8_formula(x)) by (bit_vector); +} + +/// The unit matrix constant of the GFNI arms, as `i64` and back. +proof fn lemma_unit_word() + ensures + (0x8040_2010_0804_0201u64 as i64) as u64 == 0x8040_2010_0804_0201u64, +{ + assert((0x8040_2010_0804_0201u64 as i64) as u64 == 0x8040_2010_0804_0201u64) by (bit_vector); +} + +/// Byte `s` of the unit matrix constant `0x8040_2010_0804_0201` is `1 << s`. +proof fn lemma_unit_byte(s: u64) + by (bit_vector) + requires + s < 8, + ensures + (0x8040_2010_0804_0201u64 >> mul(8, s)) as u8 == 1u8 << (s as u8), +{ +} + +/// A byte below 16 selects itself in a byte shuffle. +proof fn lemma_small_byte(v: u8) + by (bit_vector) + requires + v < 16, + ensures + v & 0x80 == 0, + v & 15 == v, + v & 63 == v, +{ +} + +/// GF2P8AFFINEQB of the byte `1 << t` with matrix word `a`: bit `x` of the result is bit `t` of byte `7 - x` +/// of `a`. +#[cfg(target_arch = "x86_64")] +proof fn lemma_affine_unit(a: u64, t: u64, x: u64) + by (bit_vector) + requires + t < 8, + x < 8, + ensures + ((affine_byte(a, 1u8 << (t as u8), 0) >> (x as u8)) & 1 == 1) == ((((a >> mul(8, sub(7, x))) as u8) >> (t as u8)) + & 1 == 1), +{ +} + +/// A reversed column through the unit affine map is output row `b`: bit `x` of byte `t` is bit `t` of +/// `input[8x + b]`. +#[cfg(target_arch = "x86_64")] +proof fn lemma_affine_column(input: [u8; 64], b: int, column: u64, t: int) + requires + 0 <= b < 8, + 0 <= t < 8, + is_reversed_column(input, b, column), + ensures + forall|x: int| + 0 <= x < 8 ==> #[trigger] bit8(affine_byte(column, 1u8 << (t as u8), 0), x) == bit8(input[x * 8 + b], t), +{ + assert forall|x: int| 0 <= x < 8 implies #[trigger] bit8(affine_byte(column, 1u8 << (t as u8), 0), x) == bit8( + input[x * 8 + b], + t, + ) by { + lemma_affine_unit(column, t as u64, x as u64); + assert(byte_of(column, 7 - x) == input[8 * (7 - (7 - x)) + b]); + } +} + +// --------------------------------------------------------------------------------------------- +// AVX-512 VBMI with GFNI +// --------------------------------------------------------------------------------------------- +/// Word `b`, byte `j`: input row `7 - j`, column `b`. +/// +/// Production builds this function-local `const` with a `while` loop, entry `i` being `(7 - i % 8) * 8 + i / 8`; +/// here it is a module-level `const` with the 64 entries written out, which [`lemma_idx`] reads as that formula. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"))] +pub const IDX: [u8; 64] = [ + 56, 48, 40, 32, 24, 16, 8, 0, 57, 49, 41, 33, 25, 17, 9, 1, 58, 50, 42, 34, 26, 18, 10, 2, 59, 51, 43, 35, 27, 19, + 11, 3, 60, 52, 44, 36, 28, 20, 12, 4, 61, 53, 45, 37, 29, 21, 13, 5, 62, 54, 46, 38, 30, 22, 14, 6, 63, 55, 47, + 39, 31, 23, 15, 7, +]; + +/// Entry `8b + y` of [`IDX`] is byte `b` of input row `7 - y`. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"))] +proof fn lemma_idx(b: int, y: int) + requires + 0 <= b < 8, + 0 <= y < 8, + ensures + IDX[8 * b + y] as int == 8 * (7 - y) + b, +{ + lemma_cases_8(b); + lemma_cases_8(y); +} + +/// `vpermb` gathers each byte column into a word, rows reversed. +/// +/// `vgf2p8affineqb` then computes, per byte `j` of the constant and word `A`: +/// +/// ```text +/// result byte j bit i = parity(A byte (7 - i) & constant byte j) +/// ``` +/// +/// With constant byte `j = 1 << j`, that is bit `j` of `A`'s byte `7 - i`: the reversal undoes it. +/// +/// Rewritten for Verus: the loads and the store through `as_ptr().cast()` are the helpers `load512_bytes` and +/// `store512_bytes`, whose bodies are those calls; `IDX` is a module-level `const` (see [`IDX`]). Production's +/// function is private. +#[cfg(all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"))] +#[target_feature(enable = "avx512f,avx512vbmi,gfni")] +pub unsafe fn bit_transpose_64bytes_gfni(input: &[u8; 64], output: &mut [u8; 64]) + ensures + is_bit_transpose_64bytes(*input, *final(output)), +{ + // SAFETY: each load and store is one 64-byte array. + unsafe { + let columns = _mm512_permutexvar_epi8(load512_bytes(&IDX), load512_bytes(input)); + let unit = _mm512_set1_epi64(0x8040_2010_0804_0201_u64 as i64); + let rows = _mm512_gf2p8affine_epi64_epi8::<0>(unit, columns); + proof { + broadcast use axiom_m512_bytes; + + lemma_unit_word(); + assert forall|b: int, t: int, x: int| 0 <= b < 8 && 0 <= t < 8 && 0 <= x < 8 implies #[trigger] bit8( + m512_bytes(rows)[b * 8 + t], + x, + ) == bit8(input[x * 8 + b], t) by { + let column = m512(columns)[b]; + assert forall|y: int| 0 <= y < 8 implies #[trigger] byte_of(column, y) == input[8 * (7 - y) + b] by { + lemma_idx(b, y); + assert(IDX[8 * b + y] & 63 == IDX[8 * b + y]) by { + let v = IDX[8 * b + y]; + assert(v < 64 ==> v & 63 == v) by (bit_vector); + } + lemma_div8(b, y); + assert(m512_bytes(columns)[8 * b + y] == input[8 * (7 - y) + b]); + } + lemma_div8(b, t); + lemma_unit_byte(t as u64); + assert(m512_bytes(unit)[8 * b + t] == 1u8 << (t as u8)); + lemma_affine_column(*input, b, column, t); + } + } + store512_bytes(output, rows); + } +} + +// --------------------------------------------------------------------------------------------- +// AVX2: the column gather +// --------------------------------------------------------------------------------------------- +/// The word `_mm256_permute4x64_epi64::<0b11_01_10_00>` puts at word `i`. +pub open spec fn permute_d8(i: int) -> int { + if i == 1 { + 2 + } else if i == 2 { + 1 + } else { + i + } +} + +/// The input byte at byte `k` of register `w` after the two unpack rounds of [`gather_columns_avx2`]: lane +/// `k / 16` holds rows `0, 1, 4, 5` (lane 0) or `2, 3, 6, 7` (lane 1), dword `(k % 16) / 4` of it column +/// `4w + (k % 16) / 4`. +pub open spec fn unpacked_src(w: int, k: int) -> int { + 16 * (k / 16) + 4 * w + (k % 16) / 4 + 8 * (k % 2) + 32 * ((k / 2) % 2) +} + +/// The input byte at byte `k` of register `w` of [`gather_columns_avx2`] before its final shuffle: lane `l` +/// holds columns `4w + 2l` and `4w + 2l + 1`. +pub open spec fn gather_src(w: int, k: int) -> int { + unpacked_src(w, 8 * permute_d8(k / 8) + k % 8) +} + +/// The selections of the immediate `0b11_01_10_00`. +proof fn lemma_permute_d8() + ensures + forall|i: int| 0 <= i < 4 ==> (((0b11_01_10_00i32 as u8) >> ((2 * i) as u8)) & 3) as int == #[trigger] permute_d8(i), +{ + assert(((0xD8u8 >> 0u8) & 3) == 0) by (bit_vector); + assert(((0xD8u8 >> 2u8) & 3) == 2) by (bit_vector); + assert(((0xD8u8 >> 4u8) & 3) == 1) by (bit_vector); + assert(((0xD8u8 >> 6u8) & 3) == 3) by (bit_vector); + assert forall|i: int| 0 <= i < 4 implies (((0b11_01_10_00i32 as u8) >> ((2 * i) as u8)) & 3) as int == #[trigger] permute_d8(i) by { + lemma_cases_4(i); + } +} + +/// The input byte at byte `k` of the first unpack round of [`gather_columns_avx2`] (`unpacklo`; the `unpackhi` +/// one is 8 further): in each lane, rows `r` and `r + 4` byte-interleaved. +pub open spec fn unpack1_src(k: int) -> int { + 16 * (k / 16) + (k % 16) / 2 + 32 * (k % 2) +} + +/// The two unpack rounds of [`gather_columns_avx2`] put input byte [`unpacked_src`] at each byte. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +#[verifier::rlimit(30)] +proof fn lemma_unpack_rounds( + input: [u8; 64], + v0: __m256i, + v1: __m256i, + t0: __m256i, + t1: __m256i, + u0: __m256i, + u1: __m256i, +) + requires + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(v0)[k] == input[0 + k], + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(v1)[k] == input[32 + k], + forall|k: int| + 0 <= k < 32 ==> #[trigger] m256_bytes(t0)[k] == unpacklo_epi8_lane(m256_bytes(v0)@, m256_bytes(v1)@, k), + forall|k: int| + 0 <= k < 32 ==> #[trigger] m256_bytes(t1)[k] == unpackhi_epi8_lane(m256_bytes(v0)@, m256_bytes(v1)@, k), + forall|k: int| + 0 <= k < 32 ==> #[trigger] m256_bytes(u0)[k] == unpacklo_epi8_lane(m256_bytes(t0)@, m256_bytes(t1)@, k), + forall|k: int| + 0 <= k < 32 ==> #[trigger] m256_bytes(u1)[k] == unpackhi_epi8_lane(m256_bytes(t0)@, m256_bytes(t1)@, k), + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(u0)[k] == input[unpacked_src(0, k)], + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(u1)[k] == input[unpacked_src(1, k)], +{ + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(t0)[k] == input[unpack1_src(k)] by { + let j = 16 * (k / 16) + (k % 16) / 2; + assert(m256_bytes(v0)[j] == input[0 + j]); + assert(m256_bytes(v1)[j] == input[32 + j]); + } + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(t1)[k] == input[unpack1_src(k) + 8] by { + let j = 16 * (k / 16) + 8 + (k % 16) / 2; + assert(m256_bytes(v0)[j] == input[0 + j]); + assert(m256_bytes(v1)[j] == input[32 + j]); + } + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(u0)[k] == input[unpacked_src(0, k)] by { + lemma_cases_32(k); + } + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(u1)[k] == input[unpacked_src(1, k)] by { + lemma_cases_32(k); + } +} + +/// The permute of [`gather_columns_avx2`] moves input byte [`unpacked_src`] to [`gather_src`]. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +proof fn lemma_permuted(input: [u8; 64], u: __m256i, p: __m256i, w: int) + requires + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(u)[k] == input[unpacked_src(w, k)], + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(p)[i] == permute4x64_lane(m256(u)@, 0b11_01_10_00i32 as u8, i), + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(p)[k] == input[gather_src(w, k)], +{ + broadcast use axiom_m256_bytes; + + lemma_permute_d8(); + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(p)[k] == input[gather_src(w, k)] by { + let (i, t) = (k / 8, k % 8); + let s = permute_d8(i); + lemma_div8(i, t); + lemma_div8(s, t); + assert(k == 8 * i + t); + assert(m256(p)[i] == m256(u)[s]); + assert(m256_bytes(u)[8 * s + t] == input[unpacked_src(w, 8 * s + t)]); + } +} + +/// Gather each byte column of the eight 8-byte rows into a word, in the row order `order` picks. +/// +/// After two unpack rounds, lane `l` of the result's register `w` holds columns `4w + 2l` and `4w + 2l + 1`. +/// Their bytes sit at: +/// +/// ```text +/// rows 0, 1, 4, 5 bytes 0..4 (first column), 4..8 (second) +/// rows 2, 3, 6, 7 bytes 8..12 (first column), 12..16 (second) +/// ``` +/// +/// So `order[8c + k]` is the byte that becomes byte `k` of the lane's column `c`. +/// +/// Rewritten for Verus: the loads through `as_ptr().cast()` are the helpers `load256_bytes_at` and +/// `load128_bytes`, whose bodies are those calls; `[u0, u1].map(|u| ...)` is written out as its two calls, the +/// permutes bound to `p0`, `p1`. Production's function is private. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +#[inline] +#[target_feature(enable = "avx2")] +pub fn gather_columns_avx2(input: &[u8; 64], order: &[u8; 16]) -> (r: [__m256i; 2]) + ensures + forall|w: int, k: int| + 0 <= w < 2 && 0 <= k < 32 && order[k % 16] < 16 ==> #[trigger] m256_bytes(r[w])[k] == input[gather_src( + w, + 16 * (k / 16) + order[k % 16] as int, + )], +{ + let ghost order_bytes = *order; + // SAFETY: the loads read the two halves of the 64-byte input and the 16-byte order. + let (v0, v1, order) = unsafe { + (load256_bytes_at(input, 0), load256_bytes_at(input, 32), _mm256_broadcastsi128_si256(load128_bytes(order))) + }; + // Lane 0: rows 0 and 4, then rows 1 and 5, byte-interleaved; lane 1: rows 2 and 6, then 3 and 7. + let (t0, t1) = (_mm256_unpacklo_epi8(v0, v1), _mm256_unpackhi_epi8(v0, v1)); + // Dword `c` of lane 0 is column `c` of rows 0, 1, 4, 5; of lane 1, of rows 2, 3, 6, 7. + let (u0, u1) = (_mm256_unpacklo_epi8(t0, t1), _mm256_unpackhi_epi8(t0, t1)); + proof { + lemma_unpack_rounds(*input, v0, v1, t0, t1, u0, u1); + } + // Bring both halves of two columns into one lane, then order their bytes. + let (p0, p1) = (_mm256_permute4x64_epi64::<0b11_01_10_00>(u0), _mm256_permute4x64_epi64::<0b11_01_10_00>(u1)); + proof { + broadcast use axiom_m256_bytes, axiom_m128_bytes; + + lemma_permuted(*input, u0, p0, 0); + lemma_permuted(*input, u1, p1, 1); + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(order)[k] == order_bytes[k % 16] by { + lemma_cases_32(k); + } + } + let r = [_mm256_shuffle_epi8(p0, order), _mm256_shuffle_epi8(p1, order)]; + proof { + assert forall|w: int, k: int| 0 <= w < 2 && 0 <= k < 32 && order_bytes[k % 16] < 16 implies #[trigger] m256_bytes( + r[w], + )[k] == input[gather_src(w, 16 * (k / 16) + order_bytes[k % 16] as int)] by { + lemma_small_byte(order_bytes[k % 16]); + assert(m256_bytes(order)[k] == order_bytes[k % 16]); + } + } + r +} + +// --------------------------------------------------------------------------------------------- +// AVX2 with GFNI +// --------------------------------------------------------------------------------------------- +/// Production's function-local `REVERSED` of `bit_transpose_64bytes_gfni_avx2`, at module level. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] +pub const REVERSED: [u8; 16] = [11, 10, 3, 2, 9, 8, 1, 0, 15, 14, 7, 6, 13, 12, 5, 4]; + +/// [`REVERSED`] orders each column's bytes by reversed row: byte `kk` of word `j` of register `w` is +/// `input[8 (7 - kk) + 4w + j]`. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] +proof fn lemma_reversed(w: int, j: int, kk: int) + requires + 0 <= w < 2, + 0 <= j < 4, + 0 <= kk < 8, + ensures + REVERSED[(8 * j + kk) % 16] < 16, + gather_src(w, 16 * ((8 * j + kk) / 16) + REVERSED[(8 * j + kk) % 16] as int) == 8 * (7 - kk) + 4 * w + j, +{ + lemma_cases_4(j); + lemma_cases_8(kk); +} + +/// One register of [`bit_transpose_64bytes_gfni_avx2`]: its four reversed columns through the unit affine map, +/// stored at `32i`, are output rows `4i..4i + 4`. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] +proof fn lemma_gfni_avx2_rows( + input: [u8; 64], + after: [u8; 64], + i: int, + unit: __m256i, + columns: __m256i, + rows: __m256i, +) + requires + 0 <= i < 2, + forall|j: int| 0 <= j < 4 ==> #[trigger] is_reversed_column(input, 4 * i + j, m256(columns)[j]), + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(unit)[k] == 1u8 << ((k % 8) as u8), + forall|k: int| + 0 <= k < 32 ==> #[trigger] m256_bytes(rows)[k] == gf2p8affine_lane(m256_bytes(unit)@, m256(columns)@, 0i32 as u8, k), + forall|k: int| 32 * i <= k < 32 * i + 32 ==> #[trigger] after[k] == m256_bytes(rows)[k - 32 * i], + ensures + forall|b: int, t: int, x: int| + 4 * i <= b < 4 * i + 4 && 0 <= t < 8 && 0 <= x < 8 ==> #[trigger] bit8(after[b * 8 + t], x) == bit8( + input[x * 8 + b], + t, + ), +{ + assert forall|b: int, t: int, x: int| 4 * i <= b < 4 * i + 4 && 0 <= t < 8 && 0 <= x < 8 implies #[trigger] bit8( + after[b * 8 + t], + x, + ) == bit8(input[x * 8 + b], t) by { + let j = b - 4 * i; + lemma_div8(j, t); + assert(after[b * 8 + t] == m256_bytes(rows)[8 * j + t]); + assert(m256_bytes(unit)[8 * j + t] == 1u8 << (t as u8)); + assert(is_reversed_column(input, 4 * i + j, m256(columns)[j])); + lemma_affine_column(input, b, m256(columns)[j], t); + } +} + +/// [`gather_columns_avx2`] with the rows reversed, then the affine map of the AVX-512 arm. +/// +/// Rewritten for Verus: `for (i, columns) in gather_columns_avx2(input, &REVERSED).into_iter().enumerate()` is an +/// index loop over the gathered pair; the store through `as_mut_ptr().add(32 * i).cast()` is the helper +/// `store256_bytes_at`, whose body is that call; `REVERSED` is a module-level `const`. Production's function is +/// private. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] +#[cfg_attr(target_feature = "avx512vbmi", allow(dead_code))] +#[target_feature(enable = "avx2,gfni")] +pub unsafe fn bit_transpose_64bytes_gfni_avx2(input: &[u8; 64], output: &mut [u8; 64]) + ensures + is_bit_transpose_64bytes(*input, *final(output)), +{ + let unit = _mm256_set1_epi64x(0x8040_2010_0804_0201_u64 as i64); + let gathered = gather_columns_avx2(input, &REVERSED); + proof { + broadcast use axiom_m256_bytes; + + lemma_unit_word(); + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(unit)[k] == 1u8 << ((k % 8) as u8) by { + lemma_unit_byte((k % 8) as u64); + } + assert forall|w: int, j: int| 0 <= w < 2 && 0 <= j < 4 implies #[trigger] is_reversed_column( + *input, + 4 * w + j, + m256(gathered[w])[j], + ) by { + assert forall|kk: int| 0 <= kk < 8 implies #[trigger] byte_of(m256(gathered[w])[j], kk) == input[8 * (7 + - kk) + 4 * w + j] by { + lemma_reversed(w, j, kk); + lemma_div8(j, kk); + assert(m256_bytes(gathered[w])[8 * j + kk] == input[8 * (7 - kk) + 4 * w + j]); + } + } + } + for i in 0..2 + invariant + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(unit)[k] == 1u8 << ((k % 8) as u8), + forall|w: int, j: int| + 0 <= w < 2 && 0 <= j < 4 ==> #[trigger] is_reversed_column(*input, 4 * w + j, m256(gathered[w])[j]), + forall|b: int, t: int, x: int| + 0 <= b < 4 * i && 0 <= t < 8 && 0 <= x < 8 ==> #[trigger] bit8(output[b * 8 + t], x) == bit8( + input[x * 8 + b], + t, + ), + { + let columns = gathered[i]; + let rows = _mm256_gf2p8affine_epi64_epi8::<0>(unit, columns); + // SAFETY: the two stores fill the 64-byte output. + unsafe { store256_bytes_at(output, 32 * i, rows) }; + proof { + assert forall|j: int| 0 <= j < 4 implies #[trigger] is_reversed_column(*input, 4 * i + j, m256(columns)[j]) by { + assert(is_reversed_column(*input, 4 * i + j, m256(gathered[i as int])[j])); + } + lemma_gfni_avx2_rows(*input, *output, i as int, unit, columns, rows); + } + } +} + +// --------------------------------------------------------------------------------------------- +// AVX2 +// --------------------------------------------------------------------------------------------- +/// Production's function-local `NATURAL` of `bit_transpose_64bytes_avx2`, at module level. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +pub const NATURAL: [u8; 16] = [0, 1, 8, 9, 2, 3, 10, 11, 4, 5, 12, 13, 6, 7, 14, 15]; + +/// [`NATURAL`] orders each column's bytes by row: byte `kk` of word `j` of register `w` is `input[8kk + 4w + j]`. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +proof fn lemma_natural(w: int, j: int, kk: int) + requires + 0 <= w < 2, + 0 <= j < 4, + 0 <= kk < 8, + ensures + NATURAL[(8 * j + kk) % 16] < 16, + gather_src(w, 16 * ((8 * j + kk) / 16) + NATURAL[(8 * j + kk) % 16] as int) == 8 * kk + 4 * w + j, +{ + lemma_cases_4(j); + lemma_cases_8(kk); +} + +/// The masked swap of [`transpose_8x8_bits`] on four words. +/// +/// Production nests this function in `bit_transpose_64bytes_avx2`; Verus does not take nested functions, so it +/// is here, under that function's `cfg`. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +#[inline] +#[target_feature(enable = "avx2")] +pub fn swap(x: __m256i, mask: u64) -> (r: __m256i) + requires + 0 < D < 64, + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == swap_word(m256(x)[i], D as u64, mask), +{ + let ghost m = mask; + let mask = _mm256_set1_epi64x(mask as i64); + proof { + assert((m as i64) as u64 == m) by (bit_vector); + } + let t = _mm256_and_si256(_mm256_xor_si256(x, _mm256_srli_epi64::(x)), mask); + _mm256_xor_si256(x, _mm256_xor_si256(t, _mm256_slli_epi64::(t))) +} + +/// One register of [`bit_transpose_64bytes_avx2`]: its four columns, transposed and stored at `32i`, are output +/// rows `4i..4i + 4`. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +proof fn lemma_avx2_rows(input: [u8; 64], after: [u8; 64], i: int, columns: __m256i, rows: __m256i) + requires + 0 <= i < 2, + forall|j: int| 0 <= j < 4 ==> #[trigger] is_column(input, 4 * i + j, m256(columns)[j]), + forall|j: int| 0 <= j < 4 ==> #[trigger] m256(rows)[j] == transpose_8x8_formula(m256(columns)[j]), + forall|k: int| 32 * i <= k < 32 * i + 32 ==> #[trigger] after[k] == m256_bytes(rows)[k - 32 * i], + ensures + forall|b: int, t: int, x: int| + 4 * i <= b < 4 * i + 4 && 0 <= t < 8 && 0 <= x < 8 ==> #[trigger] bit8(after[b * 8 + t], x) == bit8( + input[x * 8 + b], + t, + ), +{ + broadcast use axiom_m256_bytes; + + assert forall|b: int, t: int, x: int| 4 * i <= b < 4 * i + 4 && 0 <= t < 8 && 0 <= x < 8 implies #[trigger] bit8( + after[b * 8 + t], + x, + ) == bit8(input[x * 8 + b], t) by { + let j = b - 4 * i; + lemma_div8(j, t); + assert(after[b * 8 + t] == m256_bytes(rows)[8 * j + t]); + assert(m256_bytes(rows)[8 * j + t] == byte_of(transpose_8x8_formula(m256(columns)[j]), t)); + assert(is_column(input, 4 * i + j, m256(columns)[j])); + lemma_transposed_column(input, b, m256(columns)[j], t); + } +} + +/// [`gather_columns_avx2`] in row order, then the three masked-swap rounds of [`transpose_8x8_bits`] on four +/// words at a time. +/// +/// Rewritten for Verus: `for (i, x) in gather_columns_avx2(input, &NATURAL).into_iter().enumerate()` is an index +/// loop over the gathered pair; the store through `as_mut_ptr().add(32 * i).cast()` is the helper +/// `store256_bytes_at`, whose body is that call; `NATURAL` and `swap` are at module level. Production's function +/// is private. +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +#[cfg_attr(target_feature = "gfni", allow(dead_code))] +#[target_feature(enable = "avx2")] +pub unsafe fn bit_transpose_64bytes_avx2(input: &[u8; 64], output: &mut [u8; 64]) + ensures + is_bit_transpose_64bytes(*input, *final(output)), +{ + let gathered = gather_columns_avx2(input, &NATURAL); + proof { + broadcast use axiom_m256_bytes; + + assert forall|w: int, j: int| 0 <= w < 2 && 0 <= j < 4 implies #[trigger] is_column( + *input, + 4 * w + j, + m256(gathered[w])[j], + ) by { + assert forall|kk: int| 0 <= kk < 8 implies #[trigger] byte_of(m256(gathered[w])[j], kk) == input[8 * kk + 4 + * w + j] by { + lemma_natural(w, j, kk); + lemma_div8(j, kk); + assert(m256_bytes(gathered[w])[8 * j + kk] == input[8 * kk + 4 * w + j]); + } + } + } + for i in 0..2 + invariant + forall|w: int, j: int| + 0 <= w < 2 && 0 <= j < 4 ==> #[trigger] is_column(*input, 4 * w + j, m256(gathered[w])[j]), + forall|b: int, t: int, x: int| + 0 <= b < 4 * i && 0 <= t < 8 && 0 <= x < 8 ==> #[trigger] bit8(output[b * 8 + t], x) == bit8( + input[x * 8 + b], + t, + ), + { + let x = gathered[i]; + let ghost columns = x; + let x = swap::<7>(x, 0x00AA_00AA_00AA_00AA); + let x = swap::<14>(x, 0x0000_CCCC_0000_CCCC); + let x = swap::<28>(x, 0x0000_0000_F0F0_F0F0); + // SAFETY: the two stores fill the 64-byte output. + unsafe { store256_bytes_at(output, 32 * i, x) }; + proof { + assert forall|j: int| 0 <= j < 4 implies #[trigger] m256(x)[j] == transpose_8x8_formula(m256(columns)[j]) by { + lemma_three_swaps(m256(columns)[j]); + } + assert forall|j: int| 0 <= j < 4 implies #[trigger] is_column(*input, 4 * i + j, m256(columns)[j]) by { + assert(is_column(*input, 4 * i + j, m256(gathered[i as int])[j])); + } + lemma_avx2_rows(*input, *output, i as int, columns, x); + } + } +} + +// --------------------------------------------------------------------------------------------- +// NEON +// --------------------------------------------------------------------------------------------- +/// Production's function-local `vqtbl4q` indexes of `bit_transpose_64bytes_neon`, at module level: they bring +/// bytes belonging to byte-chunk `b` in `0..8` into contiguous 8-byte runs, packed two chunks per Q register. +#[cfg(target_arch = "aarch64")] +pub const IDX0: [u8; 16] = [0, 8, 16, 24, 32, 40, 48, 56, 1, 9, 17, 25, 33, 41, 49, 57]; +#[cfg(target_arch = "aarch64")] +pub const IDX1: [u8; 16] = [2, 10, 18, 26, 34, 42, 50, 58, 3, 11, 19, 27, 35, 43, 51, 59]; +#[cfg(target_arch = "aarch64")] +pub const IDX2: [u8; 16] = [4, 12, 20, 28, 36, 44, 52, 60, 5, 13, 21, 29, 37, 45, 53, 61]; +#[cfg(target_arch = "aarch64")] +pub const IDX3: [u8; 16] = [6, 14, 22, 30, 38, 46, 54, 62, 7, 15, 23, 31, 39, 47, 55, 63]; + +/// Entry `k` of `IDXj` is row `k % 8` of column `2j + k / 8`. +#[cfg(target_arch = "aarch64")] +proof fn lemma_neon_idx(k: int) + requires + 0 <= k < 16, + ensures + IDX0[k] as int == 8 * (k % 8) + k / 8, + IDX1[k] as int == 8 * (k % 8) + 2 + k / 8, + IDX2[k] as int == 8 * (k % 8) + 4 + k / 8, + IDX3[k] as int == 8 * (k % 8) + 6 + k / 8, +{ + lemma_cases_16(k); +} + +/// One table lookup of [`bit_transpose_64bytes_neon`]: with the index `idx` of register `j`, its two words are +/// columns `2j` and `2j + 1`. +#[cfg(target_arch = "aarch64")] +proof fn lemma_neon_columns( + input: [u8; 64], + v0: uint8x16_t, + v1: uint8x16_t, + v2: uint8x16_t, + v3: uint8x16_t, + idx: [u8; 16], + y: uint64x2_t, + j: int, +) + requires + 0 <= j < 4, + forall|k: int| 0 <= k < 16 ==> #[trigger] u8x16(v0)[k] == input[k], + forall|k: int| 0 <= k < 16 ==> #[trigger] u8x16(v1)[k] == input[16 + k], + forall|k: int| 0 <= k < 16 ==> #[trigger] u8x16(v2)[k] == input[32 + k], + forall|k: int| 0 <= k < 16 ==> #[trigger] u8x16(v3)[k] == input[48 + k], + forall|k: int| 0 <= k < 16 ==> #[trigger] idx[k] as int == 8 * (k % 8) + 2 * j + k / 8, + forall|k: int| + 0 <= k < 16 ==> #[trigger] u64x2_byte(u64x2(y)@, k) == tbl4_byte( + u8x16(v0)@, + u8x16(v1)@, + u8x16(v2)@, + u8x16(v3)@, + idx[k] as int, + ), + ensures + is_column(input, 2 * j, u64x2(y)[0]), + is_column(input, 2 * j + 1, u64x2(y)[1]), +{ + assert forall|x: int| 0 <= x < 8 implies #[trigger] byte_of(u64x2(y)[0], x) == input[8 * x + 2 * j] by { + lemma_div8(0, x); + assert(idx[x] as int == 8 * x + 2 * j); + assert(u64x2_byte(u64x2(y)@, x) == input[8 * x + 2 * j]); + } + assert forall|x: int| 0 <= x < 8 implies #[trigger] byte_of(u64x2(y)[1], x) == input[8 * x + 2 * j + 1] by { + lemma_div8(1, x); + assert(idx[8 + x] as int == 8 * x + 2 * j + 1); + assert(u64x2_byte(u64x2(y)@, 8 + x) == input[8 * x + 2 * j + 1]); + } +} + +/// One register of [`bit_transpose_64bytes_neon`]: its two columns, transposed and stored at `16j`, are output +/// rows `2j` and `2j + 1`. +#[cfg(target_arch = "aarch64")] +proof fn lemma_neon_rows(input: [u8; 64], output: [u8; 64], j: int, columns: uint64x2_t, rows: uint64x2_t) + requires + 0 <= j < 4, + is_column(input, 2 * j, u64x2(columns)[0]), + is_column(input, 2 * j + 1, u64x2(columns)[1]), + u64x2(rows)[0] == transpose_8x8_formula(u64x2(columns)[0]), + u64x2(rows)[1] == transpose_8x8_formula(u64x2(columns)[1]), + forall|k: int| 16 * j <= k < 16 * j + 16 ==> #[trigger] output[k] == u64x2_byte(u64x2(rows)@, k - 16 * j), + ensures + forall|b: int, t: int, x: int| + 2 * j <= b < 2 * j + 2 && 0 <= t < 8 && 0 <= x < 8 ==> #[trigger] bit8(output[b * 8 + t], x) == bit8( + input[x * 8 + b], + t, + ), +{ + assert forall|b: int, t: int, x: int| 2 * j <= b < 2 * j + 2 && 0 <= t < 8 && 0 <= x < 8 implies #[trigger] bit8( + output[b * 8 + t], + x, + ) == bit8(input[x * 8 + b], t) by { + let h = b - 2 * j; + lemma_div8(h, t); + assert(output[b * 8 + t] == u64x2_byte(u64x2(rows)@, 8 * h + t)); + assert(output[b * 8 + t] == byte_of(u64x2(rows)[h], t)); + if h == 0 { + lemma_transposed_column(input, b, u64x2(columns)[0], t); + } else { + lemma_transposed_column(input, b, u64x2(columns)[1], t); + } + } +} + +/// `vqtbl4q_u8` gathers each byte column into a word, then three masked-swap rounds transpose its bits. +/// +/// Rewritten for Verus: the loads `vld1q_u8(in_ptr.add(16 * j))` and `vld1q_u8(IDXj.as_ptr())` and the stores +/// `vst1q_u8(out_ptr.add(16 * j), ..)` are the helpers `vld1q_u8_at`, `vld1q_u8_16` and `vst1q_u8_at`, whose +/// bodies are those calls (so `in_ptr` and `out_ptr` are gone); `IDX0..IDX3` are module-level `const`s. +/// Production's function is private. +#[cfg(target_arch = "aarch64")] +#[inline(always)] +pub unsafe fn bit_transpose_64bytes_neon(input: &[u8; 64], output: &mut [u8; 64]) + ensures + is_bit_transpose_64bytes(*input, *final(output)), +{ + // SAFETY: NEON is part of the aarch64 baseline, and every load and store offset is below 64, the length of + // both arrays. + unsafe { + let v0 = vld1q_u8_at(input, 0); + let v1 = vld1q_u8_at(input, 16); + let v2 = vld1q_u8_at(input, 32); + let v3 = vld1q_u8_at(input, 48); + let table = uint8x16x4_t(v0, v1, v2, v3); + + let mut y0 = vreinterpretq_u64_u8(vqtbl4q_u8(table, vld1q_u8_16(&IDX0))); + let mut y1 = vreinterpretq_u64_u8(vqtbl4q_u8(table, vld1q_u8_16(&IDX1))); + let mut y2 = vreinterpretq_u64_u8(vqtbl4q_u8(table, vld1q_u8_16(&IDX2))); + let mut y3 = vreinterpretq_u64_u8(vqtbl4q_u8(table, vld1q_u8_16(&IDX3))); + let ghost (c0, c1, c2, c3) = (y0, y1, y2, y3); + proof { + assert forall|k: int| 0 <= k < 16 implies #[trigger] IDX0[k] as int == 8 * (k % 8) + 2 * 0 + k / 8 && IDX1[k] + as int == 8 * (k % 8) + 2 * 1 + k / 8 && IDX2[k] as int == 8 * (k % 8) + 2 * 2 + k / 8 && IDX3[k] as int + == 8 * (k % 8) + 2 * 3 + k / 8 by { + lemma_neon_idx(k); + } + lemma_neon_columns(*input, v0, v1, v2, v3, IDX0, c0, 0); + lemma_neon_columns(*input, v0, v1, v2, v3, IDX1, c1, 1); + lemma_neon_columns(*input, v0, v1, v2, v3, IDX2, c2, 2); + lemma_neon_columns(*input, v0, v1, v2, v3, IDX3, c3, 3); + } + + let mask1 = vdupq_n_u64(0x00AA00AA00AA00AA); + let mask2 = vdupq_n_u64(0x0000CCCC0000CCCC); + let mask3 = vdupq_n_u64(0x00000000F0F0F0F0); + + // Round 1: distance 7. + let t0 = vandq_u64(veorq_u64(y0, vshrq_n_u64::<7>(y0)), mask1); + let t1 = vandq_u64(veorq_u64(y1, vshrq_n_u64::<7>(y1)), mask1); + let t2 = vandq_u64(veorq_u64(y2, vshrq_n_u64::<7>(y2)), mask1); + let t3 = vandq_u64(veorq_u64(y3, vshrq_n_u64::<7>(y3)), mask1); + y0 = veorq_u64(y0, veorq_u64(t0, vshlq_n_u64::<7>(t0))); + y1 = veorq_u64(y1, veorq_u64(t1, vshlq_n_u64::<7>(t1))); + y2 = veorq_u64(y2, veorq_u64(t2, vshlq_n_u64::<7>(t2))); + y3 = veorq_u64(y3, veorq_u64(t3, vshlq_n_u64::<7>(t3))); + + // Round 2: distance 14. + let t0 = vandq_u64(veorq_u64(y0, vshrq_n_u64::<14>(y0)), mask2); + let t1 = vandq_u64(veorq_u64(y1, vshrq_n_u64::<14>(y1)), mask2); + let t2 = vandq_u64(veorq_u64(y2, vshrq_n_u64::<14>(y2)), mask2); + let t3 = vandq_u64(veorq_u64(y3, vshrq_n_u64::<14>(y3)), mask2); + y0 = veorq_u64(y0, veorq_u64(t0, vshlq_n_u64::<14>(t0))); + y1 = veorq_u64(y1, veorq_u64(t1, vshlq_n_u64::<14>(t1))); + y2 = veorq_u64(y2, veorq_u64(t2, vshlq_n_u64::<14>(t2))); + y3 = veorq_u64(y3, veorq_u64(t3, vshlq_n_u64::<14>(t3))); + + // Round 3: distance 28. + let t0 = vandq_u64(veorq_u64(y0, vshrq_n_u64::<28>(y0)), mask3); + let t1 = vandq_u64(veorq_u64(y1, vshrq_n_u64::<28>(y1)), mask3); + let t2 = vandq_u64(veorq_u64(y2, vshrq_n_u64::<28>(y2)), mask3); + let t3 = vandq_u64(veorq_u64(y3, vshrq_n_u64::<28>(y3)), mask3); + y0 = veorq_u64(y0, veorq_u64(t0, vshlq_n_u64::<28>(t0))); + y1 = veorq_u64(y1, veorq_u64(t1, vshlq_n_u64::<28>(t1))); + y2 = veorq_u64(y2, veorq_u64(t2, vshlq_n_u64::<28>(t2))); + y3 = veorq_u64(y3, veorq_u64(t3, vshlq_n_u64::<28>(t3))); + proof { + lemma_three_swaps(u64x2(c0)[0]); + lemma_three_swaps(u64x2(c0)[1]); + lemma_three_swaps(u64x2(c1)[0]); + lemma_three_swaps(u64x2(c1)[1]); + lemma_three_swaps(u64x2(c2)[0]); + lemma_three_swaps(u64x2(c2)[1]); + lemma_three_swaps(u64x2(c3)[0]); + lemma_three_swaps(u64x2(c3)[1]); + } + + vst1q_u8_at(output, 0, vreinterpretq_u8_u64(y0)); + vst1q_u8_at(output, 16, vreinterpretq_u8_u64(y1)); + vst1q_u8_at(output, 32, vreinterpretq_u8_u64(y2)); + vst1q_u8_at(output, 48, vreinterpretq_u8_u64(y3)); + proof { + lemma_neon_rows(*input, *output, 0, c0, y0); + lemma_neon_rows(*input, *output, 1, c1, y1); + lemma_neon_rows(*input, *output, 2, c2, y2); + lemma_neon_rows(*input, *output, 3, c3, y3); + } + } +} + +} // verus! diff --git a/verification/verus/src/blake2s.rs b/verification/verus/src/blake2s.rs new file mode 100644 index 0000000000..d829b7f730 --- /dev/null +++ b/verification/verus/src/blake2s.rs @@ -0,0 +1,891 @@ +//! BLAKE2s (RFC 7693): the compression, and the one-shot, streaming and continued hashes. +//! +//! The executable functions are the scalar code of `crates/primitives/src/hash/mod.rs`, copied with the same names +//! and bodies where Verus accepts them; `tests/equivalence/blake2s.rs` checks the copies against production. Each +//! rewrite Verus forces (index loops for `for` over tables and `as_chunks`, shifts for `from_le_bytes` and +//! `to_le_bytes`, a loop for `fill`, the IV's values in `PARAM_IV`) is noted at the function. +//! +//! Specification, a transcription of RFC 7693 independent of production's structure: +//! +//! - [`g_spec`]: the mixing function G with rotations 16, 12, 8, 7 ([`rotr`] is the RFC's `>>>`). +//! - [`sigma`], [`iv`]: the message schedule and the initialization vector; [`round_spec`]: G on the columns, then +//! on the diagonals; [`rounds_spec`]: the ten rounds. +//! - [`f_spec`]: the compression F, from the work vector [`init_spec`] (counter in words 12 and 13, word 14 +//! inverted on the final block) to `h ^ v[0..8] ^ v[8..16]`. +//! - [`blake2s_spec`]: the unkeyed BLAKE2s-256 of a message: [`param_iv`], the zero-padded blocks +//! ([`block_spec`], little-endian words), [`chain_spec`] over all blocks but the last, the last block at counter +//! `ll` with the final flag, and the digest's 32 little-endian bytes ([`digest_spec`]). +//! +//! Main results: `compress`'s postcondition (`h' = F(h, m, t, last)`), `hash`'s and `Hasher::finalize`'s +//! (`blake2s_spec` of the bytes passed), `hash_from_state`'s, `zero_prefix_state`'s, and +//! [`lemma_zero_prefix_continuation`]. +//! +//! Trusted: `u32::rotate_right`, which vstd does not specify, is the RFC's rotation for `0 < n < 32` +//! (`assume_specification`, checked by `rotate_right_is_the_rfc_rotation`). +#[cfg(verus_keep_ghost)] +use vstd::compute::RangeAll; +use vstd::prelude::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification: RFC 7693, transcribed +// --------------------------------------------------------------------------------------------- +/// `x >>> n`, rotation of a 32-bit word right by `n` bits (RFC 7693, section 2.3). +pub open spec fn rotr(x: u32, n: u32) -> u32 { + (x >> n) ^ (x << sub(32, n)) +} + +/// `(x + y + z) mod 2^32`. +pub open spec fn add_mod(x: u32, y: u32, z: u32) -> u32 { + ((x + y + z) % 0x1_0000_0000) as u32 +} + +/// The mixing function G (RFC 7693, section 3.1), with BLAKE2s's rotations `R1..R4 = 16, 12, 8, 7`. +/// +/// Opaque: proofs above one G treat it as a function. +#[verifier::opaque] +pub open spec fn g_spec(v: Seq, a: int, b: int, c: int, d: int, x: u32, y: u32) -> Seq { + let v = v.update(a, add_mod(v[a], v[b], x)); + let v = v.update(d, rotr(v[d] ^ v[a], 16)); + let v = v.update(c, add_mod(v[c], v[d], 0)); + let v = v.update(b, rotr(v[b] ^ v[c], 12)); + let v = v.update(a, add_mod(v[a], v[b], y)); + let v = v.update(d, rotr(v[d] ^ v[a], 8)); + let v = v.update(c, add_mod(v[c], v[d], 0)); + v.update(b, rotr(v[b] ^ v[c], 7)) +} + +/// The initialization vector (RFC 7693, section 2.6). +pub open spec fn iv() -> Seq { + seq![ + 0x6A09E667u32, 0xBB67AE85u32, 0x3C6EF372u32, 0xA54FF53Au32, + 0x510E527Fu32, 0x9B05688Cu32, 0x1F83D9ABu32, 0x5BE0CD19u32, + ] +} + +/// The message schedule SIGMA (RFC 7693, section 2.7). +pub open spec fn sigma(i: int) -> Seq { + if i == 0 { + seq![0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] + } else if i == 1 { + seq![14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3] + } else if i == 2 { + seq![11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4] + } else if i == 3 { + seq![7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8] + } else if i == 4 { + seq![9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13] + } else if i == 5 { + seq![2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9] + } else if i == 6 { + seq![12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11] + } else if i == 7 { + seq![13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10] + } else if i == 8 { + seq![6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5] + } else { + seq![10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0] + } +} + +/// Round `i` of the compression (RFC 7693, section 3.2): G on the four columns, then on the four diagonals. +pub open spec fn round_spec(v: Seq, m: Seq, i: int) -> Seq { + let s = sigma(i % 10); + let v = g_spec(v, 0, 4, 8, 12, m[s[0]], m[s[1]]); + let v = g_spec(v, 1, 5, 9, 13, m[s[2]], m[s[3]]); + let v = g_spec(v, 2, 6, 10, 14, m[s[4]], m[s[5]]); + let v = g_spec(v, 3, 7, 11, 15, m[s[6]], m[s[7]]); + let v = g_spec(v, 0, 5, 10, 15, m[s[8]], m[s[9]]); + let v = g_spec(v, 1, 6, 11, 12, m[s[10]], m[s[11]]); + let v = g_spec(v, 2, 7, 8, 13, m[s[12]], m[s[13]]); + g_spec(v, 3, 4, 9, 14, m[s[14]], m[s[15]]) +} + +/// Rounds `0..n`. +pub open spec fn rounds_spec(v: Seq, m: Seq, n: nat) -> Seq + decreases n, +{ + if n == 0 { + v + } else { + round_spec(rounds_spec(v, m, (n - 1) as nat), m, n - 1) + } +} + +/// The local work vector at the start of F (RFC 7693, section 3.2): `h`, then the IV with the offset counter +/// `t` (`0 <= t < 2^64`) in words 12 and 13 and, on the final block (`f`), word 14 inverted. +pub open spec fn init_spec(h: Seq, t: int, f: bool) -> Seq { + let v = h + iv(); + let v = v.update(12, v[12] ^ ((t % 0x1_0000_0000) as u32)); + let v = v.update(13, v[13] ^ ((t / 0x1_0000_0000) as u32)); + if f { v.update(14, v[14] ^ 0xFFFF_FFFFu32) } else { v } +} + +/// The compression function F (RFC 7693, section 3.2): chaining value `h`, message block `m`, offset counter `t`, +/// final block flag `f`; ten rounds, then the two halves of the work vector folded into `h`. +/// +/// Opaque: the message-level proofs treat F as a function, only the compression proofs look inside. +#[verifier::opaque] +pub open spec fn f_spec(h: Seq, m: Seq, t: int, f: bool) -> Seq { + let v = rounds_spec(init_spec(h, t, f), m, 10); + Seq::new(8, |i: int| h[i] ^ v[i] ^ v[i + 8]) +} + +/// Byte `k` of the message `d` padded with zeros (RFC 7693, section 3.3). +pub open spec fn padded_byte(d: Seq, k: int) -> u8 { + if 0 <= k < d.len() { d[k] } else { 0 } +} + +/// The 32-bit word of padded bytes `k..k + 4`, little-endian (RFC 7693, section 2.4). +pub open spec fn le_word(d: Seq, k: int) -> u32 { + (padded_byte(d, k) as int + 0x100 * padded_byte(d, k + 1) + 0x1_0000 * padded_byte(d, k + 2) + 0x100_0000 + * padded_byte(d, k + 3)) as u32 +} + +/// Message block `d[i]`: the 16 words of padded bytes `64 i .. 64 i + 64` (RFC 7693, section 3.3). +pub open spec fn block_spec(d: Seq, i: int) -> Seq { + Seq::new(16, |w: int| le_word(d, 64 * i + 4 * w)) +} + +/// The chaining value after blocks `0..k` of `d`, none final, from `h`: block `i` at counter `t0 + 64 (i + 1)`. +/// +/// The RFC's loop over the blocks before the last one, with `t0 = 0`; a nonzero `t0` continues a message whose +/// first `t0` bytes are already absorbed into `h`. +pub open spec fn chain_spec(h: Seq, d: Seq, t0: int, k: nat) -> Seq + decreases k, +{ + if k == 0 { + h + } else { + f_spec(chain_spec(h, d, t0, (k - 1) as nat), block_spec(d, k - 1), t0 + 64 * k, false) + } +} + +/// The unkeyed BLAKE2s-256 parameter block in the IV: `h[0] ^= 0x01010000 ^ (kk << 8) ^ nn` with `kk = 0` and +/// `nn = 32` (RFC 7693, section 3.3). +pub open spec fn param_iv() -> Seq { + iv().update(0, iv()[0] ^ 0x0101_0000u32 ^ (0u32 << 8u32) ^ 32u32) +} + +/// The number of blocks `dd` of an unkeyed message of `ll` bytes: `ceil(ll / 64)`, and one for the empty message. +pub open spec fn block_count(ll: int) -> int { + if ll == 0 { 1 } else { (ll + 63) / 64 } +} + +/// Byte `j` (`0 <= j < 4`) of a word, little-endian (RFC 7693, section 2.4). +pub open spec fn word_byte(x: u32, j: int) -> u8 { + if j == 0 { + (x % 0x100) as u8 + } else if j == 1 { + (x / 0x100 % 0x100) as u8 + } else if j == 2 { + (x / 0x1_0000 % 0x100) as u8 + } else { + (x / 0x100_0000 % 0x100) as u8 + } +} + +/// The first `nn = 32` bytes of the little-endian word array `h`. +pub open spec fn digest_spec(h: Seq) -> Seq { + Seq::new(32, |i: int| word_byte(h[i / 4], i % 4)) +} + +/// The finalization from chaining value `h` with `d` the message after the `t0` bytes already in `h`: the blocks +/// before the last, then the last at counter `t0 + ll` with the final flag, then the digest. +pub open spec fn finish_spec(h: Seq, d: Seq, t0: int) -> Seq { + let dd = block_count(d.len() as int); + let h = chain_spec(h, d, t0, (dd - 1) as nat); + digest_spec(f_spec(h, block_spec(d, dd - 1), t0 + d.len(), true)) +} + +/// Unkeyed BLAKE2s-256 of the message `d` (`ll = d.len() < 2^64`; RFC 7693, section 3.3). +pub open spec fn blake2s_spec(d: Seq) -> Seq { + finish_spec(param_iv(), d, 0) +} + +// --------------------------------------------------------------------------------------------- +// Production constants and compression +// --------------------------------------------------------------------------------------------- +/// BLAKE2s initial values: the SHA-256 IV. +pub const IV: [u32; 8] = [ + 0x6A09_E667, + 0xBB67_AE85, + 0x3C6E_F372, + 0xA54F_F53A, + 0x510E_527F, + 0x9B05_688C, + 0x1F83_D9AB, + 0x5BE0_CD19, +]; + +/// Digest length in bytes: BLAKE2s-256 throughout. +pub const OUT_LEN: usize = 32; +/// Compression block length in bytes. +pub const BLOCK_LEN: usize = 64; +/// Rounds per compression. +pub const ROUNDS: usize = 10; + +/// BLAKE2s message schedule. +pub const SIGMA: [[usize; 16]; ROUNDS] = [ + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], + [11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4], + [7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8], + [9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13], + [2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9], + [12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11], + [13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10], + [6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5], + [10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0], +]; + +/// Lanes touched by G index `g` within a round: `[a, b, c, d]`. +pub const G_LANES: [[usize; 4]; 8] = [ + [0, 4, 8, 12], + [1, 5, 9, 13], + [2, 6, 10, 14], + [3, 7, 11, 15], + [0, 5, 10, 15], + [1, 6, 11, 12], + [2, 7, 8, 13], + [3, 4, 9, 14], +]; + +/// The unkeyed BLAKE2s-256 initial chaining value: the IV with the parameter block in word 0. +/// +/// Hashing 64 bytes is one compression from here, at counter 64 with the final flag. +/// +/// Verus form: production's `let mut h = IV; h[0] ^= ..; h` block is the array written out, with the IV's values (a +/// const's body must be a pure expression, and indexing `IV` is not). +pub const PARAM_IV: [u32; 8] = [ + // Digest length 32, key length 0, fanout 1, depth 1. + 0x6A09_E667 ^ 0x0101_0000 ^ OUT_LEN as u32, + 0xBB67_AE85, + 0x3C6E_F372, + 0xA54F_F53A, + 0x510E_527F, + 0x9B05_688C, + 0x1F83_D9AB, + 0x5BE0_CD19, +]; + +/// `u32::rotate_right`, which vstd does not specify: for `0 < n < 32` it is the RFC's rotation. +pub assume_specification[ u32::rotate_right ](x: u32, n: u32) -> (r: u32) + requires + 0 < n < 32, + ensures + r == rotr(x, n), +; + +/// The state lanes `[a, b, c, d]` of G number `g` in a round (RFC 7693, section 3.2). +pub open spec fn g_lanes(g: int) -> Seq { + if g == 0 { + seq![0, 4, 8, 12] + } else if g == 1 { + seq![1, 5, 9, 13] + } else if g == 2 { + seq![2, 6, 10, 14] + } else if g == 3 { + seq![3, 7, 11, 15] + } else if g == 4 { + seq![0, 5, 10, 15] + } else if g == 5 { + seq![1, 6, 11, 12] + } else if g == 6 { + seq![2, 7, 8, 13] + } else { + seq![3, 4, 9, 14] + } +} + +/// G number `g` of round `i`. +pub open spec fn g_of_round(v: Seq, m: Seq, i: int, g: int) -> Seq { + let l = g_lanes(g); + let s = sigma(i % 10); + g_spec(v, l[0], l[1], l[2], l[3], m[s[2 * g]], m[s[2 * g + 1]]) +} + +/// The first `g` G's of round `i`. +pub open spec fn round_prefix(v: Seq, m: Seq, i: int, g: int) -> Seq + decreases g, +{ + if g <= 0 { + v + } else { + g_of_round(round_prefix(v, m, i, g - 1), m, i, g - 1) + } +} + +proof fn lemma_round_prefix(v: Seq, m: Seq, i: int) + ensures + round_prefix(v, m, i, 8) == round_spec(v, m, i), +{ + reveal_with_fuel(round_prefix, 9); +} + +/// The production tables `SIGMA` and `G_LANES` are the RFC's. +proof fn lemma_tables() + ensures + forall|i: int, j: int| 0 <= i < 10 && 0 <= j < 16 ==> (#[trigger] SIGMA[i][j]) as int == sigma(i)[j] && SIGMA[i][j] < 16, + forall|g: int, j: int| 0 <= g < 8 && 0 <= j < 4 ==> (#[trigger] G_LANES[g][j]) as int == g_lanes(g)[j] && G_LANES[g][j] < 16, +{ + assert((0..10int).all_spec(|i: int| (0..16int).all_spec(|j: int| SIGMA[i][j] as int == sigma(i)[j] && SIGMA[i][j] < 16))) by (compute); + assert((0..8int).all_spec(|g: int| (0..4int).all_spec(|j: int| G_LANES[g][j] as int == g_lanes(g)[j] && G_LANES[g][j] < 16))) by (compute); + broadcast use vstd::compute::all_spec_ensures; +} + +/// Two wrapping additions are one addition modulo `2^32`. +pub broadcast proof fn lemma_add_mod3(x: u32, y: u32, z: u32) + ensures + #[trigger] vstd::wrapping::u32_specs::wrapping_add(vstd::wrapping::u32_specs::wrapping_add(x, y), z) == add_mod(x, y, z), +{ +} + +/// A wrapping addition is an addition modulo `2^32`. +pub broadcast proof fn lemma_add_mod2(x: u32, y: u32) + ensures + #[trigger] vstd::wrapping::u32_specs::wrapping_add(x, y) == add_mod(x, y, 0), +{ +} + +/// XOR associates. +pub proof fn lemma_xor3(a: u32, b: u32, c: u32) + by (bit_vector) + ensures + a ^ (b ^ c) == a ^ b ^ c, +{ +} + +/// Bitwise not is the RFC's XOR with all ones. +pub proof fn lemma_not_is_xor(x: u32) + by (bit_vector) + ensures + !x == x ^ 0xFFFF_FFFFu32, +{ +} + +/// The two halves of the counter are its residue and quotient modulo `2^32`. +pub proof fn lemma_cast_t(t: u64) + ensures + t as u32 == (t as int % 0x1_0000_0000) as u32, + (t >> 32) as u32 == (t as int / 0x1_0000_0000) as u32, +{ + assert(t as u32 == (t % 0x1_0000_0000u64) as u32 && t >> 32 == t / 0x1_0000_0000u64) by (bit_vector); +} + +/// The BLAKE2s compression: absorb block `m` at byte counter `t` into `h`. +/// +/// `last` sets the final-block flag. +/// +/// The last-node flag stays zero: nothing here uses the tree mode. +/// +/// Verus form: the `for` loops over `&SIGMA` and `G_LANES.iter().enumerate()` are index loops, and the +/// destructuring of `G_LANES[g]` is indexing. +pub fn compress(h: &mut [u32; 8], m: &[u32; 16], t: u64, last: bool) + ensures + final(h)@ == f_spec(old(h)@, m@, t as int, last), +{ + let mut v = [0u32; 16]; + v[..8].copy_from_slice(h); + v[8..].copy_from_slice(&IV); + v[12] ^= t as u32; + v[13] ^= (t >> 32) as u32; + if last { + v[14] = !v[14]; + } + let ghost v0 = v@; + proof { + lemma_cast_t(t); + lemma_not_is_xor(IV[6]); + assert(IV@ =~= iv()); + assert(v0 =~= init_spec(old(h)@, t as int, last)); + } + for r in 0..ROUNDS + invariant + v0 == init_spec(old(h)@, t as int, last), + v@ == rounds_spec(v0, m@, r as nat), + { + let round = &SIGMA[r]; + let ghost vr = v@; + for g in 0..8 + invariant + 0 <= r < 10, + round == &SIGMA[r as int], + vr == rounds_spec(v0, m@, r as nat), + v0 == init_spec(old(h)@, t as int, last), + v@ == round_prefix(vr, m@, r as int, g as int), + { + proof { + lemma_tables(); + } + let (a, b, c, d) = (G_LANES[g][0], G_LANES[g][1], G_LANES[g][2], G_LANES[g][3]); + let (mx, my) = (m[round[2 * g]], m[round[2 * g + 1]]); + let ghost vg = v@; + v[a] = v[a].wrapping_add(v[b]).wrapping_add(mx); + v[d] = (v[d] ^ v[a]).rotate_right(16); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(12); + v[a] = v[a].wrapping_add(v[b]).wrapping_add(my); + v[d] = (v[d] ^ v[a]).rotate_right(8); + v[c] = v[c].wrapping_add(v[d]); + v[b] = (v[b] ^ v[c]).rotate_right(7); + proof { + broadcast use lemma_add_mod3, lemma_add_mod2; + reveal(g_spec); + assert(v@ =~= g_of_round(vg, m@, r as int, g as int)); + } + } + proof { + lemma_round_prefix(vr, m@, r as int); + } + } + let ghost vf = v@; + for i in 0..8 + invariant + v@ == vf, + forall|j: int| 0 <= j < i ==> #[trigger] h[j] == old(h)[j] ^ vf[j] ^ vf[j + 8], + forall|j: int| i <= j < 8 ==> h[j] == old(h)[j], + { + h[i] ^= v[i] ^ v[i + 8]; + proof { + lemma_xor3(old(h)[i as int], vf[i as int], vf[i + 8]); + } + } + proof { + reveal(f_spec); + } + assert(h@ =~= f_spec(old(h)@, m@, t as int, last)); +} + +// --------------------------------------------------------------------------------------------- +// Messages: blocks, chaining and continuation +// --------------------------------------------------------------------------------------------- +/// Two blocks with the same padded bytes are the same words. +pub proof fn lemma_block_eq(a: Seq, i: int, b: Seq, j: int) + requires + forall|x: int| 64 * i <= x < 64 * i + 64 ==> #[trigger] padded_byte(a, x) == padded_byte(b, x + 64 * (j - i)), + ensures + block_spec(a, i) == block_spec(b, j), +{ + assert forall|w: int| 0 <= w < 16 implies #[trigger] le_word(a, 64 * i + 4 * w) == le_word(b, 64 * j + 4 * w) by { + assert(padded_byte(a, 64 * i + 4 * w) == padded_byte(b, 64 * j + 4 * w)); + assert(padded_byte(a, 64 * i + 4 * w + 1) == padded_byte(b, 64 * j + 4 * w + 1)); + assert(padded_byte(a, 64 * i + 4 * w + 2) == padded_byte(b, 64 * j + 4 * w + 2)); + assert(padded_byte(a, 64 * i + 4 * w + 3) == padded_byte(b, 64 * j + 4 * w + 3)); + } + assert(block_spec(a, i) =~= block_spec(b, j)); +} + +/// The chaining value after `k` blocks depends on the first `64 k` bytes only. +pub proof fn lemma_chain_prefix(h: Seq, a: Seq, b: Seq, t0: int, k: nat) + requires + forall|x: int| 0 <= x < 64 * k ==> #[trigger] padded_byte(a, x) == padded_byte(b, x), + ensures + chain_spec(h, a, t0, k) == chain_spec(h, b, t0, k), + decreases k, +{ + if k > 0 { + lemma_chain_prefix(h, a, b, t0, (k - 1) as nat); + lemma_block_eq(a, k - 1, b, k - 1); + } +} + +/// Chaining over `a + b`, `a` being `ka` whole blocks, is chaining over `a` then over `b` from counter `64 ka`. +pub proof fn lemma_chain_concat(h: Seq, a: Seq, b: Seq, t0: int, ka: nat, k: nat) + requires + a.len() == 64 * ka, + ensures + chain_spec(h, a + b, t0, ka + k) == chain_spec(chain_spec(h, a, t0, ka), b, t0 + 64 * ka, k), + decreases k, +{ + if k == 0 { + assert forall|x: int| 0 <= x < 64 * ka implies #[trigger] padded_byte(a + b, x) == padded_byte(a, x) by {} + lemma_chain_prefix(h, a + b, a, t0, ka); + } else { + lemma_chain_concat(h, a, b, t0, ka, (k - 1) as nat); + assert forall|x: int| 64 * (ka + k - 1) <= x < 64 * (ka + k - 1) + 64 implies #[trigger] padded_byte(a + b, x) + == padded_byte(b, x + 64 * ((k - 1) - (ka + k - 1))) by {} + lemma_block_eq(a + b, ka + k - 1, b, k - 1); + assert(((ka + k) - 1) as nat == ka + (k - 1) as nat); + } +} + +/// Continuation: finishing `a + b` from `h`, `a` being `ka` whole blocks and `b` not empty, is finishing `b` from the +/// chaining value after `a`, at counter offset `64 ka`. +pub proof fn lemma_finish_concat(h: Seq, a: Seq, b: Seq, t0: int, ka: nat) + requires + a.len() == 64 * ka, + b.len() > 0, + ensures + finish_spec(h, a + b, t0) == finish_spec(chain_spec(h, a, t0, ka), b, t0 + 64 * ka), +{ + let db = block_count(b.len() as int); + let dab = block_count((a + b).len() as int); + assert(dab == ka + db); + lemma_chain_concat(h, a, b, t0, ka, (db - 1) as nat); + assert((dab - 1) as nat == ka + (db - 1) as nat); + assert forall|x: int| 64 * (dab - 1) <= x < 64 * (dab - 1) + 64 implies #[trigger] padded_byte(a + b, x) + == padded_byte(b, x + 64 * ((db - 1) - (dab - 1))) by {} + lemma_block_eq(a + b, dab - 1, b, db - 1); +} + +/// `n` zero bytes. +pub open spec fn zeros(n: nat) -> Seq { + Seq::new(n, |i: int| 0u8) +} + +/// The claim of `zero_prefix_state`'s doc: continuing from the chaining value after `z` zero blocks, at counter +/// offset `64 z`, gives the hash of the whole image, the `64 z` zero bytes then `rest` (not empty). +pub proof fn lemma_zero_prefix_continuation(z: nat, rest: Seq) + requires + rest.len() > 0, + ensures + finish_spec(chain_spec(param_iv(), zeros(64 * z), 0, z), rest, 64 * z as int) == blake2s_spec(zeros(64 * z) + rest), +{ + lemma_finish_concat(param_iv(), zeros(64 * z), rest, 0, z); +} + +// --------------------------------------------------------------------------------------------- +// Production: bytes, the one-shot and streaming hashes +// --------------------------------------------------------------------------------------------- +/// Four bytes assembled with shifts are their little-endian word. +proof fn lemma_le_word(b0: u8, b1: u8, b2: u8, b3: u8) + ensures + (b0 as u32 | (b1 as u32) << 8u32 | (b2 as u32) << 16u32 | (b3 as u32) << 24u32) == (b0 as int + 0x100 * b1 + + 0x1_0000 * b2 + 0x100_0000 * b3) as u32, +{ + assert((b0 as u32 | (b1 as u32) << 8u32 | (b2 as u32) << 16u32 | (b3 as u32) << 24u32) == add( + add(add(b0 as u32, mul(b1 as u32, 0x100u32)), mul(b2 as u32, 0x1_0000u32)), + mul(b3 as u32, 0x100_0000u32), + )) by (bit_vector); +} + +/// The four bytes of a word, by shifts and truncation. +proof fn lemma_word_bytes(x: u32) + ensures + x as u8 == word_byte(x, 0), + (x >> 8u32) as u8 == word_byte(x, 1), + (x >> 16u32) as u8 == word_byte(x, 2), + (x >> 24u32) as u8 == word_byte(x, 3), +{ + assert(x as u8 == (x % 0x100) as u8 && (x >> 8u32) as u8 == (x / 0x100 % 0x100) as u8 && (x >> 16u32) as u8 == (x + / 0x1_0000 % 0x100) as u8 && (x >> 24u32) as u8 == (x / 0x100_0000 % 0x100) as u8) by (bit_vector); +} + +/// Read a 64-byte block as 16 little-endian words. +/// +/// Verus form: `array::from_fn` over `u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap())` is an index +/// loop assembling each word from its four bytes with shifts. +pub fn block_words(block: &[u8; BLOCK_LEN]) -> (words: [u32; 16]) + ensures + words@ == block_spec(block@, 0), +{ + let mut words = [0u32; 16]; + for i in 0..16 + invariant + forall|j: int| 0 <= j < i ==> #[trigger] words[j] == le_word(block@, 4 * j), + { + words[i] = block[4 * i] as u32 | (block[4 * i + 1] as u32) << 8 | (block[4 * i + 2] as u32) << 16 + | (block[4 * i + 3] as u32) << 24; + proof { + lemma_le_word(block[4 * i as int], block[4 * i + 1], block[4 * i + 2], block[4 * i + 3]); + } + } + assert(words@ =~= block_spec(block@, 0)); + words +} + +/// Serialize a chaining value as the 32-byte digest. +/// +/// Verus form: the loop over `out.as_chunks_mut::<4>()` zipped with `h`, storing `word.to_le_bytes()`, is an index +/// loop writing each word's four bytes with shifts. +pub fn state_bytes(h: &[u32; 8]) -> (out: [u8; OUT_LEN]) + ensures + out@ == digest_spec(h@), +{ + let mut out = [0u8; OUT_LEN]; + for i in 0..8 + invariant + forall|j: int| 0 <= j < 4 * i ==> #[trigger] out[j] == word_byte(h[j / 4], j % 4), + { + out[4 * i] = h[i] as u8; + out[4 * i + 1] = (h[i] >> 8) as u8; + out[4 * i + 2] = (h[i] >> 16) as u8; + out[4 * i + 3] = (h[i] >> 24) as u8; + proof { + lemma_word_bytes(h[i as int]); + } + } + assert(out@ =~= digest_spec(h@)); + out +} + +/// `PARAM_IV` is the RFC's parameter block XORed into the IV. +proof fn lemma_param_iv() + ensures + PARAM_IV@ == param_iv(), +{ + assert(0x6A09_E667u32 ^ 0x0101_0000u32 ^ 32u32 == 0x6A09_E667u32 ^ 0x0101_0000u32 ^ (0u32 << 8u32) ^ 32u32) + by (bit_vector); + assert(PARAM_IV@ =~= param_iv()); +} + +/// One step of the whole-block loop of `hash` and `hash_from_state`: compressing block `b` of `n` gives the chaining +/// value after `b + 1` blocks, or the final one after the last. +pub open spec fn whole_blocks_state(h: Seq, d: Seq, t0: int, b: nat) -> Seq { + if b * 64 < d.len() { + chain_spec(h, d, t0, b) + } else { + f_spec(chain_spec(h, d, t0, (b - 1) as nat), block_spec(d, b - 1), t0 + 64 * b, true) + } +} + +/// The block of a whole-block message copied out of it is that block. +proof fn lemma_copied_block(d: Seq, b: int, block: Seq) + requires + 0 <= b, + 64 * b + 64 <= d.len(), + block == d.subrange(64 * b, 64 * b + 64), + ensures + block_spec(block, 0) == block_spec(d, b), +{ + assert forall|x: int| 64 * 0 <= x < 64 * 0 + 64 implies #[trigger] padded_byte(block, x) == padded_byte(d, x + 64 * (b - 0)) by {} + lemma_block_eq(block, 0, d, b); +} + +/// One-shot unkeyed BLAKE2s-256. +/// +/// Verus form: the loop over `data.as_chunks::()` with `enumerate` is an index loop that copies block `b` +/// into an array with `copy_from_slice`. +pub fn hash(data: &[u8]) -> (digest: [u8; OUT_LEN]) + ensures + digest@ == blake2s_spec(data@), +{ + // Whole blocks, the shape hashed in bulk, need no buffering. + if !data.is_empty() && data.len().is_multiple_of(BLOCK_LEN) { + let mut h = PARAM_IV; + let n = data.len() / BLOCK_LEN; + proof { + lemma_param_iv(); + } + for b in 0..n + invariant + n * 64 == data.len(), + n > 0, + h@ == whole_blocks_state(param_iv(), data@, 0, b as nat), + { + let mut block = [0u8; BLOCK_LEN]; + block.copy_from_slice(&data[b * BLOCK_LEN..(b + 1) * BLOCK_LEN]); + let t = ((b + 1) * BLOCK_LEN) as u64; + proof { + lemma_copied_block(data@, b as int, block@); + } + compress(&mut h, &block_words(&block), t, b + 1 == n); + } + return state_bytes(&h); + } + let mut hasher = Hasher::new(); + hasher.update(data); + hasher.finalize() +} + +/// Streaming BLAKE2s-256. +/// +/// The final block carries the final flag, so a full buffer is held back until more input arrives. +/// +/// Verus form: the ghost field `msg` is added for the proof; it is erased from the compiled struct. +#[derive(Clone)] +#[verifier::allow(autoderive_clone_without_spec)] +pub struct Hasher { + h: [u32; 8], + buf: [u8; BLOCK_LEN], + /// Bytes currently in `buf`, in `0..=BLOCK_LEN`. + buf_len: usize, + /// Bytes already compressed. + counter: u64, + /// Every byte passed to `update` so far. + msg: Ghost>, +} + +impl Hasher { + /// The bytes passed to `update` so far. + pub closed spec fn absorbed(&self) -> Seq { + self.msg@ + } + + /// The streaming invariant: the absorbed bytes are the compressed blocks (whole, counted by `counter`, chained + /// into `h`) then the `buf_len` buffered bytes, at least one once anything is absorbed. + pub closed spec fn wf(&self) -> bool { + let msg = self.msg@; + &&& self.buf_len <= BLOCK_LEN + &&& msg.len() == self.counter + self.buf_len + &&& msg.len() < 0x1_0000_0000_0000_0000 + &&& self.counter % 64 == 0 + &&& msg.len() > 0 ==> self.buf_len > 0 + &&& self.h@ == chain_spec(param_iv(), msg, 0, (self.counter / 64) as nat) + &&& forall|j: int| 0 <= j < self.buf_len ==> #[trigger] self.buf[j] == msg[self.counter + j] + } + + pub const fn new() -> (hasher: Self) + ensures + hasher.wf(), + hasher.absorbed() == Seq::::empty(), + { + proof { + lemma_param_iv(); + } + Self { h: PARAM_IV, buf: [0u8; BLOCK_LEN], buf_len: 0, counter: 0, msg: Ghost(Seq::empty()) } + } + + /// The total length stays below `2^64`, the range of the RFC's counter. + /// + /// The returned reference is `self`, for chaining: the hasher's final value is the one left behind it. + pub fn update(&mut self, mut data: &[u8]) -> (this: &mut Self) + requires + old(self).wf(), + old(self).absorbed().len() + data.len() < 0x1_0000_0000_0000_0000, + ensures + this.wf(), + this.absorbed() == old(self).absorbed() + data@, + *final(this) == *final(self), + { + let ghost total = self.msg@ + data@; + while !data.is_empty() + invariant + self.wf(), + self.msg@ + data@ == total, + total.len() < 0x1_0000_0000_0000_0000, + decreases data.len(), + { + if self.buf_len == BLOCK_LEN { + // More input follows, so this buffered block is not the last. + let ghost msg = self.msg@; + proof { + assert forall|x: int| 64 * 0 <= x < 64 * 0 + 64 implies #[trigger] padded_byte(self.buf@, x) + == padded_byte(msg, x + 64 * (self.counter / 64 - 0)) by {} + lemma_block_eq(self.buf@, 0, msg, self.counter as int / 64); + } + self.counter += BLOCK_LEN as u64; + compress(&mut self.h, &block_words(&self.buf), self.counter, false); + self.buf_len = 0; + } + let take = (BLOCK_LEN - self.buf_len).min(data.len()); + self.buf[self.buf_len..self.buf_len + take].copy_from_slice(&data[..take]); + self.buf_len += take; + let ghost old_msg = self.msg@; + self.msg = Ghost(self.msg@ + data@.subrange(0, take as int)); + proof { + assert forall|x: int| 0 <= x < 64 * (self.counter / 64) implies #[trigger] padded_byte(old_msg, x) + == padded_byte(self.msg@, x) by {} + lemma_chain_prefix(param_iv(), old_msg, self.msg@, 0, (self.counter / 64) as nat); + } + data = &data[take..]; + } + self + } + + /// Verus form: `block[self.buf_len..].fill(0)` is a loop. + pub fn finalize(&self) -> (digest: [u8; OUT_LEN]) + requires + self.wf(), + ensures + digest@ == blake2s_spec(self.absorbed()), + { + let mut h = self.h; + let mut block = self.buf; + let mut i = self.buf_len; + while i < BLOCK_LEN + invariant + self.buf_len <= i <= BLOCK_LEN, + forall|j: int| 0 <= j < self.buf_len ==> #[trigger] block[j] == self.buf[j], + forall|j: int| self.buf_len <= j < i ==> #[trigger] block[j] == 0, + decreases BLOCK_LEN - i, + { + block[i] = 0; + i += 1; + } + let t = self.counter + self.buf_len as u64; + let ghost msg = self.msg@; + proof { + let k = self.counter as int / 64; + assert(block_count(msg.len() as int) - 1 == k); + assert forall|x: int| 64 * 0 <= x < 64 * 0 + 64 implies #[trigger] padded_byte(block@, x) + == padded_byte(msg, x + 64 * (k - 0)) by {} + lemma_block_eq(block@, 0, msg, k); + } + compress(&mut h, &block_words(&block), t, true); + state_bytes(&h) + } +} + +/// The chaining value after `n_blocks` all-zero blocks from the unkeyed IV. +/// +/// PCS leaves share such a prefix, their absent interleaving lanes. +/// +/// The committer absorbs it once and starts every leaf here. +/// +/// Digests are unchanged: a prefix's compressions depend on nothing after them ([`lemma_zero_prefix_continuation`]). +pub fn zero_prefix_state(n_blocks: usize) -> (h: [u32; 8]) + requires + n_blocks * 64 <= usize::MAX, + ensures + h@ == chain_spec(param_iv(), zeros(64 * n_blocks as nat), 0, n_blocks as nat), +{ + let mut h = PARAM_IV; + proof { + lemma_param_iv(); + } + for b in 0..n_blocks + invariant + n_blocks * 64 <= usize::MAX, + h@ == chain_spec(param_iv(), zeros(64 * n_blocks as nat), 0, b as nat), + { + proof { + let z = zeros(64 * n_blocks as nat); + assert forall|w: int| 0 <= w < 16 implies #[trigger] le_word(z, 64 * b + 4 * w) == 0 by {} + // The spec of the exec array `[0u32; 16]`. + let zero = vstd::array::spec_array_fill_for_copy_type::(0u32); + assert(zero@.len() == 16); + assert(block_spec(z, b as int) =~= zero@); + } + compress(&mut h, &[0u32; 16], ((b + 1) * BLOCK_LEN) as u64, false); + } + h +} + +/// The one-shot hash, continued from a chaining value. +/// +/// - `data` is the rest of the image, a nonzero whole number of blocks. +/// - `t_offset` counts the bytes already absorbed into `state`. +/// +/// Verus form: the `assert!` on `data` is a `requires`, as is the counter's range; the loop over +/// `data.as_chunks::()` with `enumerate` is an index loop copying block `b` out with `copy_from_slice`. +pub fn hash_from_state(data: &[u8], state: &[u32; 8], t_offset: u64) -> (digest: [u8; OUT_LEN]) + requires + data.len() > 0 && data.len() % 64 == 0, + t_offset + data.len() < 0x1_0000_0000_0000_0000, + ensures + digest@ == finish_spec(state@, data@, t_offset as int), +{ + let mut h = *state; + let n = data.len() / BLOCK_LEN; + for b in 0..n + invariant + n * 64 == data.len(), + n > 0, + t_offset + data.len() < 0x1_0000_0000_0000_0000, + h@ == whole_blocks_state(state@, data@, t_offset as int, b as nat), + { + let mut block = [0u8; BLOCK_LEN]; + block.copy_from_slice(&data[b * BLOCK_LEN..(b + 1) * BLOCK_LEN]); + let t = t_offset + ((b + 1) * BLOCK_LEN) as u64; + proof { + lemma_copied_block(data@, b as int, block@); + } + compress(&mut h, &block_words(&block), t, b + 1 == n); + } + state_bytes(&h) +} + +} diff --git a/verification/verus/src/blake2s_batch.rs b/verification/verus/src/blake2s_batch.rs new file mode 100644 index 0000000000..30a8abd2a9 --- /dev/null +++ b/verification/verus/src/blake2s_batch.rs @@ -0,0 +1,682 @@ +//! Batched BLAKE2s: portable compression copies against the RFC 7693 model in `blake2s`. +//! +//! Lane arithmetic contracts are proved for `Scalar8`. Safe array layout models connect transposed +//! states and messages to lane-wise RFC compression; production pointer operations, the batch +//! driver, dispatch and SIMD instructions are not verified. +use crate::blake2s::*; +use vstd::prelude::*; + +/// The G function over LITERAL state and message indices. +/// +/// `$m` is the transposed block, so each message operand is a load at a constant offset. +/// +/// Verus form: an inline helper gives each G a separate proof context. +macro_rules! g { + ($v:ident, $m:ident, $a:expr, $b:expr, $c:expr, $d:expr, $x:expr, $y:expr) => { + ::vstd::prelude::verus_exec_expr! { + mix($v, $m, $a, $b, $c, $d, $x, $y) + } + }; +} + +/// One round with the message schedule `s`: columns, then diagonals. +/// +/// Verus form: the ghost lines (snapshots of `$v` between the G's, and [`round_ok`] at the end) are added. +macro_rules! round { + ($v:ident, $m:ident, [$s0:expr, $s1:expr, $s2:expr, $s3:expr, $s4:expr, $s5:expr, $s6:expr, $s7:expr, + $s8:expr, $s9:expr, $s10:expr, $s11:expr, $s12:expr, $s13:expr, $s14:expr, $s15:expr]) => { + ::vstd::prelude::verus_exec_expr! { { + let ghost v0 = *$v; + g!($v, $m, 0, 4, 8, 12, $s0, $s1); + let ghost v1 = *$v; + g!($v, $m, 1, 5, 9, 13, $s2, $s3); + let ghost v2 = *$v; + g!($v, $m, 2, 6, 10, 14, $s4, $s5); + let ghost v3 = *$v; + g!($v, $m, 3, 7, 11, 15, $s6, $s7); + let ghost v4 = *$v; + g!($v, $m, 0, 5, 10, 15, $s8, $s9); + let ghost v5 = *$v; + g!($v, $m, 1, 6, 11, 12, $s10, $s11); + let ghost v6 = *$v; + g!($v, $m, 2, 7, 8, 13, $s12, $s13); + let ghost v7 = *$v; + g!($v, $m, 3, 4, 9, 14, $s14, $s15); + proof { + lemma_round_ok(v0, v1, v2, v3, v4, v5, v6, v7, *$v, *$m, + seq![$s0, $s1, $s2, $s3, $s4, $s5, $s6, $s7, $s8, $s9, $s10, $s11, $s12, $s13, $s14, $s15]); + } + } } + }; +} + +/// The ten rounds as out-of-line functions, for interleaving groups. +/// +/// Verus form: the functions carry their specification, inside `verus!`. +macro_rules! round_fns { + ($($name:ident [$($s:expr),*],)*) => { + ::vstd::prelude::verus! { + $( + #[inline(never)] + fn $name(v: &mut [S; 16], m: &[S; 16]) + ensures + round_ok(*final(v), *old(v), *m, seq![$($s),*]), + { + round!(v, m, [$($s),*]) + } + )* + } + }; +} + +round_fns! { + round_0 [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], + round_1 [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3], + round_2 [11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4], + round_3 [7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8], + round_4 [9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13], + round_5 [2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9], + round_6 [12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11], + round_7 [13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10], + round_8 [6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5], + round_9 [10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0], +} + +/// Production's `round_all!`, local to `compress_groups`: each round over every group, in turn. +/// +/// Verus form: each group loop is an inline helper with the work vectors' initial ghost snapshot and +/// a round-indexed contract. Its invariant advances groups before `g` by one round. +macro_rules! round_all { + ($v:ident, $m:ident, $v0:ident, $($name:ident $advance:ident $k:literal),*) => { + $( + ::vstd::prelude::verus_exec_expr! { { + #[inline(always)] + fn $advance( + v: &mut [[S; 16]; G], m: [&[S; 16]; G], Ghost(v0): Ghost<[[S; 16]; G]>) + requires + groups_rounds_ok(*old(v), v0, m, $k as nat), + ensures + groups_rounds_ok(*final(v), v0, m, ($k + 1) as nat), + { + proof { reveal(groups_rounds_ok); } + for g in 0..G + invariant + forall|j: int| 0 <= j < g ==> #[trigger] rounds_ok(v[j], v0[j], *m[j], ($k + 1) as nat), + forall|j: int| g <= j < G ==> #[trigger] rounds_ok(v[j], v0[j], *m[j], $k as nat), + { + let ghost before = v[g as int]; + $name(&mut v[g], m[g]); + proof { + lemma_round_step(before, v[g as int], *m[g as int], v0[g as int], $k); + } + } + } + $advance(&mut $v, $m, Ghost($v0)); + } } + )* + }; +} + +verus! { + +/// Two additions modulo `2^32` are one. +pub broadcast proof fn lemma_add_mod_twice(x: u32, y: u32, z: u32) + ensures + #[trigger] add_mod(add_mod(x, y, 0), z, 0) == add_mod(x, y, z), +{ +} + +/// One state word across all lanes of a group: a vector of `WIDTH` 32-bit lanes. +/// +/// Each backend supplies the lane arithmetic, specified lane by lane through [`Lanes32::lane`]. +/// +/// Verus form: only the lane arithmetic is copied. The raw-pointer methods (`load`, `store`, the block `transpose` and +/// `store_digests`), the walk's tuning constants (`GROUPS`, `TRANSPOSE_AHEAD`) and the default `compress_groups`, which +/// forwards to the free [`compress_groups`] (a trait method's specification cannot name the trait's own bound), are not. +pub trait Lanes32: Copy + Sized { + /// Lanes per vector. + const WIDTH: usize; + + /// Lane `l` of the vector, for `0 <= l < WIDTH`. + spec fn lane(&self, l: int) -> u32; + + /// Every lane set to `x`. + fn splat(x: u32) -> (r: Self) + ensures + forall|l: int| 0 <= l < Self::WIDTH ==> #[trigger] r.lane(l) == x; + + /// Lane-wise wrapping addition. + fn add(self, o: Self) -> (r: Self) + ensures + forall|l: int| 0 <= l < Self::WIDTH ==> #[trigger] r.lane(l) == add_mod(self.lane(l), o.lane(l), 0); + + /// Lane-wise xor. + fn xor(self, o: Self) -> (r: Self) + ensures + forall|l: int| 0 <= l < Self::WIDTH ==> #[trigger] r.lane(l) == self.lane(l) ^ o.lane(l); + + /// Rotate every lane right by `N`, one of BLAKE2s's 16, 12, 8, 7. + fn rotr(self) -> (r: Self) + requires + 0 < N < 32, + ensures + forall|l: int| 0 <= l < Self::WIDTH ==> #[trigger] r.lane(l) == rotr(self.lane(l), N); +} + +/// Lane `l` of a transposed chaining value: that lane's own 8 words. +/// +/// Opaque, as [`col16`]. +#[verifier::opaque] +pub open spec fn col8(h: [S; 8], l: int) -> Seq { + Seq::new(8, |i: int| h[i].lane(l)) +} + +/// Lane `l` of a transposed work vector or block: that lane's own 16 words. +/// +/// Opaque: the round-level proofs relate whole lanes; only one G, `init` and the feed-forward look at single words. +#[verifier::opaque] +pub open spec fn col16(v: [S; 16], l: int) -> Seq { + Seq::new(16, |i: int| v[i].lane(l)) +} + +/// The main statement of the batched compression: in every group `g` and lane `l`, the chaining value `after` is the +/// RFC compression of that lane's chaining value `before` with that lane's block of `m[g]`. +pub open spec fn compresses_lanes( + before: [[S; 8]; G], + after: [[S; 8]; G], + m: [&[S; 16]; G], + t: u64, + last: bool, +) -> bool { + forall|g: int, l: int| + 0 <= g < G && 0 <= l < S::WIDTH ==> #[trigger] col8(after[g], l) == f_spec( + col8(before[g], l), + col16(*m[g], l), + t as int, + last, + ) +} + +/// A round with the schedule `s` written out: [`round_spec`] with `s = sigma(i)`. +pub open spec fn round_sched(v: Seq, m: Seq, s: Seq) -> Seq { + let v = g_spec(v, 0, 4, 8, 12, m[s[0]], m[s[1]]); + let v = g_spec(v, 1, 5, 9, 13, m[s[2]], m[s[3]]); + let v = g_spec(v, 2, 6, 10, 14, m[s[4]], m[s[5]]); + let v = g_spec(v, 3, 7, 11, 15, m[s[6]], m[s[7]]); + let v = g_spec(v, 0, 5, 10, 15, m[s[8]], m[s[9]]); + let v = g_spec(v, 1, 6, 11, 12, m[s[10]], m[s[11]]); + let v = g_spec(v, 2, 7, 8, 13, m[s[12]], m[s[13]]); + g_spec(v, 3, 4, 9, 14, m[s[14]], m[s[15]]) +} + +// --------------------------------------------------------------------------------------------- +// The portable backend +// --------------------------------------------------------------------------------------------- +/// The portable backend, and the reference the SIMD ones are checked against. +/// +/// Dispatch is compile time, so only tests use it where a SIMD backend exists. +#[derive(Clone, Copy)] +pub struct Scalar8(pub [u32; 8]); + +/// Safe word-level form of the portable block/state transpose. +/// Array indexing replaces pointer gathers and vector loads; allocation and pointer safety are not modeled. +pub fn transpose_words(rows: &[[u32; W]; 8]) -> (out: [Scalar8; W]) + ensures + forall|w: int, l: int| 0 <= w < W && 0 <= l < 8 ==> #[trigger] out[w].0[l] == rows[l][w], +{ + let mut out = [Scalar8([0u32; 8]); W]; + for l in 0..8 + invariant + forall|w: int, j: int| 0 <= w < W && 0 <= j < l ==> #[trigger] out[w].0[j] == rows[j][w], + { + for w in 0..W + invariant + l < 8, + forall|i: int, j: int| 0 <= i < W && 0 <= j < l ==> #[trigger] out[i].0[j] == rows[j][i], + forall|i: int| 0 <= i < w ==> #[trigger] out[i].0[l as int] == rows[l as int][i], + { + out[w].0[l] = rows[l][w]; + } + } + out +} + +/// Safe byte-array form of the portable digest scatter, reusing the verified little-endian encoder. +pub fn store_digests(h: &[Scalar8; 8]) -> (out: [[u8; 32]; 8]) + ensures + forall|l: int| 0 <= l < 8 ==> #[trigger] out[l]@ == digest_spec(col8(*h, l)), +{ + let mut out = [[0u8; 32]; 8]; + for l in 0..8 + invariant + forall|j: int| 0 <= j < l ==> #[trigger] out[j]@ == digest_spec(col8(*h, j)), + { + let mut words = [0u32; 8]; + for w in 0..8 + invariant + l < 8, + forall|i: int| 0 <= i < w ==> #[trigger] words[i] == h[i].0[l as int], + { + words[w] = h[w].0[l]; + } + proof { + reveal(col8); + assert(words@ =~= col8(*h, l as int)); + } + out[l] = state_bytes(&words); + } + out +} + +impl Lanes32 for Scalar8 { + const WIDTH: usize = 8; + + open spec fn lane(&self, l: int) -> u32 { + self.0[l] + } + + #[inline(always)] + fn splat(x: u32) -> (r: Self) { + Self([x; 8]) + } + + /// Verus form: `array::from_fn` is an index loop. + #[inline(always)] + fn add(self, o: Self) -> (r: Self) { + let mut r = [0u32; 8]; + for i in 0..8 + invariant + forall|j: int| 0 <= j < i ==> #[trigger] r[j] == add_mod(self.0[j], o.0[j], 0), + { + r[i] = self.0[i].wrapping_add(o.0[i]); + proof { + lemma_add_mod2(self.0[i as int], o.0[i as int]); + } + } + Self(r) + } + + /// Verus form: `array::from_fn` is an index loop. + #[inline(always)] + fn xor(self, o: Self) -> (r: Self) { + let mut r = [0u32; 8]; + for i in 0..8 + invariant + forall|j: int| 0 <= j < i ==> #[trigger] r[j] == self.0[j] ^ o.0[j], + { + r[i] = self.0[i] ^ o.0[i]; + } + Self(r) + } + + /// Verus form: `array::from_fn` is an index loop. + #[inline(always)] + fn rotr(self) -> (r: Self) { + let mut r = [0u32; 8]; + for i in 0..8 + invariant + 0 < N < 32, + forall|j: int| 0 <= j < i ==> #[trigger] r[j] == rotr(self.0[j], N), + { + r[i] = self.0[i].rotate_right(N); + } + Self(r) + } +} + +// --------------------------------------------------------------------------------------------- +// Rounds and the compression +// --------------------------------------------------------------------------------------------- +/// In every lane, `after` is G on `before` at state words `a, b, c, d` with message words `x, y` of `m`. +/// +/// Opaque, as [`round_ok`] and [`rounds_ok`]: each step's statement stays an atom in the long straight-line bodies, +/// so the solver does not chain back through every earlier step. +#[verifier::opaque] +pub open spec fn g_ok( + after: [S; 16], + before: [S; 16], + m: [S; 16], + a: int, + b: int, + c: int, + d: int, + x: int, + y: int, +) -> bool { + forall|l: int| + 0 <= l < S::WIDTH ==> #[trigger] col16(after, l) == g_spec(col16(before, l), a, b, c, d, col16(m, l)[x], col16(m, l)[y]) +} + +/// In every lane, `after` is one round with schedule `s` on `before`. +#[verifier::opaque] +pub open spec fn round_ok(after: [S; 16], before: [S; 16], m: [S; 16], s: Seq) -> bool { + forall|l: int| 0 <= l < S::WIDTH ==> #[trigger] col16(after, l) == round_sched(col16(before, l), col16(m, l), s) +} + +/// In every lane, `v` is rounds `0..k` on `v0`. +#[verifier::opaque] +pub open spec fn rounds_ok(v: [S; 16], v0: [S; 16], m: [S; 16], k: nat) -> bool { + forall|l: int| 0 <= l < S::WIDTH ==> #[trigger] col16(v, l) == rounds_spec(col16(v0, l), col16(m, l), k) +} + +/// Keep the inter-round group invariant opaque between independently verified group loops. +#[verifier::opaque] +pub open spec fn groups_rounds_ok( + v: [[S; 16]; G], v0: [[S; 16]; G], m: [&[S; 16]; G], k: nat) -> bool { + forall|j: int| 0 <= j < G ==> #[trigger] rounds_ok(v[j], v0[j], *m[j], k) +} + +proof fn lemma_g_ok(after: [S; 16], before: [S; 16], m: [S; 16], a: int, b: int, c: int, d: int, x: int, y: int) + requires + forall|l: int| 0 <= l < S::WIDTH ==> #[trigger] col16(after, l) == g_spec(col16(before, l), a, b, c, d, col16(m, l)[x], col16(m, l)[y]), + ensures + g_ok(after, before, m, a, b, c, d, x, y), +{ + reveal(g_ok); +} + +/// The production G operations, isolated so lane arithmetic facts do not accumulate across rounds. +#[inline(always)] +fn mix(v: &mut [S; 16], m: &[S; 16], a: usize, b: usize, c: usize, d: usize, x: usize, y: usize) + requires + a < 16, b < 16, c < 16, d < 16, x < 16, y < 16, + a != b, a != c, a != d, b != c, b != d, c != d, + ensures + g_ok(*final(v), *old(v), *m, a as int, b as int, c as int, d as int, x as int, y as int), +{ + let ghost before = *v; + v[a] = v[a].add(v[b]).add(m[x]); + v[d] = v[d].xor(v[a]).rotr::<16>(); + v[c] = v[c].add(v[d]); + v[b] = v[b].xor(v[c]).rotr::<12>(); + v[a] = v[a].add(v[b]).add(m[y]); + v[d] = v[d].xor(v[a]).rotr::<8>(); + v[c] = v[c].add(v[d]); + v[b] = v[b].xor(v[c]).rotr::<7>(); + proof { + assert forall|l: int| 0 <= l < S::WIDTH implies #[trigger] col16(*v, l) == g_spec( + col16(before, l), a as int, b as int, c as int, d as int, col16(*m, l)[x as int], col16(*m, l)[y as int]) by { + broadcast use lemma_add_mod_twice; + reveal(g_spec); + reveal(col16); + assert(col16(*v, l) =~= g_spec(col16(before, l), a as int, b as int, c as int, d as int, + m[x as int].lane(l), m[y as int].lane(l))); + } + lemma_g_ok(*v, before, *m, a as int, b as int, c as int, d as int, x as int, y as int); + } +} + +/// The eight G's of a round, columns then diagonals, make the round. +proof fn lemma_round_ok( + v0: [S; 16], + v1: [S; 16], + v2: [S; 16], + v3: [S; 16], + v4: [S; 16], + v5: [S; 16], + v6: [S; 16], + v7: [S; 16], + v8: [S; 16], + m: [S; 16], + s: Seq, +) + requires + s.len() == 16, + g_ok(v1, v0, m, 0, 4, 8, 12, s[0], s[1]), + g_ok(v2, v1, m, 1, 5, 9, 13, s[2], s[3]), + g_ok(v3, v2, m, 2, 6, 10, 14, s[4], s[5]), + g_ok(v4, v3, m, 3, 7, 11, 15, s[6], s[7]), + g_ok(v5, v4, m, 0, 5, 10, 15, s[8], s[9]), + g_ok(v6, v5, m, 1, 6, 11, 12, s[10], s[11]), + g_ok(v7, v6, m, 2, 7, 8, 13, s[12], s[13]), + g_ok(v8, v7, m, 3, 4, 9, 14, s[14], s[15]), + ensures + round_ok(v8, v0, m, s), +{ + reveal(g_ok); + reveal(round_ok); + assert forall|l: int| 0 <= l < S::WIDTH implies #[trigger] col16(v8, l) == round_sched(col16(v0, l), col16(m, l), s) by { + assert(col16(v1, l) == g_spec(col16(v0, l), 0, 4, 8, 12, col16(m, l)[s[0]], col16(m, l)[s[1]])); + assert(col16(v2, l) == g_spec(col16(v1, l), 1, 5, 9, 13, col16(m, l)[s[2]], col16(m, l)[s[3]])); + assert(col16(v3, l) == g_spec(col16(v2, l), 2, 6, 10, 14, col16(m, l)[s[4]], col16(m, l)[s[5]])); + assert(col16(v4, l) == g_spec(col16(v3, l), 3, 7, 11, 15, col16(m, l)[s[6]], col16(m, l)[s[7]])); + assert(col16(v5, l) == g_spec(col16(v4, l), 0, 5, 10, 15, col16(m, l)[s[8]], col16(m, l)[s[9]])); + assert(col16(v6, l) == g_spec(col16(v5, l), 1, 6, 11, 12, col16(m, l)[s[10]], col16(m, l)[s[11]])); + assert(col16(v7, l) == g_spec(col16(v6, l), 2, 7, 8, 13, col16(m, l)[s[12]], col16(m, l)[s[13]])); + } +} + +/// A round with the production schedule of round `k`, after rounds `0..k`, gives rounds `0..k + 1`, in every lane. +proof fn lemma_round_step(before: [S; 16], after: [S; 16], m: [S; 16], v0: [S; 16], k: int) + requires + 0 <= k < 10, + rounds_ok(before, v0, m, k as nat), + round_ok(after, before, m, sigma(k)), + ensures + rounds_ok(after, v0, m, (k + 1) as nat), +{ + reveal(round_ok); + reveal(rounds_ok); + assert forall|l: int| 0 <= l < S::WIDTH implies #[trigger] col16(after, l) == rounds_spec(col16(v0, l), col16(m, l), (k + 1) as nat) by { + assert(col16(before, l) == rounds_spec(col16(v0, l), col16(m, l), k as nat)); + assert(col16(after, l) == round_sched(col16(before, l), col16(m, l), sigma(k))); + } +} + +/// Bitwise identities of the work vector's initialization. +proof fn lemma_xor_zero(x: u32) + by (bit_vector) + ensures + x ^ 0u32 == x, + 0u32.wrapping_sub(1u32) == 0xFFFF_FFFFu32, +{ +} + +/// The working state of one compression (RFC 7693, section 3.2). +/// +/// ```text +/// v[0..8] chaining value h +/// v[8..16] IV, with v[12] ^= t_lo, v[13] ^= t_hi, v[14] = !v[14] on the final block +/// ``` +/// +/// Verus form: `(last as u32).wrapping_neg()` is `0u32.wrapping_sub(last as u32)` (vstd specifies `wrapping_sub`, not +/// `wrapping_neg`), and the `array::from_fn` is the array written out. +#[inline(always)] +fn init(h: &[S; 8], t: u64, last: bool) -> (v: [S; 16]) + ensures + forall|l: int| 0 <= l < S::WIDTH ==> #[trigger] col16(v, l) == init_spec(col8(*h, l), t as int, last), +{ + // All ones on the final block, zero otherwise. + let f0 = 0u32.wrapping_sub(last as u32); + let row = [ + IV[0], + IV[1], + IV[2], + IV[3], + IV[4] ^ t as u32, + IV[5] ^ (t >> 32) as u32, + IV[6] ^ f0, + IV[7], + ]; + let v = [ + h[0], + h[1], + h[2], + h[3], + h[4], + h[5], + h[6], + h[7], + S::splat(row[0]), + S::splat(row[1]), + S::splat(row[2]), + S::splat(row[3]), + S::splat(row[4]), + S::splat(row[5]), + S::splat(row[6]), + S::splat(row[7]), + ]; + proof { + lemma_cast_t(t); + lemma_not_is_xor(IV[6]); + lemma_xor_zero(IV[6]); + assert(IV@ =~= iv()); + assert forall|l: int| 0 <= l < S::WIDTH implies #[trigger] col16(v, l) == init_spec(col8(*h, l), t as int, last) by { + reveal(col16); + reveal(col8); + assert(col16(v, l) =~= init_spec(col8(*h, l), t as int, last)); + } + } + v +} + +/// Compress `G` groups. +/// +/// One group runs its ten rounds inline, in registers. +/// +/// Several groups keep their states in memory, one out-of-line call per round. +/// +/// Each call fits the register file, and the calls of a round fill each other's stalls. +/// +/// Verus form: not `unsafe` (`m` holds references, so there is nothing for the caller to uphold); the `array::from_fn` +/// building `v` is a loop over the groups, from an array of splats; ghost lines between the rounds; the `G > 1` arm's +/// local `round_all!` macro becomes inline group-loop helpers with round-indexed contracts and invariants. +#[inline(always)] +#[verifier::rlimit(30)] +pub fn compress_groups(h: &mut [[S; 8]; G], m: [&[S; 16]; G], t: u64, last: bool) + ensures + compresses_lanes(*old(h), *final(h), m, t, last), +{ + let mut v: [[S; 16]; G] = [[S::splat(0); 16]; G]; + for g in 0..G + invariant + *h == *old(h), + forall|j: int, l: int| 0 <= j < g && 0 <= l < S::WIDTH ==> #[trigger] col16(v[j], l) == init_spec(col8(h[j], l), t as int, last), + { + v[g] = init(&h[g], t, last); + } + let ghost v0 = v; + // `G` is a constant, so only one of the two arms survives monomorphization. + if G == 1 { + let (v, m) = (&mut v[0], m[0]); + let ghost s = *v; + proof { + assert(rounds_ok(*v, s, *m, 0)) by { + reveal(rounds_ok); + } + } + let ghost b0 = *v; + round!(v, m, [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]); + proof { lemma_round_step(b0, *v, *m, s, 0); } + let ghost b1 = *v; + round!(v, m, [14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3]); + proof { lemma_round_step(b1, *v, *m, s, 1); } + let ghost b2 = *v; + round!(v, m, [11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4]); + proof { lemma_round_step(b2, *v, *m, s, 2); } + let ghost b3 = *v; + round!(v, m, [7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8]); + proof { lemma_round_step(b3, *v, *m, s, 3); } + let ghost b4 = *v; + round!(v, m, [9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13]); + proof { lemma_round_step(b4, *v, *m, s, 4); } + let ghost b5 = *v; + round!(v, m, [2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9]); + proof { lemma_round_step(b5, *v, *m, s, 5); } + let ghost b6 = *v; + round!(v, m, [12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11]); + proof { lemma_round_step(b6, *v, *m, s, 6); } + let ghost b7 = *v; + round!(v, m, [13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10]); + proof { lemma_round_step(b7, *v, *m, s, 7); } + let ghost b8 = *v; + round!(v, m, [6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5]); + proof { lemma_round_step(b8, *v, *m, s, 8); } + let ghost b9 = *v; + round!(v, m, [10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0]); + proof { lemma_round_step(b9, *v, *m, s, 9); } + } else { + proof { + assert forall|j: int| 0 <= j < G implies #[trigger] rounds_ok(v[j], v0[j], *m[j], 0) by { + reveal(rounds_ok); + } + reveal(groups_rounds_ok); + assert(groups_rounds_ok(v, v0, m, 0)); + } + round_all!( + v, m, v0, + round_0 advance_0 0, round_1 advance_1 1, round_2 advance_2 2, round_3 advance_3 3, + round_4 advance_4 4, round_5 advance_5 5, round_6 advance_6 6, round_7 advance_7 7, + round_8 advance_8 8, round_9 advance_9 9 + ); + } + proof { + reveal(groups_rounds_ok); + assert(forall|j: int| 0 <= j < G ==> #[trigger] rounds_ok(v[j], v0[j], *m[j], 10)); + } + // Feed-forward: h[i] ^= v[i] ^ v[i + 8]. + for g in 0..G + invariant + forall|j: int, l: int| 0 <= j < g && 0 <= l < S::WIDTH ==> #[trigger] col8(h[j], l) == f_spec( + col8(old(h)[j], l), col16(*m[j], l), t as int, last), + forall|j: int| g <= j < G ==> #[trigger] h[j] == old(h)[j], + forall|j: int| 0 <= j < G ==> #[trigger] rounds_ok(v[j], v0[j], *m[j], 10), + forall|j: int, l: int| 0 <= j < G && 0 <= l < S::WIDTH ==> #[trigger] col16(v0[j], l) == init_spec( + col8(old(h)[j], l), t as int, last), + { + for i in 0..8 + invariant + g < G, + forall|j: int, l: int| 0 <= j < g && 0 <= l < S::WIDTH ==> #[trigger] col8(h[j], l) == f_spec( + col8(old(h)[j], l), col16(*m[j], l), t as int, last), + forall|j: int| g < j < G ==> #[trigger] h[j] == old(h)[j], + forall|j: int| 0 <= j < G ==> #[trigger] rounds_ok(v[j], v0[j], *m[j], 10), + forall|j: int, l: int| 0 <= j < G && 0 <= l < S::WIDTH ==> #[trigger] col16(v0[j], l) == init_spec( + col8(old(h)[j], l), t as int, last), + forall|k: int, l: int| 0 <= k < i && 0 <= l < S::WIDTH ==> #[trigger] h[g as int][k].lane(l) == old( + h)[g as int][k].lane(l) ^ v[g as int][k].lane(l) ^ v[g as int][k + 8].lane(l), + forall|k: int| i <= k < 8 ==> #[trigger] h[g as int][k] == old(h)[g as int][k], + { + h[g][i] = h[g][i].xor(v[g][i]).xor(v[g][i + 8]); + } + proof { + assert forall|l: int| 0 <= l < S::WIDTH implies #[trigger] col8(h[g as int], l) == f_spec( + col8(old(h)[g as int], l), col16(*m[g as int], l), t as int, last) by { + reveal(f_spec); + reveal(rounds_ok); + assert(rounds_ok(v[g as int], v0[g as int], *m[g as int], 10)); + assert(col16(v[g as int], l) == rounds_spec(col16(v0[g as int], l), col16(*m[g as int], l), 10)); + assert(col16(v0[g as int], l) == init_spec(col8(old(h)[g as int], l), t as int, last)); + reveal(col16); + reveal(col8); + assert(col8(h[g as int], l) =~= f_spec(col8(old(h)[g as int], l), col16(*m[g as int], l), t as int, last)); + } + } + } +} + +/// Safe adapter composing state/block transposition, one-group compression and digest scatter. +/// This adapter models the layout; it is not a copy or proof of the production raw-pointer driver. +pub fn compress_rows(states: &[[u32; 8]; 8], messages: &[[u32; 16]; 8], t: u64, last: bool) + -> (out: [[u8; 32]; 8]) + ensures + forall|l: int| 0 <= l < 8 ==> #[trigger] out[l]@ == digest_spec( + f_spec(states[l]@, messages[l]@, t as int, last)), +{ + let mut h = [transpose_words(states)]; + let block = transpose_words(messages); + let ghost initial = h[0]; + proof { + reveal(col8); + reveal(col16); + assert forall|l: int| #![trigger col8(initial, l)] #![trigger col16(block, l)] + 0 <= l < 8 implies col8(initial, l) == states[l]@ && col16(block, l) == messages[l]@ by { + assert(col8(initial, l) =~= states[l]@); + assert(col16(block, l) =~= messages[l]@); + } + } + compress_groups(&mut h, [&block], t, last); + store_digests(&h[0]) +} + +} diff --git a/verification/verus/src/clmul.rs b/verification/verus/src/clmul.rs new file mode 100644 index 0000000000..c34befe36b --- /dev/null +++ b/verification/verus/src/clmul.rs @@ -0,0 +1,434 @@ +//! Carry-less multiplication: the product of polynomials over GF(2). +//! +//! A polynomial over GF(2) of degree below `w` is a `w`-bit word, bit `i` its coefficient of `x^i`. +//! Addition is XOR, and the product is the schoolbook one: XOR in `b * x^i` for every set bit `i` of `a`. +//! That is the definition below ([`clmul`]), from which every lemma here is proven. +use vstd::prelude::*; + +verus! { + +/// Bit `i` of `a`: its coefficient of `x^i`. +pub open spec fn bit(a: u64, i: nat) -> bool { + (a >> (i as u64)) & 1 == 1 +} + +/// The `i`-th partial product `[a_i] * b * x^i`. +pub open spec fn term(a: u64, b: u128, i: nat) -> u128 { + if bit(a, i) { + b << (i as u128) + } else { + 0 + } +} + +/// `sum_{i < n} [a_i] * b * x^i` over GF(2): XOR of shifted ANDs. +pub open spec fn clmul_upto(a: u64, b: u128, n: nat) -> u128 + decreases n, +{ + if n == 0 { + 0 + } else { + clmul_upto(a, b, (n - 1) as nat) ^ term(a, b, (n - 1) as nat) + } +} + +/// The carry-less product `a * b` of a polynomial `a` of degree below 64 and a polynomial `b`, modulo `x^128`. +/// +/// Every use below has `deg a + deg b < 128`, where the truncation is vacuous (see [`lemma_clmul_bound`]). +pub open spec fn clmul(a: u64, b: u128) -> u128 { + clmul_upto(a, b, 64) +} + +/// The low `n` coefficients of `a`. +pub open spec fn low(a: u64, n: nat) -> u64 { + if n >= 64 { + a + } else { + a & ((1u64 << (n as u64)) - 1) as u64 + } +} + +/// Two words with the same bits are equal. +pub proof fn lemma_u64_bits_eq(x: u64, y: u64) + requires + forall|j: nat| j < 64 ==> #[trigger] bit(x, j) == bit(y, j), + ensures + x == y, +{ + assert(bit(x, 0) == bit(y, 0)); + assert(bit(x, 1) == bit(y, 1)); + assert(bit(x, 2) == bit(y, 2)); + assert(bit(x, 3) == bit(y, 3)); + assert(bit(x, 4) == bit(y, 4)); + assert(bit(x, 5) == bit(y, 5)); + assert(bit(x, 6) == bit(y, 6)); + assert(bit(x, 7) == bit(y, 7)); + assert(bit(x, 8) == bit(y, 8)); + assert(bit(x, 9) == bit(y, 9)); + assert(bit(x, 10) == bit(y, 10)); + assert(bit(x, 11) == bit(y, 11)); + assert(bit(x, 12) == bit(y, 12)); + assert(bit(x, 13) == bit(y, 13)); + assert(bit(x, 14) == bit(y, 14)); + assert(bit(x, 15) == bit(y, 15)); + assert(bit(x, 16) == bit(y, 16)); + assert(bit(x, 17) == bit(y, 17)); + assert(bit(x, 18) == bit(y, 18)); + assert(bit(x, 19) == bit(y, 19)); + assert(bit(x, 20) == bit(y, 20)); + assert(bit(x, 21) == bit(y, 21)); + assert(bit(x, 22) == bit(y, 22)); + assert(bit(x, 23) == bit(y, 23)); + assert(bit(x, 24) == bit(y, 24)); + assert(bit(x, 25) == bit(y, 25)); + assert(bit(x, 26) == bit(y, 26)); + assert(bit(x, 27) == bit(y, 27)); + assert(bit(x, 28) == bit(y, 28)); + assert(bit(x, 29) == bit(y, 29)); + assert(bit(x, 30) == bit(y, 30)); + assert(bit(x, 31) == bit(y, 31)); + assert(bit(x, 32) == bit(y, 32)); + assert(bit(x, 33) == bit(y, 33)); + assert(bit(x, 34) == bit(y, 34)); + assert(bit(x, 35) == bit(y, 35)); + assert(bit(x, 36) == bit(y, 36)); + assert(bit(x, 37) == bit(y, 37)); + assert(bit(x, 38) == bit(y, 38)); + assert(bit(x, 39) == bit(y, 39)); + assert(bit(x, 40) == bit(y, 40)); + assert(bit(x, 41) == bit(y, 41)); + assert(bit(x, 42) == bit(y, 42)); + assert(bit(x, 43) == bit(y, 43)); + assert(bit(x, 44) == bit(y, 44)); + assert(bit(x, 45) == bit(y, 45)); + assert(bit(x, 46) == bit(y, 46)); + assert(bit(x, 47) == bit(y, 47)); + assert(bit(x, 48) == bit(y, 48)); + assert(bit(x, 49) == bit(y, 49)); + assert(bit(x, 50) == bit(y, 50)); + assert(bit(x, 51) == bit(y, 51)); + assert(bit(x, 52) == bit(y, 52)); + assert(bit(x, 53) == bit(y, 53)); + assert(bit(x, 54) == bit(y, 54)); + assert(bit(x, 55) == bit(y, 55)); + assert(bit(x, 56) == bit(y, 56)); + assert(bit(x, 57) == bit(y, 57)); + assert(bit(x, 58) == bit(y, 58)); + assert(bit(x, 59) == bit(y, 59)); + assert(bit(x, 60) == bit(y, 60)); + assert(bit(x, 61) == bit(y, 61)); + assert(bit(x, 62) == bit(y, 62)); + assert(bit(x, 63) == bit(y, 63)); + assert(x == y) by (bit_vector) + requires + ((x >> 0u64) & 1 == 1) == ((y >> 0u64) & 1 == 1), + ((x >> 1u64) & 1 == 1) == ((y >> 1u64) & 1 == 1), + ((x >> 2u64) & 1 == 1) == ((y >> 2u64) & 1 == 1), + ((x >> 3u64) & 1 == 1) == ((y >> 3u64) & 1 == 1), + ((x >> 4u64) & 1 == 1) == ((y >> 4u64) & 1 == 1), + ((x >> 5u64) & 1 == 1) == ((y >> 5u64) & 1 == 1), + ((x >> 6u64) & 1 == 1) == ((y >> 6u64) & 1 == 1), + ((x >> 7u64) & 1 == 1) == ((y >> 7u64) & 1 == 1), + ((x >> 8u64) & 1 == 1) == ((y >> 8u64) & 1 == 1), + ((x >> 9u64) & 1 == 1) == ((y >> 9u64) & 1 == 1), + ((x >> 10u64) & 1 == 1) == ((y >> 10u64) & 1 == 1), + ((x >> 11u64) & 1 == 1) == ((y >> 11u64) & 1 == 1), + ((x >> 12u64) & 1 == 1) == ((y >> 12u64) & 1 == 1), + ((x >> 13u64) & 1 == 1) == ((y >> 13u64) & 1 == 1), + ((x >> 14u64) & 1 == 1) == ((y >> 14u64) & 1 == 1), + ((x >> 15u64) & 1 == 1) == ((y >> 15u64) & 1 == 1), + ((x >> 16u64) & 1 == 1) == ((y >> 16u64) & 1 == 1), + ((x >> 17u64) & 1 == 1) == ((y >> 17u64) & 1 == 1), + ((x >> 18u64) & 1 == 1) == ((y >> 18u64) & 1 == 1), + ((x >> 19u64) & 1 == 1) == ((y >> 19u64) & 1 == 1), + ((x >> 20u64) & 1 == 1) == ((y >> 20u64) & 1 == 1), + ((x >> 21u64) & 1 == 1) == ((y >> 21u64) & 1 == 1), + ((x >> 22u64) & 1 == 1) == ((y >> 22u64) & 1 == 1), + ((x >> 23u64) & 1 == 1) == ((y >> 23u64) & 1 == 1), + ((x >> 24u64) & 1 == 1) == ((y >> 24u64) & 1 == 1), + ((x >> 25u64) & 1 == 1) == ((y >> 25u64) & 1 == 1), + ((x >> 26u64) & 1 == 1) == ((y >> 26u64) & 1 == 1), + ((x >> 27u64) & 1 == 1) == ((y >> 27u64) & 1 == 1), + ((x >> 28u64) & 1 == 1) == ((y >> 28u64) & 1 == 1), + ((x >> 29u64) & 1 == 1) == ((y >> 29u64) & 1 == 1), + ((x >> 30u64) & 1 == 1) == ((y >> 30u64) & 1 == 1), + ((x >> 31u64) & 1 == 1) == ((y >> 31u64) & 1 == 1), + ((x >> 32u64) & 1 == 1) == ((y >> 32u64) & 1 == 1), + ((x >> 33u64) & 1 == 1) == ((y >> 33u64) & 1 == 1), + ((x >> 34u64) & 1 == 1) == ((y >> 34u64) & 1 == 1), + ((x >> 35u64) & 1 == 1) == ((y >> 35u64) & 1 == 1), + ((x >> 36u64) & 1 == 1) == ((y >> 36u64) & 1 == 1), + ((x >> 37u64) & 1 == 1) == ((y >> 37u64) & 1 == 1), + ((x >> 38u64) & 1 == 1) == ((y >> 38u64) & 1 == 1), + ((x >> 39u64) & 1 == 1) == ((y >> 39u64) & 1 == 1), + ((x >> 40u64) & 1 == 1) == ((y >> 40u64) & 1 == 1), + ((x >> 41u64) & 1 == 1) == ((y >> 41u64) & 1 == 1), + ((x >> 42u64) & 1 == 1) == ((y >> 42u64) & 1 == 1), + ((x >> 43u64) & 1 == 1) == ((y >> 43u64) & 1 == 1), + ((x >> 44u64) & 1 == 1) == ((y >> 44u64) & 1 == 1), + ((x >> 45u64) & 1 == 1) == ((y >> 45u64) & 1 == 1), + ((x >> 46u64) & 1 == 1) == ((y >> 46u64) & 1 == 1), + ((x >> 47u64) & 1 == 1) == ((y >> 47u64) & 1 == 1), + ((x >> 48u64) & 1 == 1) == ((y >> 48u64) & 1 == 1), + ((x >> 49u64) & 1 == 1) == ((y >> 49u64) & 1 == 1), + ((x >> 50u64) & 1 == 1) == ((y >> 50u64) & 1 == 1), + ((x >> 51u64) & 1 == 1) == ((y >> 51u64) & 1 == 1), + ((x >> 52u64) & 1 == 1) == ((y >> 52u64) & 1 == 1), + ((x >> 53u64) & 1 == 1) == ((y >> 53u64) & 1 == 1), + ((x >> 54u64) & 1 == 1) == ((y >> 54u64) & 1 == 1), + ((x >> 55u64) & 1 == 1) == ((y >> 55u64) & 1 == 1), + ((x >> 56u64) & 1 == 1) == ((y >> 56u64) & 1 == 1), + ((x >> 57u64) & 1 == 1) == ((y >> 57u64) & 1 == 1), + ((x >> 58u64) & 1 == 1) == ((y >> 58u64) & 1 == 1), + ((x >> 59u64) & 1 == 1) == ((y >> 59u64) & 1 == 1), + ((x >> 60u64) & 1 == 1) == ((y >> 60u64) & 1 == 1), + ((x >> 61u64) & 1 == 1) == ((y >> 61u64) & 1 == 1), + ((x >> 62u64) & 1 == 1) == ((y >> 62u64) & 1 == 1), + ((x >> 63u64) & 1 == 1) == ((y >> 63u64) & 1 == 1), + ; +} + +/// The product is GF(2)-linear in its right factor. +pub proof fn lemma_clmul_upto_xor_right(a: u64, b1: u128, b2: u128, n: nat) + ensures + clmul_upto(a, b1 ^ b2, n) == clmul_upto(a, b1, n) ^ clmul_upto(a, b2, n), + decreases n, +{ + if n > 0 { + let i = (n - 1) as nat; + lemma_clmul_upto_xor_right(a, b1, b2, i); + let (x1, x2) = (clmul_upto(a, b1, i), clmul_upto(a, b2, i)); + let s = i as u128; + if bit(a, i) { + assert((x1 ^ x2) ^ ((b1 ^ b2) << s) == (x1 ^ (b1 << s)) ^ (x2 ^ (b2 << s))) by (bit_vector); + assert(term(a, b1 ^ b2, i) == (b1 ^ b2) << s); + assert(clmul_upto(a, b1 ^ b2, n) == clmul_upto(a, b1 ^ b2, i) ^ term(a, b1 ^ b2, i)); + } else { + assert((x1 ^ x2) ^ 0 == (x1 ^ 0) ^ (x2 ^ 0)) by (bit_vector); + } + } else { + assert(0u128 ^ 0u128 == 0u128) by (bit_vector); + } +} + +pub proof fn lemma_clmul_xor_right(a: u64, b1: u128, b2: u128) + ensures + clmul(a, b1 ^ b2) == clmul(a, b1) ^ clmul(a, b2), +{ + lemma_clmul_upto_xor_right(a, b1, b2, 64); +} + +/// The product is GF(2)-linear in its left factor. +pub proof fn lemma_clmul_upto_xor_left(a1: u64, a2: u64, b: u128, n: nat) + ensures + clmul_upto(a1 ^ a2, b, n) == clmul_upto(a1, b, n) ^ clmul_upto(a2, b, n), + decreases n, +{ + if n > 0 { + let i = (n - 1) as nat; + lemma_clmul_upto_xor_left(a1, a2, b, i); + let (x1, x2) = (clmul_upto(a1, b, i), clmul_upto(a2, b, i)); + let (t1, t2, t) = (term(a1, b, i), term(a2, b, i), term(a1 ^ a2, b, i)); + let s = i as u64; + assert(((a1 ^ a2) >> s) & 1 == 1 <==> (((a1 >> s) & 1 == 1) != ((a2 >> s) & 1 == 1))) by (bit_vector); + assert(t == t1 ^ t2) by { + let v = b << (i as u128); + assert(v ^ 0 == v && 0 ^ v == v && v ^ v == 0 && 0u128 ^ 0u128 == 0) by (bit_vector); + } + assert((x1 ^ x2) ^ (t1 ^ t2) == (x1 ^ t1) ^ (x2 ^ t2)) by (bit_vector); + } else { + assert(0u128 ^ 0u128 == 0u128) by (bit_vector); + } +} + +pub proof fn lemma_clmul_xor_left(a1: u64, a2: u64, b: u128) + ensures + clmul(a1 ^ a2, b) == clmul(a1, b) ^ clmul(a2, b), +{ + lemma_clmul_upto_xor_left(a1, a2, b, 64); +} + +/// Multiplying by `x` (a left shift) commutes with the product: `a * (b x) = (a * b) x`, modulo `x^128`. +pub proof fn lemma_clmul_upto_shl1(a: u64, b: u128, n: nat) + requires + n <= 64, + ensures + clmul_upto(a, b << 1u128, n) == clmul_upto(a, b, n) << 1u128, + decreases n, +{ + if n > 0 { + let i = (n - 1) as nat; + lemma_clmul_upto_shl1(a, b, i); + let x = clmul_upto(a, b, i); + let s = i as u128; + if bit(a, i) { + assert(s < 64 ==> (x << 1u128) ^ ((b << 1u128) << s) == (x ^ (b << s)) << 1u128) by (bit_vector); + } else { + assert((x << 1u128) ^ 0 == (x ^ 0) << 1u128) by (bit_vector); + } + } else { + assert(0u128 << 1u128 == 0u128) by (bit_vector); + } +} + +pub proof fn lemma_clmul_shl1(a: u64, b: u128) + ensures + clmul(a, b << 1u128) == clmul(a, b) << 1u128, +{ + lemma_clmul_upto_shl1(a, b, 64); +} + +/// The product by a monomial is a shift: `a * x^j = a << j`, for `j <= 64`. +pub proof fn lemma_clmul_upto_monomial(a: u64, j: u128, n: nat) + requires + j <= 64, + n <= 64, + ensures + clmul_upto(a, 1u128 << j, n) == (low(a, n) as u128) << j, + decreases n, +{ + if n == 0 { + assert((a & ((1u64 << 0u64) - 1) as u64) as u128 == 0) by (bit_vector); + assert(0u128 << j == 0) by (bit_vector); + } else { + let i = (n - 1) as nat; + lemma_clmul_upto_monomial(a, j, i); + let s = i as u64; + let lo = low(a, i); + assert(lo == a & ((1u64 << s) - 1) as u64); + if n < 64 { + assert(low(a, n) == a & ((1u64 << (s + 1)) - 1) as u64); + assert(s < 63 && j <= 64 ==> { + let t = if (a >> s) & 1 == 1 { (1u128 << j) << (s as u128) } else { 0 }; + ((a & ((1u64 << s) - 1) as u64) as u128) << j ^ t == ((a & ((1u64 << (s + 1)) - 1) as u64) as u128) << j + }) by (bit_vector); + } else { + assert(low(a, n) == a); + assert(s == 63 && j <= 64 ==> { + let t = if (a >> s) & 1 == 1 { (1u128 << j) << (s as u128) } else { 0 }; + ((a & ((1u64 << s) - 1) as u64) as u128) << j ^ t == (a as u128) << j + }) by (bit_vector); + } + } +} + +pub proof fn lemma_clmul_monomial(a: u64, j: u128) + requires + j <= 64, + ensures + clmul(a, 1u128 << j) == (a as u128) << j, +{ + lemma_clmul_upto_monomial(a, j, 64); +} + +/// Splitting the left factor into its low `n` bits and its bit `n`. +pub proof fn lemma_low_step(a: u64, n: nat) + requires + n < 64, + ensures + low(a, n + 1) == low(a, n) ^ (if bit(a, n) { 1u64 << (n as u64) } else { 0 }), +{ + let s = n as u64; + if n + 1 < 64 { + assert(s < 63 ==> a & ((1u64 << (s + 1)) - 1) as u64 == (a & ((1u64 << s) - 1) as u64) ^ (if (a >> s) + & 1 == 1 { + 1u64 << s + } else { + 0 + })) by (bit_vector); + } else { + assert(s == 63 ==> a == (a & ((1u64 << s) - 1) as u64) ^ (if (a >> s) & 1 == 1 { + 1u64 << s + } else { + 0 + })) by (bit_vector); + } +} + +/// The product is commutative: `a * b = b * a` for polynomials of degree below 64. +pub proof fn lemma_clmul_upto_comm(a: u64, b: u64, n: nat) + requires + n <= 64, + ensures + clmul(a, low(b, n) as u128) == clmul_upto(b, a as u128, n), + decreases n, +{ + if n == 0 { + assert(low(b, 0) == 0) by { + assert(b & ((1u64 << 0u64) - 1) as u64 == 0) by (bit_vector); + } + lemma_clmul_upto_xor_right(a, 0, 0, 64); + assert(0u128 ^ 0u128 == 0u128) by (bit_vector); + let z = clmul(a, 0); + assert(z ^ z == 0) by (bit_vector); + } else { + let i = (n - 1) as nat; + lemma_clmul_upto_comm(a, b, i); + lemma_low_step(b, i); + let e: u64 = if bit(b, i) { 1u64 << (i as u64) } else { 0 }; + let lo = low(b, i); + assert((lo ^ e) as u128 == (lo as u128) ^ (e as u128)) by (bit_vector); + lemma_clmul_xor_right(a, lo as u128, e as u128); + if bit(b, i) { + assert(((1u64 << (i as u64)) as u128) == 1u128 << (i as u128)) by { + let s = i as u64; + assert(s < 64 ==> ((1u64 << s) as u128) == 1u128 << (s as u128)) by (bit_vector); + } + lemma_clmul_monomial(a, i as u128); + } else { + lemma_clmul_upto_xor_right(a, 0, 0, 64); + let z = clmul(a, 0); + assert(0u128 ^ 0u128 == 0u128) by (bit_vector); + assert(z ^ z == 0) by (bit_vector); + } + } +} + +pub proof fn lemma_clmul_comm(a: u64, b: u64) + ensures + clmul(a, b as u128) == clmul(b, a as u128), +{ + lemma_clmul_upto_comm(a, b, 64); +} + +/// The product of two polynomials of degree below 64 has degree below 127. +pub proof fn lemma_clmul_upto_bound(a: u64, b: u128, n: nat) + requires + b >> 64u128 == 0, + n <= 64, + ensures + clmul_upto(a, b, n) >> ((63 + n) as u128) == 0, + decreases n, +{ + if n == 0 { + assert(0u128 >> 63u128 == 0) by (bit_vector); + } else { + let i = (n - 1) as nat; + lemma_clmul_upto_bound(a, b, i); + let x = clmul_upto(a, b, i); + let s = i as u128; + assert(s < 64 && b >> 64u128 == 0 && x >> (63 + s) == 0 ==> (x ^ (b << s)) >> (64 + s) == 0 && (x ^ 0) + >> (64 + s) == 0) by (bit_vector); + } +} + +pub proof fn lemma_clmul_bound(a: u64, b: u64) + ensures + clmul(a, b as u128) >> 127u128 == 0, +{ + assert((b as u128) >> 64u128 == 0) by (bit_vector); + lemma_clmul_upto_bound(a, b as u128, 64); +} + +/// The product by zero. +pub proof fn lemma_clmul_zero(a: u64) + ensures + clmul(a, 0) == 0, +{ + lemma_clmul_xor_right(a, 0, 0); + let z = clmul(a, 0); + assert(0u128 ^ 0u128 == 0u128) by (bit_vector); + assert(z ^ z == 0) by (bit_vector); +} + +} // verus! diff --git a/verification/verus/src/fiat_shamir.rs b/verification/verus/src/fiat_shamir.rs new file mode 100644 index 0000000000..a37ff26418 --- /dev/null +++ b/verification/verus/src/fiat_shamir.rs @@ -0,0 +1,203 @@ +//! The block of a Fiat-Shamir transcript step. +//! +//! The executable functions are `step_block` of `crates/fiat_shamir/src/lib.rs` and the message the circuit's +//! `Builder::step` (`crates/leanvm_core/src/rec/circuit/builder/hash.rs`) hashes for the same step; +//! `tests/equivalence/fiat_shamir.rs` checks the first against production. Where Verus rejects the production form +//! (slice patterns, `Option::map_or` with a closure, the panic on a long slice), the copy spells it out; each such +//! spot says what it replaces. +//! +//! Specification: [`block_word`] is word `i` of the block of a step absorbing `scalars` under `tag`: the last +//! scalar in words 4 to 6, the one before it (if any) in words 0 to 2, their count in word 3, the tag in word 7, and +//! zero in the words no scalar fills. On the steps production takes (at most [`MAX_PENDING`] scalars) the block +//! determines the scalars and the tag ([`lemma_step_block_decodes`]), so two steps with the same block absorb the +//! same scalars under the same tag ([`lemma_step_block_injective`]). +use crate::gf2_64::F64; +use crate::gf2_64x3::F192; +use vstd::prelude::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// The scalar in words 0 to 2: the one before the last, or zero. +pub open spec fn first_scalar(scalars: Seq) -> F192 { + if scalars.len() == 2 { + scalars[0] + } else { + F192::ZERO + } +} + +/// The scalar in words 4 to 6: the last, or zero. +pub open spec fn last_scalar(scalars: Seq) -> F192 { + if scalars.len() >= 1 { + scalars[scalars.len() - 1] + } else { + F192::ZERO + } +} + +/// Word `i` of the block of a step absorbing `scalars` under `tag`. +pub open spec fn block_word(scalars: Seq, tag: u64, i: int) -> u64 { + let (a, b) = (first_scalar(scalars), last_scalar(scalars)); + if i == 0 { + a.c0 + } else if i == 1 { + a.c1 + } else if i == 2 { + a.c2 + } else if i == 3 { + scalars.len() as u64 + } else if i == 4 { + b.c0 + } else if i == 5 { + b.c1 + } else if i == 6 { + b.c2 + } else { + tag + } +} + +/// The scalars a block names: its word 3 counts them, the last is in words 4 to 6, the one before it in words 0 to 2. +pub open spec fn decode_scalars(w: spec_fn(int) -> u64) -> Seq { + let a = F192 { c0: w(0), c1: w(1), c2: w(2) }; + let b = F192 { c0: w(4), c1: w(5), c2: w(6) }; + if w(3) == 0 { + seq![] + } else if w(3) == 1 { + seq![b] + } else { + seq![a, b] + } +} + +// --------------------------------------------------------------------------------------------- +// Theorems +// --------------------------------------------------------------------------------------------- +/// A step's block names its scalars and its tag: decoding the block returns them. +pub proof fn lemma_step_block_decodes(scalars: Seq, tag: u64) + requires + scalars.len() <= MAX_PENDING, + ensures + decode_scalars(|i: int| block_word(scalars, tag, i)) == scalars, + block_word(scalars, tag, 7) == tag, +{ + let d = decode_scalars(|i: int| block_word(scalars, tag, i)); + if scalars.len() == 0 { + assert(d =~= scalars); + } else if scalars.len() == 1 { + assert(d =~= scalars); + } else { + assert(d =~= scalars); + } +} + +/// Injectivity: two steps of at most [`MAX_PENDING`] scalars with the same block absorb the same scalars (so the +/// same count) under the same tag. +pub proof fn lemma_step_block_injective(s1: Seq, t1: u64, s2: Seq, t2: u64) + requires + s1.len() <= MAX_PENDING, + s2.len() <= MAX_PENDING, + forall|i: int| 0 <= i < 8 ==> #[trigger] block_word(s1, t1, i) == block_word(s2, t2, i), + ensures + s1 == s2, + t1 == t2, +{ + lemma_step_block_decodes(s1, t1); + lemma_step_block_decodes(s2, t2); + let (w1, w2) = (|i: int| block_word(s1, t1, i), |i: int| block_word(s2, t2, i)); + assert(w1(0) == w2(0) && w1(1) == w2(1) && w1(2) == w2(2) && w1(3) == w2(3)); + assert(w1(4) == w2(4) && w1(5) == w2(5) && w1(6) == w2(6) && w1(7) == w2(7)); + assert(decode_scalars(w1) == decode_scalars(w2)); +} + +/// The domain-separation tags are pairwise distinct, so the blocks of two roles never coincide. +pub proof fn lemma_tags_distinct() + ensures + DS_OBSERVE.0 != DS_SQUEEZE.0, + DS_OBSERVE.0 != DS_POW_BASE.0, + DS_OBSERVE.0 != DS_POW_NONCE.0, + DS_SQUEEZE.0 != DS_POW_BASE.0, + DS_SQUEEZE.0 != DS_POW_NONCE.0, + DS_POW_BASE.0 != DS_POW_NONCE.0, +{ +} + +// --------------------------------------------------------------------------------------------- +// Executable code +// --------------------------------------------------------------------------------------------- +/// The tag of an absorbed scalar. +pub const DS_OBSERVE: F64 = F64(1); + +/// The tag of a challenge. +pub const DS_SQUEEZE: F64 = F64(2); + +/// The tag of the proof-of-work base. +pub const DS_POW_BASE: F64 = F64(3); + +/// The tag of a grinding nonce. +pub const DS_POW_NONCE: F64 = F64(4); + +/// The most scalars one transcript step absorbs. +pub const MAX_PENDING: usize = 2; + +/// The 64-byte block of a transcript step absorbing `scalars` (at most [`MAX_PENDING`]) under `tag`. +/// +/// Production matches slice patterns and panics on a longer slice; the copy indexes, and the panic is a `requires`. +pub fn step_block(scalars: &[F192], tag: F64) -> (r: [F64; 8]) + requires + scalars.len() <= MAX_PENDING, + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] r[i].0 == block_word(scalars@, tag.0, i), +{ + let (first, last) = if scalars.len() == 2 { + (scalars[0], scalars[1]) + } else if scalars.len() == 1 { + (F192::ZERO, scalars[0]) + } else { + (F192::ZERO, F192::ZERO) + }; + let count = F64(scalars.len() as u64); + [ + F64(first.c0), + F64(first.c1), + F64(first.c2), + count, + F64(last.c0), + F64(last.c1), + F64(last.c2), + tag, + ] +} + +/// The message `Builder::step` hashes, over the values of its wires: `scalars` are the values of the absorbed +/// wires and `F192::ZERO` the value of the zero wire. +/// +/// Production asserts the length, matches slice patterns into an `Option` and reads it with `map_or`; the copy +/// indexes and matches, and the assert is a `requires`. The wire reads `self.e(..)` are the values themselves. +pub fn builder_step_message(scalars: &[F192], tag: u64) -> (m: [u64; 8]) + requires + scalars.len() <= 2, + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m[i] == block_word(scalars@, tag, i), +{ + let count = scalars.len() as u64; + let zero = F192::ZERO; + let (first, last) = if scalars.len() == 2 { + (Some(scalars[0]), scalars[1]) + } else if scalars.len() == 1 { + (None, scalars[0]) + } else { + (None, zero) + }; + let a = match first { + Some(a) => a, + None => F192::ZERO, + }; + let b = last; + [a.c0, a.c1, a.c2, count, b.c0, b.c1, b.c2, tag] +} + +} // verus! diff --git a/verification/verus/src/flock_ntt.rs b/verification/verus/src/flock_ntt.rs new file mode 100644 index 0000000000..08b79ac71a --- /dev/null +++ b/verification/verus/src/flock_ntt.rs @@ -0,0 +1,2498 @@ +//! The additive NTT over `GF(2^8)` of flock's zerocheck, and the table that collapses the round-1 extension +//! through it. +//! +//! The executable functions are the portable paths of `crates/flock/src/zerocheck/ntt.rs` and +//! `crates/flock/src/zerocheck/ntt/inv_table.rs`, copied with the same names and bodies where Verus accepts +//! them; `tests/equivalence/flock_ntt.rs` checks the copies against production. +//! +//! Specification, over the field of `crate::gf2_8` (`f8_mul`), with the standard basis `b_i = x^i` +//! ([`basis8`], the byte with bit `i` alone): +//! +//! - [`subspace_poly8`]: the subspace polynomial `s_i`, `s_0(x) = x`, `s_i(x) = s_{i-1}(x) (s_{i-1}(x) + +//! s_{i-1}(b_{i-1}))`, and [`normalized_poly8`], `Ŵ_i(x) = s_i(b_i)^(-1) s_i(x)`. +//! - [`novel8`]: the novel-basis polynomial `Σ_{j < 2^m} a_j X_j(x)`, `X_j = Π_{i < m} Ŵ_i(x)^(bit_i(j))`, by its +//! split on the top bit of `j`; [`lemma_novel8_flat`] proves it equal to the flat sum [`novel8_sum`]. +//! - [`fft_spec`] and [`ifft_spec`]: the recursions of `fft_rec` and `ifft_rec`, butterfly by butterfly. +//! - [`twiddles_of`]: the claim of `compute_twiddles`' documentation, entry `2^d - 1 + j` is the twiddle of +//! block `j` at depth `d`, `Ŵ_(k-1-d)` of the block's first point. +//! +//! Main results: `compute_twiddles`' postcondition; [`lemma_fft_evaluates`] (the forward transform of an +//! NTT built with offset `β` maps the coefficients to the evaluations at `β + v`, `v < 2^k`); +//! [`lemma_ifft_after_fft`] and [`lemma_fft_after_ifft`] (the two transforms undo each other, so +//! [`lemma_ifft_interpolates`]); [`lemma_lde_shift`] (the columns of `M = forward_Λ ∘ inverse_S` satisfy +//! `M[i][j] = M[i ⊕ j][0]`); `InvNttTableByteSingleGf8::new`'s postcondition (row `w` of the table is the XOR of +//! the columns `t < 8` of `M` over the set bits `t` of `w`); and [`lemma_table_applies_lde`] (`apply_scalar` +//! multiplies `M` by the row's bits). +//! +//! The SIMD arms of `apply` are not copied. +use crate::gf2_8::*; +use vstd::arithmetic::div_mod::*; +use vstd::arithmetic::mul::*; +use vstd::arithmetic::power2::*; +use vstd::bits::*; +use vstd::prelude::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// The basis element `b_i = x^i`: the byte with bit `i` alone (`i < 8`). +pub open spec fn basis8(i: nat) -> u8 { + 1u8 << (i as u8) +} + +/// The subspace polynomial `s_i` of the standard basis, evaluated at `x`: +/// `s_0(x) = x` and `s_i(x) = s_{i-1}(x) · (s_{i-1}(x) + s_{i-1}(b_{i-1}))`. +pub open spec fn subspace_poly8(i: nat, x: u8) -> u8 + decreases i, +{ + if i == 0 { + x + } else { + let p = subspace_poly8((i - 1) as nat, x); + f8_mul(p, p ^ subspace_poly8((i - 1) as nat, basis8((i - 1) as nat))) + } +} + +/// The inverse in `GF(2^8)`, `y^254` (zero for zero), as `F8::inv` computes it. +pub open spec fn f8_inv(y: u8) -> u8 { + f8_pow(y, 254) +} + +/// The normalized subspace polynomial `Ŵ_i(x) = s_i(b_i)^(-1) · s_i(x)`. +pub open spec fn normalized_poly8(i: nat, x: u8) -> u8 { + f8_mul(f8_inv(subspace_poly8(i, basis8(i))), subspace_poly8(i, x)) +} + +/// The novel-basis polynomial with the `2^m` coefficients `a`, evaluated at `x`: +/// `Σ_{j < 2^m} a_j X_j(x)` with `X_j = Π_{i < m} Ŵ_i(x)^(bit_i(j))`, split on the top bit of `j`: the +/// polynomial of the low half of the coefficients, plus `Ŵ_(m-1)(x)` times the polynomial of the high half. +pub open spec fn novel8(m: nat, a: Seq, x: u8) -> u8 + decreases m, +{ + if m == 0 { + a[0].0 + } else { + let h = pow2((m - 1) as nat) as int; + novel8((m - 1) as nat, a.subrange(0, h), x) ^ f8_mul( + normalized_poly8((m - 1) as nat, x), + novel8((m - 1) as nat, a.subrange(h, 2 * h), x), + ) + } +} + +/// `a + t b`, word by word. +pub open spec fn lincomb(a: Seq, t: u8, b: Seq) -> Seq { + Seq::new(a.len(), |j: int| F8(a[j].0 ^ f8_mul(t, b[j].0))) +} + +/// `fft_butterfly` on the halves of `v`: `(u, w) -> (u + λ w, w + (u + λ w))`. +pub open spec fn fft_butterfly_spec(v: Seq, lambda: u8) -> Seq { + let h = v.len() / 2; + Seq::new( + v.len(), + |p: int| + if p < h { + F8(v[p].0 ^ f8_mul(lambda, v[p + h].0)) + } else if p < 2 * h { + F8(v[p].0 ^ (v[p - h].0 ^ f8_mul(lambda, v[p].0))) + } else { + v[p] + }, + ) +} + +/// `ifft_butterfly` on the halves of `v`: `(u, w) -> (u + λ (w + u), w + u)`. +pub open spec fn ifft_butterfly_spec(v: Seq, lambda: u8) -> Seq { + let h = v.len() / 2; + Seq::new( + v.len(), + |p: int| + if p < h { + F8(v[p].0 ^ f8_mul(lambda, v[p + h].0 ^ v[p].0)) + } else if p < 2 * h { + F8(v[p].0 ^ v[p - h].0) + } else { + v[p] + }, + ) +} + +/// The recursion of `fft_rec(v, tw, idx)`: the butterfly with twiddle `tw[idx - 1]`, then the low half at +/// `2 idx` and the high half at `2 idx + 1`. +pub open spec fn fft_spec(v: Seq, tw: Seq, idx: int) -> Seq + decreases v.len(), +{ + if v.len() <= 1 { + v + } else { + let w = fft_butterfly_spec(v, tw[idx - 1].0); + let h = v.len() / 2; + fft_spec(w.subrange(0, h as int), tw, 2 * idx) + fft_spec(w.subrange(h as int, v.len() as int), tw, 2 * idx + 1) + } +} + +/// The recursion of `ifft_rec(v, tw, idx)`: the low half at `2 idx` and the high half at `2 idx + 1`, then +/// the inverse butterfly with twiddle `tw[idx - 1]`. +pub open spec fn ifft_spec(v: Seq, tw: Seq, idx: int) -> Seq + decreases v.len(), +{ + if v.len() <= 1 { + v + } else { + let h = v.len() / 2; + ifft_butterfly_spec( + ifft_spec(v.subrange(0, h as int), tw, 2 * idx) + ifft_spec(v.subrange(h as int, v.len() as int), tw, 2 * idx + 1), + tw[idx - 1].0, + ) + } +} + +/// The first point of block `blk` of `2^m` points: `blk 2^m`, as a byte. +pub open spec fn block_offset(blk: nat, m: nat) -> u8 { + (blk * pow2(m)) as u8 +} + +/// The twiddle table of a `2^k`-point NTT with offset `β`: entry `2^d - 1 + j`, for depth `d < k` and block +/// `j < 2^d`, is `Ŵ_(k-1-d)(β + j 2^(k-d))`, `Ŵ` of the first point of the block. +pub open spec fn twiddles_of(tw: Seq, k: nat, beta: u8) -> bool { + &&& tw.len() + 1 == pow2(k) + &&& forall|d: nat, j: nat| + d < k && j < pow2(d) ==> (#[trigger] tw[pow2(d) - 1 + j]).0 == normalized_poly8( + (k - 1 - d) as nat, + beta ^ block_offset(j, (k - d) as nat), + ) +} + +/// The twiddles of the subtree of `fft_rec` at `idx`, `2^m` points from `y` on: `tw[idx - 1] = Ŵ_(m-1)(y)`, +/// and the same for the low half from `y` and the high half from `y + b_(m-1)`. +pub open spec fn node_ok(tw: Seq, idx: int, m: nat, y: u8) -> bool + decreases m, +{ + m == 0 || (tw[idx - 1].0 == normalized_poly8((m - 1) as nat, y) && node_ok(tw, 2 * idx, (m - 1) as nat, y) + && node_ok(tw, 2 * idx + 1, (m - 1) as nat, y ^ basis8((m - 1) as nat))) +} + +// --------------------------------------------------------------------------------------------- +// Field facts +// --------------------------------------------------------------------------------------------- +proof fn lemma_xor8(a: u8, b: u8, c: u8) + ensures + a ^ a == 0, + a ^ 0 == a, + 0 ^ a == a, + a ^ b == b ^ a, + (a ^ b) ^ c == a ^ (b ^ c), + (a ^ b) ^ b == a, + a ^ (a ^ b) == b, +{ + assert(a ^ a == 0 && a ^ 0 == a && 0 ^ a == a && a ^ b == b ^ a && (a ^ b) ^ c == a ^ (b ^ c) && (a ^ b) ^ b == a + && a ^ (a ^ b) == b) by (bit_vector); +} + +/// `GF(2^8)` has no zero divisors. +pub proof fn lemma_f8_no_zero_divisors(a: u8, b: u8) + requires + f8_mul(a, b) == 0, + a != 0, + ensures + b == 0, +{ + let ai = f8_inv(a); + lemma_f8_inv_correct(a); + lemma_f8_mul_comm(a, ai); + lemma_f8_mul_assoc(ai, a, b); + lemma_f8_mul_one(b); + lemma_f8_mul_zero(ai); +} + +/// `1^(-1) = 1`. +proof fn lemma_f8_inv_one() + ensures + f8_inv(1) == 1, +{ + lemma_f8_inv_correct(1); + lemma_f8_mul_one(f8_inv(1)); +} + +proof fn lemma_f8_mul_xor_left(a: u8, b: u8, c: u8) + ensures + f8_mul(a ^ b, c) == f8_mul(a, c) ^ f8_mul(b, c), +{ + lemma_f8_mul_comm(a ^ b, c); + lemma_f8_mul_comm(a, c); + lemma_f8_mul_comm(b, c); + lemma_f8_mul_xor_right(c, a, b); +} + +/// `a (b c) = b (a c)`. +proof fn lemma_f8_mul_swap(a: u8, b: u8, c: u8) + ensures + f8_mul(a, f8_mul(b, c)) == f8_mul(b, f8_mul(a, c)), +{ + lemma_f8_mul_assoc(a, b, c); + lemma_f8_mul_assoc(b, a, c); + lemma_f8_mul_comm(a, b); +} + +// --------------------------------------------------------------------------------------------- +// Bits of a byte +// --------------------------------------------------------------------------------------------- +/// `1 << s = 2^s` for a byte. +pub proof fn lemma_shl8(s: nat) + requires + s < 8, + ensures + basis8(s) as nat == pow2(s), + pow2(s) <= 128, +{ + lemma2_to64(); + if s < 7 { + lemma_pow2_strictly_increases(s, 7); + } + lemma_u8_shl_is_mul(1, s as u8); +} + +/// Bit `s` against the comparisons with `2^s` and `2^(s+1)`. +proof fn lemma_bit8(x: u8, s: u8) + requires + s < 8, + ensures + (x >> s) == 1u8 ==> (x ^ (1u8 << s)) == sub(x, 1u8 << s) && (x ^ (1u8 << s)) < (1u8 << s), + s < 7 ==> (x < (1u8 << s) ==> x < (1u8 << add(s, 1))), + s < 7 ==> ((x ^ (1u8 << s)) < (1u8 << s) ==> x < (1u8 << add(s, 1))), + s < 7 ==> (x < (1u8 << add(s, 1)) && !(x < (1u8 << s)) ==> (x ^ (1u8 << s)) < (1u8 << s)), + s == 7 ==> (!(x < (1u8 << s)) ==> (x ^ (1u8 << s)) < (1u8 << s)), + (x >> s) & 1u8 == 0u8 ==> (x ^ (1u8 << s)) == add(x, 1u8 << s), +{ + assert((x >> s) == 1u8 ==> (x ^ (1u8 << s)) == sub(x, 1u8 << s) && (x ^ (1u8 << s)) < (1u8 << s)) by (bit_vector) + requires + s < 8, + ; + assert(s < 7 ==> (x < (1u8 << s) ==> x < (1u8 << add(s, 1)))) by (bit_vector) + requires + s < 8, + ; + assert(s < 7 ==> ((x ^ (1u8 << s)) < (1u8 << s) ==> x < (1u8 << add(s, 1)))) by (bit_vector) + requires + s < 8, + ; + assert(s < 7 ==> (x < (1u8 << add(s, 1)) && !(x < (1u8 << s)) ==> (x ^ (1u8 << s)) < (1u8 << s))) by (bit_vector) + requires + s < 8, + ; + assert(s == 7 ==> (!(x < (1u8 << s)) ==> (x ^ (1u8 << s)) < (1u8 << s))) by (bit_vector); + assert((x >> s) & 1u8 == 0u8 ==> (x ^ (1u8 << s)) == add(x, 1u8 << s)) by (bit_vector) + requires + s < 8, + ; +} + +/// An index `u` of the high half of `2^(s+1)` points: `u - 2^s = u ⊕ 2^s`, below `2^s`. +proof fn lemma_high_half(u: int, s: nat) + requires + s < 8, + pow2(s) <= u < 2 * pow2(s), + ensures + (u - pow2(s)) as u8 == (u as u8) ^ basis8(s), + ((u as u8) ^ basis8(s)) < basis8(s), +{ + lemma_shl8(s); + let x = u as u8; + assert(x as int == u); + lemma_u8_shr_is_div(x, s as u8); + lemma_fundamental_div_mod_converse(u, pow2(s) as int, 1, u - pow2(s)); + lemma_bit8(x, s as u8); +} + +// --------------------------------------------------------------------------------------------- +// Subspace polynomials +// --------------------------------------------------------------------------------------------- +/// `s_i` is F_2-linear: `s_i(x + y) = s_i(x) + s_i(y)`. +pub proof fn lemma_subspace_poly8_additive(i: nat, x: u8, y: u8) + ensures + subspace_poly8(i, x ^ y) == subspace_poly8(i, x) ^ subspace_poly8(i, y), + decreases i, +{ + if i > 0 { + let j = (i - 1) as nat; + lemma_subspace_poly8_additive(j, x, y); + let a = subspace_poly8(j, x); + let b = subspace_poly8(j, y); + let c = subspace_poly8(j, basis8(j)); + // (a + b)(a + b + c) = a(a + c) + a b + b(b + c) + b a + assert((a ^ b) ^ c == (a ^ c) ^ b && (a ^ b) ^ c == (b ^ c) ^ a) by (bit_vector); + lemma_f8_mul_xor_left(a, b, (a ^ b) ^ c); + lemma_f8_mul_xor_right(a, a ^ c, b); + lemma_f8_mul_xor_right(b, b ^ c, a); + lemma_f8_mul_comm(a, b); + let (p, q, r) = (f8_mul(a, a ^ c), f8_mul(b, b ^ c), f8_mul(a, b)); + assert((p ^ r) ^ (q ^ r) == p ^ q) by (bit_vector); + } +} + +/// The normalized `Ŵ_i` is F_2-linear too. +pub proof fn lemma_normalized_poly8_additive(i: nat, x: u8, y: u8) + ensures + normalized_poly8(i, x ^ y) == normalized_poly8(i, x) ^ normalized_poly8(i, y), +{ + lemma_subspace_poly8_additive(i, x, y); + lemma_f8_mul_xor_right(f8_inv(subspace_poly8(i, basis8(i))), subspace_poly8(i, x), subspace_poly8(i, y)); +} + +/// `s_i` vanishes on `{0, .., 2^i - 1}`, the span of `b_0 .. b_(i-1)`. +pub proof fn lemma_subspace_poly8_vanishes(i: nat, x: u8) + requires + i < 8, + x < basis8(i), + ensures + subspace_poly8(i, x) == 0, + decreases i, +{ + if i == 0 { + assert(x < (1u8 << 0u8) ==> x == 0) by (bit_vector); + } else { + let j = (i - 1) as nat; + let b = basis8(j); + let p = subspace_poly8(j, x); + let c = subspace_poly8(j, b); + lemma_bit8(x, j as u8); + assert(add(j as u8, 1) == i as u8); + if x < b { + lemma_subspace_poly8_vanishes(j, x); + lemma_f8_mul_zero(p ^ c); + } else { + lemma_subspace_poly8_vanishes(j, x ^ b); + lemma_subspace_poly8_additive(j, x ^ b, b); + lemma_xor8(x, b, 0); + lemma_xor8(c, 0, 0); + lemma_f8_mul_zero(p); + } + } +} + +/// The roots of `s_i` lie below `2^i`: `s_i(x) = 0` only on `{0, .., 2^i - 1}`. +pub proof fn lemma_subspace_poly8_roots(i: nat, x: u8) + requires + i < 8, + subspace_poly8(i, x) == 0, + ensures + x < basis8(i), + decreases i, +{ + if i == 0 { + assert(0u8 < (1u8 << 0u8)) by (bit_vector); + } else { + let j = (i - 1) as nat; + let b = basis8(j); + let p = subspace_poly8(j, x); + let c = subspace_poly8(j, b); + lemma_bit8(x, j as u8); + assert(add(j as u8, 1) == i as u8); + if p == 0 { + lemma_subspace_poly8_roots(j, x); + } else { + lemma_f8_no_zero_divisors(p, p ^ c); + lemma_subspace_poly8_additive(j, x, b); + lemma_subspace_poly8_roots(j, x ^ b); + } + } +} + +/// Every normalizer works: `Ŵ_i(b_i) = 1` for `i < 8`. +pub proof fn lemma_normalized_poly8_at_basis(i: nat) + requires + i < 8, + ensures + normalized_poly8(i, basis8(i)) == 1, +{ + let y = subspace_poly8(i, basis8(i)); + if y == 0 { + lemma_subspace_poly8_roots(i, basis8(i)); + } + lemma_f8_inv_correct(y); + lemma_f8_mul_comm(y, f8_inv(y)); +} + +/// `Ŵ_i` vanishes on `{0, .., 2^i - 1}`. +pub proof fn lemma_normalized_poly8_vanishes(i: nat, x: u8) + requires + i < 8, + x < basis8(i), + ensures + normalized_poly8(i, x) == 0, +{ + lemma_subspace_poly8_vanishes(i, x); + lemma_f8_mul_zero(f8_inv(subspace_poly8(i, basis8(i)))); +} + +/// `Ŵ_0(x) = x`. +proof fn lemma_normalized_poly8_zero(x: u8) + ensures + normalized_poly8(0, x) == x, +{ + assert(basis8(0) == 1) by { + assert((1u8 << 0u8) == 1u8) by (bit_vector); + } + lemma_f8_inv_one(); + lemma_f8_mul_one(x); +} + +// --------------------------------------------------------------------------------------------- +// The novel-basis polynomial is linear in its coefficients +// --------------------------------------------------------------------------------------------- +proof fn lemma_lincomb_halves(a: Seq, t: u8, b: Seq, h: int) + requires + a.len() == b.len() == 2 * h, + h >= 0, + ensures + lincomb(a, t, b).subrange(0, h) == lincomb(a.subrange(0, h), t, b.subrange(0, h)), + lincomb(a, t, b).subrange(h, 2 * h) == lincomb(a.subrange(h, 2 * h), t, b.subrange(h, 2 * h)), +{ + assert(lincomb(a, t, b).subrange(0, h) =~= lincomb(a.subrange(0, h), t, b.subrange(0, h))); + assert(lincomb(a, t, b).subrange(h, 2 * h) =~= lincomb(a.subrange(h, 2 * h), t, b.subrange(h, 2 * h))); +} + +/// `P_(a + t b) = P_a + t P_b`. +pub proof fn lemma_novel8_lincomb(m: nat, a: Seq, t: u8, b: Seq, x: u8) + requires + a.len() == pow2(m), + b.len() == pow2(m), + ensures + novel8(m, lincomb(a, t, b), x) == novel8(m, a, x) ^ f8_mul(t, novel8(m, b, x)), + decreases m, +{ + if m > 0 { + let j = (m - 1) as nat; + let h = pow2(j) as int; + lemma_pow2_unfold(m); + lemma_lincomb_halves(a, t, b, h); + let (al, ah, bl, bh) = (a.subrange(0, h), a.subrange(h, 2 * h), b.subrange(0, h), b.subrange(h, 2 * h)); + lemma_novel8_lincomb(j, al, t, bl, x); + lemma_novel8_lincomb(j, ah, t, bh, x); + let w = normalized_poly8(j, x); + let (pal, pah, pbl, pbh) = (novel8(j, al, x), novel8(j, ah, x), novel8(j, bl, x), novel8(j, bh, x)); + // (pal + t pbl) + w (pah + t pbh) = (pal + w pah) + t (pbl + w pbh) + lemma_f8_mul_xor_right(w, pah, f8_mul(t, pbh)); + lemma_f8_mul_swap(w, t, pbh); + lemma_f8_mul_xor_right(t, pbl, f8_mul(w, pbh)); + let (p1, p2, p3) = (f8_mul(t, pbl), f8_mul(w, pah), f8_mul(t, f8_mul(w, pbh))); + assert((pal ^ p1) ^ (p2 ^ p3) == (pal ^ p2) ^ (p1 ^ p3)) by (bit_vector); + } else { + lemma2_to64(); + } +} + +// --------------------------------------------------------------------------------------------- +// The novel-basis polynomial as a flat sum +// --------------------------------------------------------------------------------------------- +/// The novel basis polynomial `X_j(x) = Π_{i < m} Ŵ_i(x)^(bit_i(j))`, bit `i` of `j` being `(j / 2^i) mod 2`. +pub open spec fn novel_basis8(m: nat, j: int, x: u8) -> u8 + decreases m, +{ + if m == 0 { + 1 + } else { + let rest = novel_basis8((m - 1) as nat, j, x); + if (j / pow2((m - 1) as nat) as int) % 2 == 1 { + f8_mul(rest, normalized_poly8((m - 1) as nat, x)) + } else { + rest + } + } +} + +/// The novel-basis polynomial with coefficients `a`, as the flat sum `Σ_{j < 2^m} a_j X_j(x)`. +pub open spec fn novel8_sum(m: nat, a: Seq, x: u8) -> u8 { + xor_sum8(|j: int| f8_mul(a[j].0, novel_basis8(m, j, x)), pow2(m)) +} + +/// `X_j` reads only the low `n` bits of `j`. +proof fn lemma_novel_basis8_periodic(n: nat, j: int, q: int, x: u8) + requires + j >= 0, + q >= 0, + ensures + novel_basis8(n, j + q * pow2(n), x) == novel_basis8(n, j, x), + decreases n, +{ + if n > 0 { + let p = pow2((n - 1) as nat) as int; + lemma_pow2_unfold(n); + lemma_pow2_pos((n - 1) as nat); + assert(j + q * pow2(n) == j + (2 * q) * p) by (nonlinear_arith) + requires + pow2(n) == 2 * p, + ; + lemma_novel_basis8_periodic((n - 1) as nat, j, 2 * q, x); + lemma_mod_multiples_vanish(q, j / p, 2); + assert((j + (2 * q) * p) / p == 2 * q + j / p) by { + lemma_fundamental_div_mod(j, p); + lemma_mod_pos_bound(j, p); + assert(j + (2 * q) * p == p * (2 * q + j / p) + j % p) by (nonlinear_arith) + requires + j == p * (j / p) + j % p, + ; + lemma_fundamental_div_mod_converse(j + (2 * q) * p, p, 2 * q + j / p, j % p); + } + } +} + +/// Multiplication distributes over a sum. +proof fn lemma_xor_sum8_scale(c: u8, f: spec_fn(int) -> u8, n: nat) + ensures + f8_mul(c, xor_sum8(f, n)) == xor_sum8(|j: int| f8_mul(c, f(j)), n), + decreases n, +{ + if n == 0 { + lemma_f8_mul_zero(c); + } else { + lemma_xor_sum8_scale(c, f, (n - 1) as nat); + lemma_f8_mul_xor_right(c, xor_sum8(f, (n - 1) as nat), f(n - 1)); + } +} + +/// The top-bit split is the flat sum: `novel8(m, a, x) = Σ_{j < 2^m} a_j X_j(x)`. +pub proof fn lemma_novel8_flat(m: nat, a: Seq, x: u8) + requires + a.len() == pow2(m), + ensures + novel8(m, a, x) == novel8_sum(m, a, x), + decreases m, +{ + if m == 0 { + lemma2_to64(); + lemma_f8_mul_one(a[0].0); + let y = a[0].0; + assert(0u8 ^ y == y) by (bit_vector); + let f = |j: int| f8_mul(a[j].0, novel_basis8(0, j, x)); + assert(novel_basis8(0, 0, x) == 1); + assert(xor_sum8(f, 1) == xor_sum8(f, 0) ^ f(0)); + assert(novel8_sum(0, a, x) == xor_sum8(f, 1)); + } else { + let n = (m - 1) as nat; + let h = pow2(n) as int; + lemma_pow2_unfold(m); + lemma_pow2_pos(n); + let (al, ah) = (a.subrange(0, h), a.subrange(h, 2 * h)); + lemma_novel8_flat(n, al, x); + lemma_novel8_flat(n, ah, x); + let w = normalized_poly8(n, x); + let f = |j: int| f8_mul(a[j].0, novel_basis8(m, j, x)); + lemma_xor_sum8_split(f, h as nat, h as nat); + // The low half: bit `m - 1` of `j < 2^(m-1)` is zero. + let fl = |j: int| f8_mul(al[j].0, novel_basis8(n, j, x)); + assert forall|j: int| 0 <= j < h implies #[trigger] f(j) == fl(j) by { + assert(j / h == 0) by { + lemma_fundamental_div_mod_converse(j, h, 0, j); + } + } + lemma_xor_sum8_ext(f, fl, h as nat); + // The high half: bit `m - 1` of `2^(m-1) + t` is one, and the lower bits are those of `t`. + let fh = |t: int| f8_mul(ah[t].0, novel_basis8(n, t, x)); + let g = |t: int| f(h + t); + assert forall|t: int| 0 <= t < h implies #[trigger] g(t) == f8_mul(w, fh(t)) by { + lemma_novel_basis8_periodic(n, t, 1, x); + assert((h + t) / h == 1) by { + lemma_fundamental_div_mod_converse(h + t, h, 1, t); + } + assert(t + 1 * pow2(n) == h + t); + let (c, b) = (ah[t].0, novel_basis8(n, t, x)); + assert(a[h + t].0 == c); + // c (b w) = w (c b) + lemma_f8_mul_comm(b, w); + lemma_f8_mul_swap(c, w, b); + } + lemma_xor_sum8_ext(g, |t: int| f8_mul(w, fh(t)), h as nat); + lemma_xor_sum8_scale(w, fh, h as nat); + lemma_xor_sum8_ext(|t: int| f((h as nat) + t), g, h as nat); + assert(novel8_sum(n, al, x) == xor_sum8(fl, h as nat)); + assert(novel8_sum(n, ah, x) == xor_sum8(fh, h as nat)); + assert(novel8_sum(m, a, x) == xor_sum8(f, (h as nat) + (h as nat))); + } +} + +// --------------------------------------------------------------------------------------------- +// The forward transform evaluates, the inverse undoes it +// --------------------------------------------------------------------------------------------- +proof fn lemma_fft_spec_len(v: Seq, tw: Seq, idx: int) + ensures + fft_spec(v, tw, idx).len() == v.len(), + decreases v.len(), +{ + if v.len() > 1 { + let w = fft_butterfly_spec(v, tw[idx - 1].0); + let h = v.len() / 2; + lemma_fft_spec_len(w.subrange(0, h as int), tw, 2 * idx); + lemma_fft_spec_len(w.subrange(h as int, v.len() as int), tw, 2 * idx + 1); + } +} + +proof fn lemma_ifft_spec_len(v: Seq, tw: Seq, idx: int) + ensures + ifft_spec(v, tw, idx).len() == v.len(), + decreases v.len(), +{ + if v.len() > 1 { + let h = v.len() / 2; + lemma_ifft_spec_len(v.subrange(0, h as int), tw, 2 * idx); + lemma_ifft_spec_len(v.subrange(h as int, v.len() as int), tw, 2 * idx + 1); + } +} + +/// The forward transform evaluates the novel-basis polynomial: on `2^m` coefficients, with the twiddles of +/// a subtree from `y` ([`node_ok`]), output word `u` is `P(y + u)`. +pub proof fn lemma_fft_evaluates_from(v: Seq, tw: Seq, idx: int, m: nat, y: u8) + requires + m <= 8, + v.len() == pow2(m), + node_ok(tw, idx, m, y), + ensures + fft_spec(v, tw, idx).len() == v.len(), + forall|u: int| 0 <= u < v.len() ==> (#[trigger] fft_spec(v, tw, idx)[u]).0 == novel8(m, v, y ^ (u as u8)), + decreases m, +{ + lemma_fft_spec_len(v, tw, idx); + if m == 0 { + lemma2_to64(); + assert forall|u: int| 0 <= u < v.len() implies (#[trigger] fft_spec(v, tw, idx)[u]).0 == novel8(m, v, y ^ (u as u8)) by { + assert(u == 0); + assert(y ^ 0u8 == y) by (bit_vector); + } + } else { + let j = (m - 1) as nat; + let h = pow2(j) as int; + let n = v.len() as int; + lemma_pow2_unfold(m); + lemma_pow2_pos(j); + lemma_shl8(j); + let lambda = tw[idx - 1].0; + let w = fft_butterfly_spec(v, lambda); + let (lo, hi) = (w.subrange(0, h), w.subrange(h, n)); + let (vl, vh) = (v.subrange(0, h), v.subrange(h, n)); + let b = basis8(j); + lemma_fft_evaluates_from(lo, tw, 2 * idx, j, y); + lemma_fft_evaluates_from(hi, tw, 2 * idx + 1, j, y ^ b); + // The two halves after the butterfly, as combinations of the coefficient halves. + assert(lo =~= lincomb(vl, lambda, vh)); + assert forall|i: int| 0 <= i < h implies #[trigger] hi[i] == lincomb(vl, lambda ^ 1, vh)[i] by { + let (p, q) = (v[i].0, v[i + h].0); + lemma_f8_mul_xor_left(lambda, 1, q); + lemma_f8_mul_one(q); + let r = f8_mul(lambda, q); + assert(q ^ (p ^ r) == p ^ (r ^ q)) by (bit_vector); + } + assert(hi =~= lincomb(vl, lambda ^ 1, vh)); + lemma_novel8_lincomb(j, vl, lambda, vh, y); + assert forall|u: int| 0 <= u < n implies (#[trigger] fft_spec(v, tw, idx)[u]).0 == novel8(m, v, y ^ (u as u8)) by { + let x = y ^ (u as u8); + lemma_normalized_poly8_additive(j, y, u as u8); + lemma_novel8_lincomb(j, vl, lambda, vh, x); + lemma_novel8_lincomb(j, vl, lambda ^ 1, vh, x); + if u < h { + assert((u as u8) < b); + lemma_normalized_poly8_vanishes(j, u as u8); + lemma_xor8(lambda, 0, 0); + assert(fft_spec(v, tw, idx)[u] == fft_spec(lo, tw, 2 * idx)[u]); + } else { + lemma_high_half(u, j); + let u8l = (u - h) as u8; + lemma_normalized_poly8_additive(j, b, u8l); + lemma_normalized_poly8_at_basis(j); + lemma_normalized_poly8_vanishes(j, u8l); + lemma_xor8(1, 0, 0); + let ux = u as u8; + assert((y ^ b) ^ (ux ^ b) == y ^ ux) by (bit_vector); + assert(b ^ (ux ^ b) == ux) by (bit_vector); + assert(fft_spec(v, tw, idx)[u] == fft_spec(hi, tw, 2 * idx + 1)[u - h]); + } + } + } +} + +/// The inverse butterfly undoes the forward one. +proof fn lemma_butterfly8_inverse(u: u8, w: u8, t: u8) + ensures + ({ + let (u1, w1) = (u ^ f8_mul(t, w), w ^ (u ^ f8_mul(t, w))); + u1 ^ f8_mul(t, w1 ^ u1) == u && w1 ^ u1 == w + }), + ({ + let (u1, w1) = (u ^ f8_mul(t, w ^ u), w ^ u); + u1 ^ f8_mul(t, w1) == u && w1 ^ (u1 ^ f8_mul(t, w1)) == w + }), +{ + let r = f8_mul(t, w); + assert((w ^ (u ^ r)) ^ (u ^ r) == w) by (bit_vector); + assert((u ^ r) ^ r == u) by (bit_vector); + let s = f8_mul(t, w ^ u); + assert((u ^ s) ^ s == u) by (bit_vector); + assert((w ^ u) ^ ((u ^ s) ^ s) == w) by (bit_vector); +} + +/// The inverse transform undoes the forward one, for every twiddle table and every buffer. +pub proof fn lemma_ifft_after_fft(v: Seq, tw: Seq, idx: int) + ensures + ifft_spec(fft_spec(v, tw, idx), tw, idx) == v, + decreases v.len(), +{ + if v.len() > 1 { + let n = v.len() as int; + let h = n / 2; + let lambda = tw[idx - 1].0; + let w = fft_butterfly_spec(v, lambda); + let (lo, hi) = (w.subrange(0, h), w.subrange(h, n)); + lemma_ifft_after_fft(lo, tw, 2 * idx); + lemma_ifft_after_fft(hi, tw, 2 * idx + 1); + lemma_fft_spec_len(lo, tw, 2 * idx); + lemma_fft_spec_len(hi, tw, 2 * idx + 1); + let f = fft_spec(v, tw, idx); + assert(f.subrange(0, h) =~= fft_spec(lo, tw, 2 * idx)); + assert(f.subrange(h, n) =~= fft_spec(hi, tw, 2 * idx + 1)); + assert(fft_spec(lo, tw, 2 * idx) + fft_spec(hi, tw, 2 * idx + 1) == f); + let g = ifft_spec(fft_spec(lo, tw, 2 * idx), tw, 2 * idx) + ifft_spec(fft_spec(hi, tw, 2 * idx + 1), tw, 2 * idx + 1); + assert(g =~= w); + assert forall|p: int| 0 <= p < n implies #[trigger] ifft_butterfly_spec(w, lambda)[p] == v[p] by { + if p < h { + lemma_butterfly8_inverse(v[p].0, v[p + h].0, lambda); + } else if p < 2 * h { + lemma_butterfly8_inverse(v[p - h].0, v[p].0, lambda); + } + } + assert(ifft_butterfly_spec(w, lambda) =~= v); + } +} + +/// The forward transform undoes the inverse one, for every twiddle table and every buffer. +#[verifier::rlimit(30)] +pub proof fn lemma_fft_after_ifft(v: Seq, tw: Seq, idx: int) + ensures + fft_spec(ifft_spec(v, tw, idx), tw, idx) == v, + decreases v.len(), +{ + if v.len() > 1 { + let n = v.len() as int; + let h = n / 2; + let lambda = tw[idx - 1].0; + let (vl, vh) = (v.subrange(0, h), v.subrange(h, n)); + lemma_fft_after_ifft(vl, tw, 2 * idx); + lemma_fft_after_ifft(vh, tw, 2 * idx + 1); + lemma_ifft_spec_len(vl, tw, 2 * idx); + lemma_ifft_spec_len(vh, tw, 2 * idx + 1); + let g = ifft_spec(vl, tw, 2 * idx) + ifft_spec(vh, tw, 2 * idx + 1); + let iv = ifft_spec(v, tw, idx); + assert(iv == ifft_butterfly_spec(g, lambda)); + let w = fft_butterfly_spec(iv, lambda); + assert forall|p: int| 0 <= p < n implies #[trigger] w[p] == g[p] by { + if p < h { + lemma_butterfly8_inverse(g[p].0, g[p + h].0, lambda); + } else if p < 2 * h { + lemma_butterfly8_inverse(g[p - h].0, g[p].0, lambda); + } + } + assert(w =~= g); + assert(w.subrange(0, h) =~= ifft_spec(vl, tw, 2 * idx)); + assert(w.subrange(h, n) =~= ifft_spec(vh, tw, 2 * idx + 1)); + assert(fft_spec(iv, tw, idx) =~= v); + } +} + +/// `2^k <= 256` for `k <= 8`. +proof fn lemma_pow2_le_256(k: nat) + requires + k <= 8, + ensures + pow2(k) <= 256, + pow2(k) >= 1, +{ + lemma2_to64(); + if k < 8 { + lemma_pow2_strictly_increases(k, 8); + } +} + +/// The first point of the high half of a block: `blk 2^m + 2^(m-1) = (blk 2^m) ⊕ b_(m-1)`. +proof fn lemma_block_offset_halves(blk: nat, m: nat) + requires + 1 <= m <= 8, + blk * pow2(m) + pow2(m) <= 256, + ensures + block_offset(2 * blk, (m - 1) as nat) == block_offset(blk, m), + block_offset(2 * blk + 1, (m - 1) as nat) == block_offset(blk, m) ^ basis8((m - 1) as nat), +{ + let j = (m - 1) as nat; + let h = pow2(j); + lemma_pow2_unfold(m); + lemma_pow2_pos(j); + lemma_shl8(j); + let x = blk * pow2(m); + assert(2 * blk * h == x) by (nonlinear_arith) + requires + x == blk * pow2(m), + pow2(m) == 2 * h, + ; + assert((2 * blk + 1) * h == x + h) by (nonlinear_arith) + requires + x == blk * pow2(m), + pow2(m) == 2 * h, + ; + assert(x + h < 256) by (nonlinear_arith) + requires + x + 2 * h <= 256, + h >= 1, + ; + let xb = x as u8; + lemma_u8_shr_is_div(xb, j as u8); + lemma_div_multiples_vanish((2 * blk) as int, h as int); + assert(x as int == h * (2 * blk)) by (nonlinear_arith) + requires + 2 * blk * h == x, + ; + assert(xb >> (j as u8) == 2 * blk); + let q = (2 * blk) as u8; + assert(q == ((2 * blk) % 256) as u8); + assert(((2 * blk) as u8) & 1u8 == 0u8) by { + lemma_mod_multiples_basic(blk as int, 2); + assert((2 * blk) % 2 == 0); + let qq = (2 * blk) as u8; + assert(qq % 2 == 0) by { + lemma_mod_mod((2 * blk) as int, 2, 128); + } + assert(qq % 2 == 0 ==> qq & 1u8 == 0u8) by (bit_vector); + } + lemma_bit8(xb, j as u8); +} + +/// The twiddle table of offset `β` holds the twiddles of every subtree: at depth `d`, block `blk` covers the +/// `2^(k-d)` points from `β + blk 2^(k-d)` on. +proof fn lemma_twiddles_node_ok(tw: Seq, k: nat, beta: u8, d: nat, blk: nat) + requires + twiddles_of(tw, k, beta), + k <= 8, + d <= k, + blk < pow2(d), + ensures + node_ok(tw, (pow2(d) + blk) as int, (k - d) as nat, beta ^ block_offset(blk, (k - d) as nat)), + decreases k - d, +{ + if d < k { + let m = (k - d) as nat; + let j = (m - 1) as nat; + let idx = (pow2(d) + blk) as int; + let y = beta ^ block_offset(blk, m); + assert(tw[pow2(d) - 1 + blk] == tw[idx - 1]); + lemma_pow2_unfold(d + 1); + lemma_twiddles_node_ok(tw, k, beta, d + 1, 2 * blk); + lemma_twiddles_node_ok(tw, k, beta, d + 1, 2 * blk + 1); + assert(2 * idx == pow2(d + 1) + 2 * blk); + assert(2 * idx + 1 == pow2(d + 1) + (2 * blk + 1)); + assert((k - (d + 1)) as nat == j); + // The block lies inside the domain of `2^k <= 256` points. + lemma_pow2_adds(d, m); + lemma_pow2_le_256(k); + assert(blk * pow2(m) + pow2(m) <= 256) by (nonlinear_arith) + requires + blk + 1 <= pow2(d), + pow2(d) * pow2(m) == pow2(k), + pow2(k) <= 256, + ; + lemma_block_offset_halves(blk, m); + let b = basis8(j); + let o = block_offset(blk, m); + assert((beta ^ o) ^ b == beta ^ (o ^ b)) by (bit_vector); + } +} + +/// The forward transform of a `2^k`-point NTT with offset `β` (`fft_rec` from the root) evaluates the +/// novel-basis polynomial on the domain `β + {0, .., 2^k - 1}`: output word `u` is `P(β + u)`, the point of +/// index `u` being `β ⊕ u` (no bit reversal). +pub proof fn lemma_fft_evaluates(tw: Seq, k: nat, beta: u8, v: Seq) + requires + twiddles_of(tw, k, beta), + k <= 8, + v.len() == pow2(k), + ensures + fft_spec(v, tw, 1).len() == v.len(), + forall|u: int| 0 <= u < v.len() ==> (#[trigger] fft_spec(v, tw, 1)[u]).0 == novel8(k, v, beta ^ (u as u8)), +{ + lemma2_to64(); + lemma_twiddles_node_ok(tw, k, beta, 0, 0); + assert(block_offset(0, k) == 0); + assert(beta ^ 0u8 == beta) by (bit_vector); + lemma_fft_evaluates_from(v, tw, 1, k, beta); +} + +/// The inverse transform interpolates: if `e` holds the evaluations of `P` on the domain `β + {0, .., 2^k - 1}` +/// (word `u` is `P(β + u)`), then `ifft_rec` from the root returns the novel-basis coefficients of `P`. +pub proof fn lemma_ifft_interpolates(tw: Seq, k: nat, beta: u8, a: Seq, e: Seq) + requires + twiddles_of(tw, k, beta), + k <= 8, + a.len() == pow2(k), + e.len() == pow2(k), + forall|u: int| 0 <= u < e.len() ==> (#[trigger] e[u]).0 == novel8(k, a, beta ^ (u as u8)), + ensures + ifft_spec(e, tw, 1) == a, +{ + lemma_fft_evaluates(tw, k, beta, a); + assert(fft_spec(a, tw, 1) =~= e); + lemma_ifft_after_fft(a, tw, 1); +} + +// --------------------------------------------------------------------------------------------- +// Translating a polynomial stays in the novel basis +// --------------------------------------------------------------------------------------------- +/// The novel-basis coefficients of `x -> P_a(x + c)`: on the top-bit split `P_a = A + Ŵ_(m-1) B`, +/// `P_a(x + c) = A(x + c) + Ŵ_(m-1)(c) B(x + c) + Ŵ_(m-1)(x) B(x + c)`, with `A` and `B` translated recursively. +pub open spec fn translate(m: nat, a: Seq, c: u8) -> Seq + decreases m, +{ + if m == 0 { + a + } else { + let h = pow2((m - 1) as nat) as int; + let qa = translate((m - 1) as nat, a.subrange(0, h), c); + let qb = translate((m - 1) as nat, a.subrange(h, 2 * h), c); + lincomb(qa, normalized_poly8((m - 1) as nat, c), qb) + qb + } +} + +/// `P_(translate(a, c))(x) = P_a(x + c)` for every `x`. +pub proof fn lemma_translate(m: nat, a: Seq, c: u8, x: u8) + requires + a.len() == pow2(m), + ensures + translate(m, a, c).len() == pow2(m), + novel8(m, translate(m, a, c), x) == novel8(m, a, x ^ c), + decreases m, +{ + if m > 0 { + let j = (m - 1) as nat; + let h = pow2(j) as int; + lemma_pow2_unfold(m); + let (al, ah) = (a.subrange(0, h), a.subrange(h, 2 * h)); + let qa = translate(j, al, c); + let qb = translate(j, ah, c); + lemma_translate(j, al, c, x); + lemma_translate(j, ah, c, x); + let wc = normalized_poly8(j, c); + let wx = normalized_poly8(j, x); + let t = translate(m, a, c); + assert(t.subrange(0, h) =~= lincomb(qa, wc, qb)); + assert(t.subrange(h, 2 * h) =~= qb); + lemma_novel8_lincomb(j, qa, wc, qb, x); + lemma_normalized_poly8_additive(j, x, c); + let nb = novel8(j, ah, x ^ c); + lemma_f8_mul_xor_left(wx, wc, nb); + let (na, p, q) = (novel8(j, al, x ^ c), f8_mul(wc, nb), f8_mul(wx, nb)); + assert((na ^ p) ^ q == na ^ (q ^ p)) by (bit_vector); + } else { + lemma2_to64(); + assert(x ^ c == x ^ c); + } +} + +// --------------------------------------------------------------------------------------------- +// The columns of `forward_Λ ∘ inverse_S` are XOR shifts of one another +// --------------------------------------------------------------------------------------------- +/// The unit vector `e_t` of length `n`. +pub open spec fn unit(n: nat, t: int) -> Seq { + Seq::new(n, |i: int| if i == t { F8(1) } else { F8(0) }) +} + +/// Column `t` of `M = forward_Λ ∘ inverse_S`: the forward transform with twiddles `tw_l` of the inverse +/// transform with twiddles `tw_s` of `e_t`, on `n` words. +pub open spec fn lde_column(tw_s: Seq, tw_l: Seq, n: nat, t: int) -> Seq { + fft_spec(ifft_spec(unit(n, t), tw_s, 1), tw_l, 1) +} + +/// Below `2^k`, indices stay below `2^k` under XOR. +proof fn lemma_xor_below(i: int, j: int, k: nat) + requires + k <= 8, + 0 <= i < pow2(k), + 0 <= j < pow2(k), + ensures + (((i as u8) ^ (j as u8)) as int) < pow2(k), +{ + if k < 8 { + lemma_shl8(k); + let (x, y, s) = (i as u8, j as u8, k as u8); + assert(x < (1u8 << s) && y < (1u8 << s) ==> (x ^ y) < (1u8 << s)) by (bit_vector) + requires + s < 8, + ; + } else { + lemma2_to64(); + } +} + +/// The `S`-to-`Λ` extension matrix commutes with XOR shifts: `M[i][j] = M[i ⊕ j][0]`, for any two `2^k`-point +/// NTTs (offsets `β_s` and `β_l`). The column of `e_j` is the column of `e_0` read at `i ⊕ j`. +/// +/// The interpolant of `e_j` on `S` is the interpolant of `e_0` translated by `j` ([`lemma_translate`]: the +/// translate has the right evaluations on `S`, and the inverse transform is injective), so its value at +/// `β_l + i` is the value of the interpolant of `e_0` at `β_l + (i ⊕ j)`. +pub proof fn lemma_lde_shift(tw_s: Seq, tw_l: Seq, k: nat, beta_s: u8, beta_l: u8, i: int, j: int) + requires + twiddles_of(tw_s, k, beta_s), + twiddles_of(tw_l, k, beta_l), + k <= 8, + 0 <= i < pow2(k), + 0 <= j < pow2(k), + ensures + lde_column(tw_s, tw_l, pow2(k), j).len() == pow2(k), + lde_column(tw_s, tw_l, pow2(k), j)[i] == lde_column(tw_s, tw_l, pow2(k), 0)[((i as u8) ^ (j as u8)) as int], +{ + let n = pow2(k); + let (e0, ej) = (unit(n, 0), unit(n, j)); + let (p0, pj) = (ifft_spec(e0, tw_s, 1), ifft_spec(ej, tw_s, 1)); + lemma_ifft_spec_len(e0, tw_s, 1); + lemma_ifft_spec_len(ej, tw_s, 1); + lemma_fft_after_ifft(e0, tw_s, 1); + lemma_fft_evaluates(tw_s, k, beta_s, p0); + lemma_fft_evaluates(tw_l, k, beta_l, p0); + lemma_fft_evaluates(tw_l, k, beta_l, pj); + let j8 = j as u8; + let q = translate(k, p0, j8); + lemma_translate(k, p0, j8, 0); + // `q` interpolates `e_j` on `S`. + lemma_fft_evaluates(tw_s, k, beta_s, q); + assert forall|u: int| 0 <= u < n implies #[trigger] fft_spec(q, tw_s, 1)[u] == ej[u] by { + let u8v = u as u8; + lemma_translate(k, p0, j8, beta_s ^ u8v); + assert((beta_s ^ u8v) ^ j8 == beta_s ^ (u8v ^ j8)) by (bit_vector); + lemma_xor_below(u, j, k); + let w = (u8v ^ j8) as int; + assert(w as u8 == u8v ^ j8); + assert(fft_spec(p0, tw_s, 1)[w] == e0[w]); + lemma_pow2_le_256(k); + assert((u8v ^ j8 == 0u8) == (u8v == j8)) by (bit_vector); + assert((u as u8 == j as u8) == (u == j)); + } + assert(fft_spec(q, tw_s, 1) =~= ej); + lemma_ifft_after_fft(q, tw_s, 1); + assert(q == pj); + // So `M[i][j] = P_q(β_l + i) = P_(p0)(β_l + (i ⊕ j)) = M[i ⊕ j][0]`. + let i8 = i as u8; + lemma_translate(k, p0, j8, beta_l ^ i8); + assert((beta_l ^ i8) ^ j8 == beta_l ^ (i8 ^ j8)) by (bit_vector); + lemma_xor_below(i, j, k); + let w = (i8 ^ j8) as int; + assert(w as u8 == i8 ^ j8); + assert(lde_column(tw_s, tw_l, n, j)[i].0 == lde_column(tw_s, tw_l, n, 0)[w].0); +} + +// --------------------------------------------------------------------------------------------- +// Executable code: the portable paths of `crates/flock/src/zerocheck/ntt.rs` +// --------------------------------------------------------------------------------------------- +/// Twiddle recurrence used to build the next subspace layer's evaluation points: +/// `next_s(s, root) = s² + root · s = s · (s + root)`. +#[inline] +pub fn next_s(s: F8, s_at_root: F8) -> (r: F8) + ensures + r.0 == f8_mul(s.0, s.0 ^ s_at_root.0), +{ + s * (s + s_at_root) +} + +#[inline] +pub fn fft_butterfly(v: &mut [F8], lambda: F8) + ensures + final(v)@ == fft_butterfly_spec(old(v)@, lambda.0), +{ + let ghost v0 = v@; + let n = v.len(); + let half = n >> 1; + proof { + assert(n >> 1 == n / 2) by (bit_vector); + } + for i in 0..half + invariant + v.len() == n, + v0.len() == n, + half == n / 2, + forall|p: int| + 0 <= p < n ==> #[trigger] v@[p] == if (0 <= p < i) || (half <= p < half + i) { + fft_butterfly_spec(v0, lambda.0)[p] + } else { + v0[p] + }, + { + let w = v[half + i]; + v[i] += lambda * w; + v[half + i] = w + v[i]; + } + assert(v@ =~= fft_butterfly_spec(v0, lambda.0)); +} + +/// Whether `n` is a power of two. +pub open spec fn is_pow2_len(n: nat) -> bool { + exists|m: nat| n == pow2(m) +} + +/// A power of two other than one halves to a power of two. +proof fn lemma_pow2_half(n: nat) + requires + is_pow2_len(n), + n != 1, + ensures + n >= 2, + n % 2 == 0, + is_pow2_len(n / 2), +{ + let m = choose|m: nat| n == pow2(m); + lemma2_to64(); + assert(m > 0); + lemma_pow2_unfold(m); + lemma_pow2_pos((m - 1) as nat); + assert(n / 2 == pow2((m - 1) as nat)); +} + +/// Rewritten: the requirement that `v` be a power-of-two length (production's only caller passes `2^k` +/// words) and that every twiddle read lie in the table are preconditions. +pub fn fft_rec(v: &mut [F8], tw: &[F8], idx: usize) + requires + is_pow2_len(old(v).len() as nat), + idx >= 1, + (idx + 1) * old(v).len() <= 2 * (tw.len() + 1), + tw.len() < 256, + ensures + final(v)@ == fft_spec(old(v)@, tw@, idx as int), + decreases old(v).len(), +{ + let ghost v0 = v@; + let n = v.len(); + if n == 1 { + return; + } + proof { + lemma_pow2_half(n as nat); + assert(idx + 1 <= tw.len() + 1) by (nonlinear_arith) + requires + (idx + 1) * n <= 2 * (tw.len() + 1), + n >= 2, + ; + } + fft_butterfly(v, tw[idx - 1]); + let half = n >> 1; + proof { + assert(n >> 1 == n / 2) by (bit_vector); + assert((2 * idx + 1) * half <= (idx + 1) * n && (2 * idx + 1 + 1) * half <= (idx + 1) * n) by (nonlinear_arith) + requires + n == 2 * half, + ; + } + let ghost w = v@; + let (lo, hi) = v.split_at_mut(half); + fft_rec(lo, tw, 2 * idx); + fft_rec(hi, tw, 2 * idx + 1); + proof { + assert(w.subrange(half as int, n as int).len() == half); + } +} + +#[inline] +pub fn ifft_butterfly(v: &mut [F8], lambda: F8) + ensures + final(v)@ == ifft_butterfly_spec(old(v)@, lambda.0), +{ + let ghost v0 = v@; + let n = v.len(); + let half = n >> 1; + proof { + assert(n >> 1 == n / 2) by (bit_vector); + } + for i in 0..half + invariant + v.len() == n, + v0.len() == n, + half == n / 2, + forall|p: int| + 0 <= p < n ==> #[trigger] v@[p] == if (0 <= p < i) || (half <= p < half + i) { + ifft_butterfly_spec(v0, lambda.0)[p] + } else { + v0[p] + }, + { + v[half + i] += v[i]; + v[i] += lambda * v[half + i]; + } + assert(v@ =~= ifft_butterfly_spec(v0, lambda.0)); +} + +/// Rewritten: the same preconditions as [`fft_rec`]. +pub fn ifft_rec(v: &mut [F8], tw: &[F8], idx: usize) + requires + is_pow2_len(old(v).len() as nat), + idx >= 1, + (idx + 1) * old(v).len() <= 2 * (tw.len() + 1), + tw.len() < 256, + ensures + final(v)@ == ifft_spec(old(v)@, tw@, idx as int), + decreases old(v).len(), +{ + let ghost v0 = v@; + let n = v.len(); + if n == 1 { + return; + } + proof { + lemma_pow2_half(n as nat); + assert(idx + 1 <= tw.len() + 1) by (nonlinear_arith) + requires + (idx + 1) * n <= 2 * (tw.len() + 1), + n >= 2, + ; + } + let half = n >> 1; + proof { + assert(n >> 1 == n / 2) by (bit_vector); + assert((2 * idx + 1) * half <= (idx + 1) * n && (2 * idx + 1 + 1) * half <= (idx + 1) * n) by (nonlinear_arith) + requires + n == 2 * half, + ; + } + let (lo, hi) = v.split_at_mut(half); + ifft_rec(lo, tw, 2 * idx); + ifft_rec(hi, tw, 2 * idx + 1); + ifft_butterfly(v, tw[idx - 1]); +} + +/// The levels of the twiddle table occupy disjoint ranges: entry `2^d - 1 + j`, `j < 2^d`, names its level. +proof fn lemma_levels_disjoint(d: nat, j: nat, d2: nat, j2: nat) + requires + j < pow2(d), + j2 < pow2(d2), + pow2(d) + j == pow2(d2) + j2, + ensures + d == d2, + j == j2, +{ + if d < d2 { + lemma_pow2_strictly_increases(d, d2); + lemma_pow2_unfold(d + 1); + if d + 1 < d2 { + lemma_pow2_strictly_increases(d + 1, d2); + } + } else if d2 < d { + lemma_pow2_strictly_increases(d2, d); + lemma_pow2_unfold(d2 + 1); + if d2 + 1 < d { + lemma_pow2_strictly_increases(d2 + 1, d); + } + } +} + +/// Level `d < k` of the table lies inside its `2^k - 1` entries. +proof fn lemma_level_in_table(d: nat, j: nat, k: nat) + requires + d < k, + j < pow2(d), + ensures + pow2(d) - 1 + j < pow2(k) - 1, + pow2(d) >= 1, +{ + lemma_pow2_pos(d); + lemma_pow2_unfold(d + 1); + if d + 1 < k { + lemma_pow2_strictly_increases(d + 1, k); + } +} + +/// Build the size-(2^k − 1) twiddle table for the additive NTT. +/// +/// Layout: level-L twiddles live at offset (2^L − 1). +/// Level 0 has 2^{k-1} twiddles, level 1 has 2^{k-2}, …, level k−1 has 1. +/// +/// Rewritten: `k <= 8` is a precondition (the domain lies in `GF(2^8)`); the `collect` of the first layer +/// and the `copy_from_slice` of it are index loops, and the unnamed loop variable is named. +pub fn compute_twiddles(k: usize, beta: F8) -> (twiddles: Vec) + requires + k <= 8, + ensures + twiddles_of(twiddles@, k as nat, beta.0), +{ + if k == 0 { + proof { + lemma2_to64(); + } + return Vec::new(); + } + let ghost b = beta.0; + proof { + lemma_pow2_le_256(k as nat); + lemma_pow2_le_256((k - 1) as nat); + lemma_usize_shl_is_mul(1, k); + lemma_usize_shl_is_mul(1, (k - 1) as usize); + lemma_pow2_unfold(k as nat); + lemma2_to64(); + } + let n = 1usize << k; + let mut twiddles = vec![F8::ZERO; n - 1]; + + // Layer 0: 2^{k-1} points beta + {0, 2, 4, ..., 2(write_at-1)}. + let mut write_at = 1usize << (k - 1); + let mut layer: Vec = Vec::new(); + for i in 0..write_at + invariant + b == beta.0, + write_at == pow2((k - 1) as nat), + write_at <= 128, + pow2(1) == 2, + layer.len() == i, + forall|p: int| 0 <= p < i ==> (#[trigger] layer@[p]).0 == subspace_poly8(0, b ^ block_offset(p as nat, 1)), + { + proof { + assert(block_offset(i as nat, 1) == (2 * i) as u8); + } + layer.push(beta + F8((2 * i) as u8)); + } + let mut s_at_root = F8::ONE; + + // Write layer 0 directly (s_at_root = 1 ⇒ no scaling needed). + for j in 0..write_at + invariant + 1 <= k <= 8, + write_at == pow2((k - 1) as nat), + layer.len() == write_at, + twiddles.len() + 1 == pow2(k as nat), + forall|p: int| 0 <= p < write_at ==> (#[trigger] layer@[p]).0 == subspace_poly8(0, b ^ block_offset(p as nat, 1)), + forall|j0: nat| + j0 < j ==> (#[trigger] twiddles@[pow2((k - 1) as nat) - 1 + j0]).0 == normalized_poly8( + 0, + b ^ block_offset(j0, 1), + ), + { + proof { + lemma_level_in_table((k - 1) as nat, j as nat, k as nat); + lemma_normalized_poly8_zero(b ^ block_offset(j as nat, 1)); + } + twiddles[write_at - 1 + j] = layer[j]; + } + proof { + assert(basis8(0) == 1) by { + assert((1u8 << 0u8) == 1u8) by (bit_vector); + } + assert forall|d: nat, j: nat| (k - 1) as nat <= d < k && j < pow2(d) implies (#[trigger] twiddles@[pow2(d) - 1 + + j]).0 == normalized_poly8((k - 1 - d) as nat, b ^ block_offset(j, (k - d) as nat)) by { + assert(d == k - 1); + } + } + + for step in 1..k + invariant + 1 <= k <= 8, + twiddles.len() + 1 == pow2(k as nat), + write_at == pow2((k - step) as nat), + layer.len() == pow2((k - 1) as nat), + forall|p: int| + 0 <= p < write_at ==> (#[trigger] layer@[p]).0 == subspace_poly8( + (step - 1) as nat, + b ^ block_offset(p as nat, step as nat), + ), + s_at_root.0 == subspace_poly8((step - 1) as nat, basis8((step - 1) as nat)), + forall|d: nat, j: nat| + (k - step) as nat <= d < k && j < pow2(d) ==> (#[trigger] twiddles@[pow2(d) - 1 + j]).0 == normalized_poly8( + (k - 1 - d) as nat, + b ^ block_offset(j, (k - d) as nat), + ), + { + let ghost dn = (k - 1 - step) as nat; + proof { + lemma_pow2_unfold((k - step) as nat); + lemma_pow2_pos(dn); + lemma_usize_shr_is_div(write_at, 1); + lemma2_to64(); + if dn + 1 < k - 1 { + lemma_pow2_strictly_increases(dn + 1, (k - 1) as nat); + } + } + write_at >>= 1; + assert(write_at == pow2(dn)); + // The root: `s_(step-1)(β + b_step) + s_(step-1)(β) = s_(step-1)(b_step)`. + let ghost (l0, l1) = (layer@[0].0, layer@[1].0); + proof { + lemma_shl8(step as nat); + let o1 = block_offset(1, step as nat); + assert(o1 == basis8(step as nat)); + assert(block_offset(0, step as nat) == 0); + lemma_subspace_poly8_additive((step - 1) as nat, b ^ o1, b); + assert(b ^ 0u8 == b && (b ^ o1) ^ b == o1) by (bit_vector); + } + let next_s_root = next_s(layer[1] + layer[0], s_at_root); + assert(next_s_root.0 == subspace_poly8(step as nat, basis8(step as nat))); + let ghost prev = layer@; + for i in 0..write_at + invariant + 1 <= step < k <= 8, + write_at == pow2(dn), + dn == k - 1 - step, + 2 * write_at <= pow2((k - 1) as nat), + prev.len() == pow2((k - 1) as nat), + layer.len() == pow2((k - 1) as nat), + s_at_root.0 == subspace_poly8((step - 1) as nat, basis8((step - 1) as nat)), + forall|p: int| + 0 <= p < 2 * write_at ==> (#[trigger] prev[p]).0 == subspace_poly8( + (step - 1) as nat, + b ^ block_offset(p as nat, step as nat), + ), + forall|p: int| i <= p < layer.len() ==> #[trigger] layer@[p] == prev[p], + forall|p: int| + 0 <= p < i ==> (#[trigger] layer@[p]).0 == subspace_poly8( + step as nat, + b ^ block_offset(p as nat, (step + 1) as nat), + ), + { + proof { + lemma_pow2_unfold((step + 1) as nat); + assert(2 * i * pow2(step as nat) == i * pow2((step + 1) as nat)) by (nonlinear_arith) + requires + pow2((step + 1) as nat) == 2 * pow2(step as nat), + ; + assert(block_offset((2 * i) as nat, step as nat) == block_offset(i as nat, (step + 1) as nat)); + } + layer[i] = next_s(layer[2 * i], s_at_root); + } + s_at_root = next_s_root; + + let s_inv = s_at_root.inv(); + let ghost done = twiddles@; + for j in 0..write_at + invariant + 1 <= step < k <= 8, + write_at == pow2(dn), + dn == k - 1 - step, + layer.len() == pow2((k - 1) as nat), + write_at <= layer.len(), + done.len() + 1 == pow2(k as nat), + twiddles.len() + 1 == pow2(k as nat), + s_inv.0 == f8_inv(subspace_poly8(step as nat, basis8(step as nat))), + forall|p: int| + 0 <= p < write_at ==> (#[trigger] layer@[p]).0 == subspace_poly8( + step as nat, + b ^ block_offset(p as nat, (step + 1) as nat), + ), + forall|q: int| 0 <= q < twiddles.len() && !(write_at - 1 <= q < write_at - 1 + j) ==> #[trigger] twiddles@[q] == done[q], + forall|j0: nat| + j0 < j ==> (#[trigger] twiddles@[pow2(dn) - 1 + j0]).0 == normalized_poly8( + step as nat, + b ^ block_offset(j0, (step + 1) as nat), + ), + { + proof { + lemma_level_in_table(dn, j as nat, k as nat); + } + twiddles[write_at - 1 + j] = s_inv * layer[j]; + } + proof { + assert forall|d: nat, j: nat| (k - (step + 1)) as nat <= d < k && j < pow2(d) implies (#[trigger] twiddles@[pow2( + d, + ) - 1 + j]).0 == normalized_poly8((k - 1 - d) as nat, b ^ block_offset(j, (k - d) as nat)) by { + lemma_level_in_table(d, j, k as nat); + if d != dn { + let q = pow2(d) - 1 + j; + if pow2(dn) - 1 <= q < pow2(dn) - 1 + pow2(dn) { + lemma_levels_disjoint(d, j, dn, (q - (pow2(dn) - 1)) as nat); + } + assert(twiddles@[q] == done[q]); + } + } + } + } + twiddles +} + +/// Additive NTT over GF(2^8) with domain of size 2^k. +/// +/// Evaluation domain `W = β + span{1, 2, …, 2^{k-1}}` (additive coset of an +/// F_2 subspace of F_{2^8}). Maximum useful `k` is 7 (|W| = 128); k = 8 would +/// exhaust all 256 elements of F_{2^8}. +/// +/// Internal LCH basis: the forward transform maps coefficients in the +/// Lin-Chung-Han basis to evaluations at the 2^k points of the domain. +/// `inverse` is the exact reverse. +#[derive(Clone, Debug)] +#[verifier::allow(autoderive_clone_without_spec)] +pub struct AdditiveNttGf8 { + k: usize, + twiddles: Vec, +} + +impl AdditiveNttGf8 { + /// The domain has `2^k_spec()` points. + pub closed spec fn k_spec(&self) -> nat { + self.k as nat + } + + /// The twiddle table. + pub closed spec fn tw(&self) -> Seq { + self.twiddles@ + } + + /// At most `2^8` points, and a table of `2^k - 1` twiddles. + pub open spec fn well_formed(&self) -> bool { + self.k_spec() <= 8 && self.tw().len() + 1 == pow2(self.k_spec()) + } + + /// Build an NTT for a 2^k-point domain with offset β. + /// + /// Rewritten: `k <= 8` is a precondition, as for [`compute_twiddles`]. + pub fn new(k: usize, beta: F8) -> (r: Self) + requires + k <= 8, + ensures + r.k_spec() == k, + r.well_formed(), + twiddles_of(r.tw(), k as nat, beta.0), + { + Self { k, twiddles: compute_twiddles(k, beta) } + } + + pub const fn k(&self) -> (r: usize) + ensures + r == self.k_spec(), + { + self.k + } + + /// Rewritten: `k <= 8` (well-formedness) is a precondition, for the shift. + pub const fn domain_size(&self) -> (r: usize) + requires + self.well_formed(), + ensures + r == pow2(self.k_spec()), + { + proof { + lemma_pow2_le_256(self.k as nat); + lemma_usize_shl_is_mul(1, self.k); + } + 1usize << self.k + } + + /// The forward transform: `fft_rec` from the root. With the table of `new(k, β)`, word `u` of the output + /// is the novel-basis polynomial of the input at `β + u` ([`lemma_fft_evaluates`]). + /// + /// Rewritten: the `assert_eq!` on the length is a precondition. + pub fn forward(&self, v: &mut [F8]) + requires + self.well_formed(), + old(v).len() == pow2(self.k_spec()), + ensures + final(v)@ == fft_spec(old(v)@, self.tw(), 1), + { + if v.len() <= 1 { + return; + } + proof { + lemma_pow2_le_256(self.k as nat); + } + fft_rec(v, &self.twiddles, 1); + } + + /// The inverse transform: `ifft_rec` from the root, which undoes `forward` ([`lemma_ifft_after_fft`], + /// [`lemma_fft_after_ifft`]) and so interpolates ([`lemma_ifft_interpolates`]). + /// + /// Rewritten: the `assert_eq!` on the length is a precondition. + pub fn inverse(&self, v: &mut [F8]) + requires + self.well_formed(), + old(v).len() == pow2(self.k_spec()), + ensures + final(v)@ == ifft_spec(old(v)@, self.tw(), 1), + { + if v.len() <= 1 { + return; + } + proof { + lemma_pow2_le_256(self.k as nat); + } + ifft_rec(v, &self.twiddles, 1); + } +} + +// --------------------------------------------------------------------------------------------- +// The single-table collapse of `M = forward_Λ ∘ inverse_S` (`ntt/inv_table.rs`) +// --------------------------------------------------------------------------------------------- +/// `XOR_{j < n} f(j)`. +pub open spec fn xor_sum8(f: spec_fn(int) -> u8, n: nat) -> u8 + decreases n, +{ + if n == 0 { + 0 + } else { + xor_sum8(f, (n - 1) as nat) ^ f(n - 1) + } +} + +/// Bit `t < 8` of the byte `w`. +pub open spec fn byte_bit(w: usize, t: int) -> bool { + 0 <= t < 8 && (w >> (t as usize)) & 1 == 1 +} + +/// Word `i` of row `w` of the table: `Σ_{t < 8, bit_t(w) = 1} M[i][t]`, the XOR of the columns `t < 8` of `M` +/// over the set bits of `w`. +pub open spec fn table_row(tw_s: Seq, tw_l: Seq, n: nat, w: usize, i: int) -> u8 { + xor_sum8(|t: int| if byte_bit(w, t) { lde_column(tw_s, tw_l, n, t)[i].0 } else { 0 }, 8) +} + +/// Word `i` of `M x`, `x` the bits of `bytes` (bit `t` of byte `b` is `x_(8b+t)`): `Σ_{j < 8 len} x_j M[i][j]`. +pub open spec fn lde_apply(tw_s: Seq, tw_l: Seq, n: nat, bytes: Seq, i: int) -> u8 { + xor_sum8( + |j: int| if byte_bit(bytes[j / 8] as usize, j % 8) { lde_column(tw_s, tw_l, n, j)[i].0 } else { 0 }, + 8 * bytes.len(), + ) +} + +/// What `apply_scalar` computes at word `i`: `Σ_b T[bytes[b]][i ⊕ 8b]`, `T` the table of `ell`-word rows. +pub open spec fn apply_formula(data: Seq, ell: nat, bytes: Seq, i: int) -> u8 { + apply_partial(data, ell, bytes, i, bytes.len()) +} + +/// The first `nb` terms of [`apply_formula`]. +pub open spec fn apply_partial(data: Seq, ell: nat, bytes: Seq, i: int, nb: nat) -> u8 { + xor_sum8(|b: int| data[(bytes[b] as int) * ell + (((i as usize) ^ ((8 * b) as usize)) as int)].0, nb) +} + +proof fn lemma_xor_sum8_ext(f: spec_fn(int) -> u8, g: spec_fn(int) -> u8, n: nat) + requires + forall|j: int| 0 <= j < n ==> #[trigger] f(j) == g(j), + ensures + xor_sum8(f, n) == xor_sum8(g, n), + decreases n, +{ + if n > 0 { + lemma_xor_sum8_ext(f, g, (n - 1) as nat); + } +} + +/// Changing one term of a sum by `c` changes the sum by `c`. +proof fn lemma_xor_sum8_one_term(f: spec_fn(int) -> u8, g: spec_fn(int) -> u8, n: nat, t: int, c: u8) + requires + 0 <= t < n, + forall|j: int| 0 <= j < n && j != t ==> #[trigger] f(j) == g(j), + f(t) == g(t) ^ c, + ensures + xor_sum8(f, n) == xor_sum8(g, n) ^ c, + decreases n, +{ + let (a, b) = (xor_sum8(f, (n - 1) as nat), xor_sum8(g, (n - 1) as nat)); + if t == n - 1 { + lemma_xor_sum8_ext(f, g, (n - 1) as nat); + let x = g(t); + assert(a ^ (x ^ c) == (a ^ x) ^ c) by (bit_vector); + } else { + lemma_xor_sum8_one_term(f, g, (n - 1) as nat, t, c); + let x = f(n - 1); + assert((b ^ c) ^ x == (b ^ x) ^ c) by (bit_vector); + } +} + +/// A sum of zeros is zero. +proof fn lemma_xor_sum8_zero(f: spec_fn(int) -> u8, n: nat) + requires + forall|j: int| 0 <= j < n ==> #[trigger] f(j) == 0, + ensures + xor_sum8(f, n) == 0, + decreases n, +{ + if n > 0 { + lemma_xor_sum8_zero(f, (n - 1) as nat); + assert(0u8 ^ 0u8 == 0u8) by (bit_vector); + } +} + +/// `Σ_{j < a + c} f(j) = Σ_{j < a} f(j) + Σ_{t < c} f(a + t)`. +proof fn lemma_xor_sum8_split(f: spec_fn(int) -> u8, a: nat, c: nat) + ensures + xor_sum8(f, a + c) == xor_sum8(f, a) ^ xor_sum8(|t: int| f(a + t), c), + decreases c, +{ + let g = |t: int| f(a + t); + if c == 0 { + let x = xor_sum8(f, a); + assert(x ^ 0u8 == x) by (bit_vector); + } else { + lemma_xor_sum8_split(f, a, (c - 1) as nat); + assert(xor_sum8(f, a + c) == xor_sum8(f, (a + c - 1) as nat) ^ f(a + c - 1)); + assert(g(c - 1) == f(a + c - 1)); + let (x, y, z) = (xor_sum8(f, a), xor_sum8(g, (c - 1) as nat), f(a + c - 1)); + assert((x ^ y) ^ z == x ^ (y ^ z)) by (bit_vector); + } +} + +/// A sum over `8 n` terms, regrouped by bytes: `Σ_{j < 8n} f(j) = Σ_{b < n} Σ_{t < 8} f(8b + t)`. +proof fn lemma_xor_sum8_bytes(f: spec_fn(int) -> u8, n: nat) + ensures + xor_sum8(f, 8 * n) == xor_sum8(|b: int| xor_sum8(|t: int| f(8 * b + t), 8), n), + decreases n, +{ + if n > 0 { + let m = (n - 1) as nat; + lemma_xor_sum8_bytes(f, m); + lemma_xor_sum8_split(f, 8 * m, 8); + assert(8 * m + 8 == 8 * n); + lemma_xor_sum8_ext(|t: int| f(8 * m + t), |t: int| f(8 * (n - 1) + t), 8); + } +} + +/// Flipping a set bit `t` of `w` removes column `t` from row `w`. +proof fn lemma_table_row_flip(tw_s: Seq, tw_l: Seq, n: nat, w: usize, t: usize, i: int) + requires + t < 8, + (w >> t) & 1 == 1, + ensures + table_row(tw_s, tw_l, n, w, i) == table_row(tw_s, tw_l, n, w ^ (1usize << t), i) ^ lde_column( + tw_s, + tw_l, + n, + t as int, + )[i].0, +{ + let w2 = w ^ (1usize << t); + let f = |t0: int| if byte_bit(w, t0) { lde_column(tw_s, tw_l, n, t0)[i].0 } else { 0 }; + let g = |t0: int| if byte_bit(w2, t0) { lde_column(tw_s, tw_l, n, t0)[i].0 } else { 0 }; + assert forall|j: int| 0 <= j < 8 && j != t implies #[trigger] f(j) == g(j) by { + let ju = j as usize; + assert(((w ^ (1usize << t)) >> ju) & 1 == (w >> ju) & 1) by (bit_vector) + requires + ju < 8, + t < 8, + ju != t, + ; + } + assert(((w ^ (1usize << t)) >> t) & 1 == 0) by (bit_vector) + requires + t < 8, + (w >> t) & 1 == 1, + ; + let c = lde_column(tw_s, tw_l, n, t as int)[i].0; + assert(0u8 ^ c == c) by (bit_vector); + lemma_xor_sum8_one_term(f, g, 8, t as int, c); +} + +/// Row `0` is zero, and row `2^t` is column `t`. +proof fn lemma_table_row_units(tw_s: Seq, tw_l: Seq, n: nat, t: usize, i: int) + requires + t < 8, + ensures + table_row(tw_s, tw_l, n, 0, i) == 0, + table_row(tw_s, tw_l, n, 1usize << t, i) == lde_column(tw_s, tw_l, n, t as int)[i].0, +{ + let f = |t0: int| if byte_bit(0, t0) { lde_column(tw_s, tw_l, n, t0)[i].0 } else { 0 }; + assert forall|j: int| 0 <= j < 8 implies #[trigger] f(j) == 0 by { + let ju = j as usize; + assert((0usize >> ju) & 1 == 0) by (bit_vector); + } + lemma_xor_sum8_zero(f, 8); + let w = 1usize << t; + assert((w >> t) & 1 == 1 && w ^ (1usize << t) == 0) by (bit_vector) + requires + t < 8, + w == 1usize << t, + ; + lemma_table_row_flip(tw_s, tw_l, n, w, t, i); + let c = lde_column(tw_s, tw_l, n, t as int)[i].0; + assert(0u8 ^ c == c) by (bit_vector); +} + +/// XOR of indices below `2^8`, as bytes or as words. +proof fn lemma_xor_index(i: int, j: int) + requires + 0 <= i < 256, + 0 <= j < 256, + ensures + (((i as u8) ^ (j as u8)) as int) == (((i as usize) ^ (j as usize)) as int), +{ + let (x, y) = (i as usize, j as usize); + assert(x < 256 && y < 256 ==> ((x as u8) ^ (y as u8)) as usize == x ^ y) by (bit_vector); + assert(x as u8 == i as u8 && y as u8 == j as u8); +} + +/// `apply_scalar`'s sum is `M` applied to the row's bits: with the table of two `2^k`-point NTTs (offsets `β_s` +/// and `β_l`), `Σ_b T[bytes[b]][i ⊕ 8b] = Σ_j x_j M[i][j]`. Each byte's row read at `i ⊕ 8b` is the sum of the +/// columns `8b + t` by [`lemma_lde_shift`]. +pub proof fn lemma_apply_formula( + tw_s: Seq, + tw_l: Seq, + k: nat, + beta_s: u8, + beta_l: u8, + data: Seq, + bytes: Seq, + i: int, +) + requires + twiddles_of(tw_s, k, beta_s), + twiddles_of(tw_l, k, beta_l), + 3 <= k <= 7, + data.len() == 256 * pow2(k), + forall|w: int, i0: int| + 0 <= w < 256 && 0 <= i0 < pow2(k) ==> (#[trigger] data[w * pow2(k) + i0]).0 == table_row( + tw_s, + tw_l, + pow2(k), + w as usize, + i0, + ), + 8 * bytes.len() == pow2(k), + 0 <= i < pow2(k), + ensures + apply_formula(data, pow2(k), bytes, i) == lde_apply(tw_s, tw_l, pow2(k), bytes, i), +{ + let n = pow2(k); + let nc = bytes.len(); + lemma_pow2_le_256(k); + lemma_usize_shl_is_mul(1, k as usize); + let ku = k as usize; + assert(n == (1usize << ku)); + let f = |j: int| if byte_bit(bytes[j / 8] as usize, j % 8) { lde_column(tw_s, tw_l, n, j)[i].0 } else { 0 }; + lemma_xor_sum8_bytes(f, nc); + let lhs = |b: int| data[(bytes[b] as int) * n + (((i as usize) ^ ((8 * b) as usize)) as int)].0; + let rhs = |b: int| xor_sum8(|t: int| f(8 * b + t), 8); + assert forall|b: int| 0 <= b < nc implies #[trigger] lhs(b) == rhs(b) by { + let w = bytes[b] as usize; + let (iu, sh) = (i as usize, (8 * b) as usize); + assert(8 * b < n); + assert(iu < (1usize << ku) && sh < (1usize << ku) ==> (iu ^ sh) < (1usize << ku)) by (bit_vector) + requires + ku < 64, + ; + let ip = (iu ^ sh) as int; + assert(lhs(b) == table_row(tw_s, tw_l, n, w, ip)); + let g = |t: int| if byte_bit(w, t) { lde_column(tw_s, tw_l, n, t)[ip].0 } else { 0 }; + assert forall|t: int| 0 <= t < 8 implies #[trigger] g(t) == f(8 * b + t) by { + let j = 8 * b + t; + assert(j / 8 == b && j % 8 == t); + lemma_lde_shift(tw_s, tw_l, k, beta_s, beta_l, ip, t); + lemma_lde_shift(tw_s, tw_l, k, beta_s, beta_l, i, j); + lemma_xor_index(ip, t); + lemma_xor_index(i, j); + let (tu, bu, ju) = (t as usize, b as usize, j as usize); + assert(pow2(3) == 8) by { + lemma2_to64(); + } + lemma_usize_shl_is_mul(bu, 3); + assert(sh == bu << 3usize); + assert(ju == add(bu << 3usize, tu)); + assert(((iu ^ sh) ^ tu) == (iu ^ ju)) by (bit_vector) + requires + sh == bu << 3usize, + ju == add(bu << 3usize, tu), + tu < 8, + bu < 256, + ; + } + lemma_xor_sum8_ext(g, |t: int| f(8 * b + t), 8); + } + lemma_xor_sum8_ext(lhs, rhs, nc); +} + +/// Row `w` of `data` (rows of `n` words) is row `w` of the table. +pub open spec fn row_ok(data: Seq, tw_s: Seq, tw_l: Seq, n: nat, w: int) -> bool { + forall|i: int| 0 <= i < n ==> (#[trigger] data[w * n + i]).0 == table_row(tw_s, tw_l, n, w as usize, i) +} + +/// Rows of `n` words do not overlap. +proof fn lemma_row_disjoint(w0: int, i: int, r: int, n: nat) + requires + 0 <= w0, + 0 <= r, + 0 <= i < n, + w0 != r, + ensures + !(r * n <= w0 * n + i < r * n + n), +{ + if w0 < r { + assert(w0 * n + i < r * n) by (nonlinear_arith) + requires + w0 + 1 <= r, + i < n, + ; + } else { + assert(w0 * n + i >= r * n + n) by (nonlinear_arith) + requires + w0 >= r + 1, + i >= 0, + ; + } +} + +/// Writing row `r` keeps every other row. +proof fn lemma_row_kept(before: Seq, after: Seq, tw_s: Seq, tw_l: Seq, n: nat, r: int, w0: int) + requires + 0 <= r < 256, + 0 <= w0 < 256, + w0 != r, + before.len() == 256 * n, + after.len() == 256 * n, + row_ok(before, tw_s, tw_l, n, w0), + forall|q: int| 0 <= q < after.len() && !(r * n <= q < r * n + n) ==> #[trigger] after[q] == before[q], + ensures + row_ok(after, tw_s, tw_l, n, w0), +{ + assert forall|i: int| 0 <= i < n implies (#[trigger] after[w0 * n + i]).0 == table_row(tw_s, tw_l, n, w0 as usize, i) by { + lemma_row_disjoint(w0, i, r, n); + assert(w0 * n + i < 256 * n) by (nonlinear_arith) + requires + w0 < 256, + i < n, + ; + assert(0 <= w0 * n + i) by (nonlinear_arith) + requires + w0 >= 0, + i >= 0, + ; + assert(before[w0 * n + i].0 == table_row(tw_s, tw_l, n, w0 as usize, i)); + } +} + +/// A byte with one bit set is a power of two. +proof fn lemma_byte_pow2(w0: usize) + requires + 1 <= w0 < 256, + w0 & sub(w0, 1) == 0, + ensures + exists|t0: usize| t0 < 8 && w0 == 1usize << t0, +{ + broadcast use vstd::std_specs::bits::axiom_u64_trailing_zeros; + let wl = w0 as u64; + let z = vstd::std_specs::bits::u64_trailing_zeros(wl) as u64; + assert(z < 64 && (wl >> z) & 1u64 == 1u64 && wl << sub(64, z) == 0); + assert(z < 8 && w0 == 1usize << (z as usize)) by (bit_vector) + requires + wl == w0 as u64, + z < 64, + (wl >> z) & 1u64 == 1u64, + wl << sub(64, z) == 0, + w0 & sub(w0, 1) == 0, + w0 < 256, + ; + assert((z as usize) < 8 && w0 == 1usize << (z as usize)); +} + +/// The lowest set bit of a nonzero byte, as `w.trailing_zeros()` finds it: below 8, set, and clearing it +/// lowers `w`. +proof fn lemma_low_bit(w: usize) + requires + 1 <= w < 256, + ensures + ({ + let z = vstd::std_specs::bits::u64_trailing_zeros(w as u64) as usize; + &&& z < 8 + &&& (w >> z) & 1 == 1 + &&& (w ^ (1usize << z)) < w + &&& 1 <= (1usize << z) < 256 + &&& (1usize << z) & sub(1usize << z, 1) == 0 + }), +{ + broadcast use vstd::std_specs::bits::axiom_u64_trailing_zeros; + let wl = w as u64; + let z = vstd::std_specs::bits::u64_trailing_zeros(wl) as u64; + assert(z < 64 && (wl >> z) & 1u64 == 1u64); + assert(z < 8) by (bit_vector) + requires + (wl >> z) & 1u64 == 1u64, + wl < 256, + z < 64, + ; + let zu = z as usize; + assert((w >> zu) & 1 == 1 && (w ^ (1usize << zu)) < w && 1 <= (1usize << zu) < 256 && (1usize << zu) & sub(1usize + << zu, 1) == 0) by (bit_vector) + requires + wl == w as u64, + zu == z as usize, + z < 8, + (wl >> z) & 1u64 == 1u64, + w < 256, + ; +} + +/// The table that collapses the round-1 extension through the NTT, one row of `ell` words per byte value. +#[derive(Clone, Debug)] +#[verifier::allow(autoderive_clone_without_spec)] +pub struct InvNttTableByteSingleGf8 { + pub k: usize, + pub ell: usize, + pub n_chunks: usize, + /// `data[w * ell .. (w+1) * ell]` = T_0[w], the XOR-sum of columns of `M` + /// indexed by the set bits of `w`. + data: Vec, +} + +impl InvNttTableByteSingleGf8 { + /// The table, row after row. + pub closed spec fn data_spec(&self) -> Seq { + self.data@ + } + + /// The number of words per row. + pub closed spec fn ell_spec(&self) -> nat { + self.ell as nat + } + + /// The domain has `2^k_spec()` points. + pub closed spec fn k_spec(&self) -> nat { + self.k as nat + } + + /// The shape `new` builds: `ell = 2^k` words per row, `n_chunks = ell / 8` bytes per input row, 256 rows. + pub closed spec fn well_formed(&self) -> bool { + &&& 3 <= self.k <= 7 + &&& self.ell == pow2(self.k as nat) + &&& self.n_chunks == self.ell / 8 + &&& self.data_spec().len() == 256 * self.ell + } + + /// Built from the NTTs with twiddles `tw_s` (input domain) and `tw_l` (output domain): row `w` is + /// [`table_row`]. + pub closed spec fn is_table_of(&self, tw_s: Seq, tw_l: Seq) -> bool { + &&& self.well_formed() + &&& forall|w: int, i: int| + 0 <= w < 256 && 0 <= i < self.ell ==> (#[trigger] self.data_spec()[w * self.ell + i]).0 == table_row( + tw_s, + tw_l, + self.ell as nat, + w as usize, + i, + ) + } + + /// Build the table given the two NTT instances: `ntt_S` over the input + /// domain, `ntt_L` over the output (extension) domain. Both must have the + /// same `k`. + /// + /// Rewritten: the three `assert!`s are preconditions; `tmp.iter_mut().for_each(..)`, the + /// `cols.iter().enumerate()` loop and the `copy_from_slice` are index loops; `w.trailing_zeros()` is + /// taken on `w as u64` (equal for every `w`; vstd specifies `trailing_zeros` for `u64`, not `usize`). + pub fn new(ntt_s: &AdditiveNttGf8, ntt_l: &AdditiveNttGf8) -> (r: Self) + requires + ntt_s.well_formed(), + ntt_l.well_formed(), + ntt_s.k_spec() == ntt_l.k_spec(), + 3 <= ntt_s.k_spec() <= 7, + ensures + r.k_spec() == ntt_s.k_spec(), + r.ell_spec() == pow2(ntt_s.k_spec()), + r.is_table_of(ntt_s.tw(), ntt_l.tw()), + { + let ghost (tw_s, tw_l) = (ntt_s.tw(), ntt_l.tw()); + let k = ntt_s.k(); + proof { + lemma_pow2_le_256(k as nat); + lemma_usize_shl_is_mul(1, k); + lemma2_to64(); + if k > 3 { + lemma_pow2_strictly_increases(3, k as nat); + } + } + let ell = 1usize << k; + let n_chunks = ell / 8; + let ghost n = ell as nat; + + let mut data = vec![F8::ZERO; 256 * ell]; + assert(forall|q: int| 0 <= q < data.len() ==> #[trigger] data@[q] == F8(0)); + + // Compute the 8 unit-column images cols[t] = fwd_NTT_Λ ∘ inv_NTT_S (e_t) + // for t ∈ 0..8. The remaining columns of M are XOR-shifted versions. + let mut tmp = vec![F8::ZERO; ell]; + let mut cols: Vec> = Vec::with_capacity(8); + for t in 0..8 + invariant + ntt_s.well_formed(), + ntt_l.well_formed(), + tw_s == ntt_s.tw(), + tw_l == ntt_l.tw(), + ell == n, + n == pow2(ntt_s.k_spec()), + n == pow2(ntt_l.k_spec()), + 8 <= n <= 128, + data.len() == 256 * n, + forall|q: int| 0 <= q < data.len() ==> #[trigger] data@[q] == F8(0), + tmp.len() == n, + cols.len() == t, + forall|t0: int| 0 <= t0 < t ==> (#[trigger] cols@[t0])@ == lde_column(tw_s, tw_l, n, t0), + { + for x in 0..ell + invariant + tmp.len() == ell, + forall|p: int| 0 <= p < x ==> #[trigger] tmp@[p] == F8(0), + { + tmp[x] = F8::ZERO; + } + tmp[t] = F8::ONE; + assert(tmp@ =~= unit(n, t as int)); + ntt_s.inverse(&mut tmp); + proof { + lemma_ifft_spec_len(unit(n, t as int), tw_s, 1); + } + ntt_l.forward(&mut tmp); + proof { + lemma_fft_spec_len(ifft_spec(unit(n, t as int), tw_s, 1), tw_l, 1); + } + let ghost before = cols@; + cols.push(tmp.clone()); + proof { + assert(cols@[t as int]@ =~= lde_column(tw_s, tw_l, n, t as int)); + assert forall|t0: int| 0 <= t0 < t + 1 implies (#[trigger] cols@[t0])@ == lde_column(tw_s, tw_l, n, t0) by { + if t0 < t { + assert(cols@[t0] == before[t0]); + } + } + } + } + proof { + assert forall|t0: int| 0 <= t0 < 8 implies (#[trigger] cols@[t0])@.len() == n by { + lemma_fft_spec_len(ifft_spec(unit(n, t0), tw_s, 1), tw_l, 1); + lemma_ifft_spec_len(unit(n, t0), tw_s, 1); + } + } + + // T_0[0] already zero. T_0[2^t] = cols[t]. Then for non-power-of-two w, + // T_0[w] = T_0[w ^ lo_bit] ⊕ T_0[lo_bit]; this builds all 256 entries + // with one XOR per entry. + proof { + assert forall|i: int| 0 <= i < n implies (#[trigger] data@[0 * n + i]).0 == table_row(tw_s, tw_l, n, 0usize, i) by { + lemma_table_row_units(tw_s, tw_l, n, 0, i); + } + assert(row_ok(data@, tw_s, tw_l, n, 0)); + } + for t in 0..8 + invariant + ell == n, + 8 <= n <= 128, + data.len() == 256 * n, + cols.len() == 8, + forall|t0: int| 0 <= t0 < 8 ==> (#[trigger] cols@[t0])@ == lde_column(tw_s, tw_l, n, t0), + forall|t0: int| 0 <= t0 < 8 ==> (#[trigger] cols@[t0])@.len() == n, + row_ok(data@, tw_s, tw_l, n, 0), + forall|t0: usize| t0 < t ==> #[trigger] row_ok(data@, tw_s, tw_l, n, (1usize << t0) as int), + { + proof { + lemma2_to64(); + if t < 7 { + lemma_pow2_strictly_increases(t as nat, 7); + } + lemma_usize_shl_is_mul(1, t); + assert((1usize << t) * ell <= 128 * 128) by (nonlinear_arith) + requires + (1usize << t) <= 128, + ell <= 128, + ; + } + let ghost row = (1usize << t) as int; + let ghost before = data@; + let entry_start = (1usize << t) * ell; + for i in 0..ell + invariant + ell == n, + 8 <= n <= 128, + t < 8, + 1 <= row < 256, + entry_start == row * n, + data.len() == 256 * n, + before.len() == 256 * n, + cols.len() == 8, + cols@[t as int]@.len() == n, + forall|i0: int| 0 <= i0 < i ==> #[trigger] data@[entry_start + i0] == cols@[t as int]@[i0], + forall|q: int| 0 <= q < data.len() && !(entry_start <= q < entry_start + i) ==> #[trigger] data@[q] == before[q], + { + proof { + assert(entry_start + i < 256 * n) by (nonlinear_arith) + requires + entry_start == row * n, + row < 256, + i < n, + ; + } + data[entry_start + i] = cols[t][i]; + } + proof { + assert forall|i: int| 0 <= i < n implies (#[trigger] data@[row * n + i]).0 == table_row(tw_s, tw_l, n, row as usize, i) by { + lemma_table_row_units(tw_s, tw_l, n, t, i); + } + lemma_row_kept(before, data@, tw_s, tw_l, n, row, 0); + assert forall|t0: usize| t0 < t + 1 implies #[trigger] row_ok(data@, tw_s, tw_l, n, (1usize << t0) as int) by { + if t0 < t { + let r0 = (1usize << t0) as int; + assert(t0 < 8 && t0 != t ==> (1usize << t0) != (1usize << t)) by (bit_vector); + if t0 < 7 { + lemma_pow2_strictly_increases(t0 as nat, 7); + } + lemma_usize_shl_is_mul(1, t0); + lemma_row_kept(before, data@, tw_s, tw_l, n, row, r0); + } + } + } + } + proof { + assert forall|w0: usize| w0 < 256 && (w0 < 3 || (w0 >= 1 && w0 & sub(w0, 1) == 0)) implies #[trigger] row_ok( + data@, + tw_s, + tw_l, + n, + w0 as int, + ) by { + if w0 != 0 { + assert(w0 < 3 ==> w0 & sub(w0, 1) == 0) by (bit_vector); + lemma_byte_pow2(w0); + let t0 = choose|t0: usize| t0 < 8 && w0 == 1usize << t0; + assert(row_ok(data@, tw_s, tw_l, n, (1usize << t0) as int)); + } + } + } + for w in 3usize..256 + invariant + ell == n, + 8 <= n <= 128, + data.len() == 256 * n, + forall|w0: usize| + w0 < 256 && (w0 < w || (w0 >= 1 && w0 & sub(w0, 1) == 0)) ==> #[trigger] row_ok( + data@, + tw_s, + tw_l, + n, + w0 as int, + ), + { + // Rewritten: `if (w & (w - 1)) == 0 { continue; }` (Verus's `for` has no `continue`) guards the rest. + if (w & (w - 1)) != 0 { + let ghost tz = vstd::std_specs::bits::u64_trailing_zeros(w as u64); + proof { + lemma_low_bit(w); + } + let lo_bit = 1usize << (w as u64).trailing_zeros(); + let parent = w ^ lo_bit; + proof { + let zu = tz as usize; + assert(lo_bit == 1usize << zu); + assert(row_ok(data@, tw_s, tw_l, n, parent as int)); + assert(row_ok(data@, tw_s, tw_l, n, lo_bit as int)); + assert(parent * ell <= 256 * 128 && lo_bit * ell <= 256 * 128 && w * ell <= 256 * 128) by (nonlinear_arith) + requires + parent < 256, + lo_bit < 256, + w < 256, + ell <= 128, + ; + } + // Borrow-checker friendly: read parent + bit_v slices, then write entry. + let (parent_off, bit_off, entry_off) = (parent * ell, lo_bit * ell, w * ell); + let ghost before = data@; + for i in 0..ell + invariant + ell == n, + 8 <= n <= 128, + 3 <= w < 256, + parent < 256, + lo_bit < 256, + parent != w, + lo_bit != w, + parent_off == parent * n, + bit_off == lo_bit * n, + entry_off == w * n, + data.len() == 256 * n, + before.len() == 256 * n, + forall|i0: int| + 0 <= i0 < i ==> (#[trigger] data@[entry_off + i0]).0 == before[parent_off + i0].0 ^ before[bit_off + + i0].0, + forall|q: int| 0 <= q < data.len() && !(entry_off <= q < entry_off + i) ==> #[trigger] data@[q] == before[q], + { + proof { + assert(entry_off + i < 256 * n && parent_off + i < 256 * n && bit_off + i < 256 * n) by (nonlinear_arith) + requires + entry_off == w * n, + parent_off == parent * n, + bit_off == lo_bit * n, + w < 256, + parent < 256, + lo_bit < 256, + i < n, + ; + lemma_row_disjoint(parent as int, i as int, w as int, n); + lemma_row_disjoint(lo_bit as int, i as int, w as int, n); + } + let v = data[parent_off + i] + data[bit_off + i]; + data[entry_off + i] = v; + } + proof { + let zu = tz as usize; + assert forall|i: int| 0 <= i < n implies (#[trigger] data@[(w as int) * n + i]).0 == table_row( + tw_s, + tw_l, + n, + w, + i, + ) by { + assert(before[(parent as int) * n + i].0 == table_row(tw_s, tw_l, n, parent, i)); + assert(before[(lo_bit as int) * n + i].0 == table_row(tw_s, tw_l, n, lo_bit, i)); + lemma_table_row_flip(tw_s, tw_l, n, w, zu, i); + lemma_table_row_units(tw_s, tw_l, n, zu, i); + } + assert forall|w0: usize| + w0 < 256 && (w0 < w + 1 || (w0 >= 1 && w0 & sub(w0, 1) == 0)) implies #[trigger] row_ok( + data@, + tw_s, + tw_l, + n, + w0 as int, + ) by { + if w0 != w { + assert(row_ok(before, tw_s, tw_l, n, w0 as int)); + lemma_row_kept(before, data@, tw_s, tw_l, n, w as int, w0 as int); + } + } + } + } + } + proof { + assert forall|w0: int, i: int| 0 <= w0 < 256 && 0 <= i < ell implies (#[trigger] data@[w0 * ell + i]).0 + == table_row(tw_s, tw_l, ell as nat, w0 as usize, i) by { + assert(row_ok(data@, tw_s, tw_l, n, (w0 as usize) as int)); + } + } + Self { k, ell, n_chunks, data } + } + + /// The number of input bytes per row. + pub closed spec fn n_chunks_spec(&self) -> nat { + self.n_chunks as nat + } + + /// Scalar reference. Kept public so tests can use it as the cross-check + /// oracle for the NEON variant. + /// + /// Rewritten: the two `assert_eq!`s are preconditions; `out.iter_mut().for_each(..)` and the + /// `bytes.iter().enumerate()` loop are index loops. + pub fn apply_scalar(&self, bytes: &[u8], out: &mut [F8]) + requires + self.well_formed(), + bytes.len() == self.n_chunks_spec(), + old(out).len() == self.ell_spec(), + ensures + final(out).len() == self.ell_spec(), + forall|i: int| + 0 <= i < self.ell_spec() ==> (#[trigger] final(out)@[i]).0 == apply_formula( + self.data_spec(), + self.ell_spec(), + bytes@, + i, + ), + { + let ghost (data, ell) = (self.data_spec(), self.ell_spec()); + let ghost ku = self.k; + proof { + lemma_pow2_le_256(self.k as nat); + lemma_usize_shl_is_mul(1, ku); + lemma2_to64(); + if self.k > 3 { + lemma_pow2_strictly_increases(3, self.k as nat); + } + } + let n = out.len(); + for x in 0..n + invariant + out.len() == n, + forall|p: int| 0 <= p < x ==> #[trigger] out@[p] == F8(0), + { + out[x] = F8::ZERO; + } + assert forall|i: int| 0 <= i < ell implies (#[trigger] out@[i]).0 == apply_partial(data, ell, bytes@, i, 0) by { + } + for b in 0..bytes.len() + invariant + self.well_formed(), + data == self.data_spec(), + ell == self.ell_spec(), + ku == self.k, + ell == (1usize << ku), + 8 <= ell <= 128, + bytes.len() == ell / 8, + out.len() == ell, + forall|i: int| 0 <= i < ell ==> (#[trigger] out@[i]).0 == apply_partial(data, ell, bytes@, i, b as nat), + { + let byte_b = bytes[b]; + proof { + assert((byte_b as usize) * self.ell + self.ell <= 256 * self.ell) by (nonlinear_arith) + requires + byte_b < 256, + ; + } + let row_off = byte_b as usize * self.ell; + let row = &self.data[row_off..row_off + self.ell]; + let shift = 8 * b; + let ghost before = out@; + for i in 0..self.ell + invariant + ell == self.ell, + ku < 64, + ell == (1usize << ku), + 8 <= ell <= 128, + shift == 8 * b, + shift < ell, + row@ == data.subrange(row_off as int, row_off + ell), + row_off == (byte_b as int) * ell, + row_off + ell <= data.len(), + byte_b == bytes@[b as int], + b < bytes.len(), + out.len() == ell, + before.len() == ell, + forall|i0: int| 0 <= i0 < ell ==> (#[trigger] before[i0]).0 == apply_partial(data, ell, bytes@, i0, b as nat), + forall|i0: int| i <= i0 < ell ==> #[trigger] out@[i0] == before[i0], + forall|i0: int| + 0 <= i0 < i ==> (#[trigger] out@[i0]).0 == apply_partial(data, ell, bytes@, i0, (b + 1) as nat), + { + proof { + assert(i < (1usize << ku) && shift < (1usize << ku) ==> (i ^ shift) < (1usize << ku)) by (bit_vector) + requires + ku < 64, + ; + assert((i ^ shift) < ell); + assert(row.len() == ell); + assert(row@[(i ^ shift) as int] == data[row_off + ((i ^ shift) as int)]); + assert(apply_partial(data, ell, bytes@, i as int, (b + 1) as nat) == apply_partial( + data, + ell, + bytes@, + i as int, + b as nat, + ) ^ data[row_off + ((i ^ shift) as int)].0); + } + out[i] += row[i ^ shift]; + } + } + } +} + +/// `apply_scalar` applies `M`: for the table of two `2^k`-point NTTs (twiddles `tw_s` of offset `β_s`, `tw_l` of +/// offset `β_l`), its output word `i` on an input row of `ell / 8` bytes is `Σ_j x_j M[i][j]`, `x` the row's +/// bits ([`lde_apply`]). +pub proof fn lemma_table_applies_lde( + table: InvNttTableByteSingleGf8, + tw_s: Seq, + tw_l: Seq, + beta_s: u8, + beta_l: u8, + bytes: Seq, + i: int, +) + requires + table.is_table_of(tw_s, tw_l), + twiddles_of(tw_s, table.k_spec(), beta_s), + twiddles_of(tw_l, table.k_spec(), beta_l), + bytes.len() == table.n_chunks_spec(), + 0 <= i < table.ell_spec(), + ensures + apply_formula(table.data_spec(), table.ell_spec(), bytes, i) == lde_apply(tw_s, tw_l, table.ell_spec(), bytes, i), +{ + let k = table.k_spec(); + lemma_pow2_le_256(k); + lemma2_to64(); + if k > 3 { + lemma_pow2_strictly_increases(3, k); + } + lemma_apply_formula(tw_s, tw_l, k, beta_s, beta_l, table.data_spec(), bytes, i); +} + +} // verus! diff --git a/verification/verus/src/gf2_64.rs b/verification/verus/src/gf2_64.rs new file mode 100644 index 0000000000..7fe8205d2d --- /dev/null +++ b/verification/verus/src/gf2_64.rs @@ -0,0 +1,1759 @@ +//! The base field `K = GF(2)[x] / (x^64 + x^4 + x^3 + x + 1)`. +//! +//! The executable functions are the portable paths of `crates/primitives/src/field/gf2_64.rs`, copied +//! with the same bodies; `tests/equivalence.rs` checks the two agree. +//! +//! Specification: an element is a polynomial over GF(2) of degree below 64, bit `i` its coefficient of +//! `x^i`. The product [`k_mul`] is the carry-less product ([`clmul`]) reduced modulo +//! `M = x^64 + x^4 + x^3 + x + 1`, where "reduced" is the definition of a remainder ([`is_remainder`]). +use crate::clmul::*; +use core::ops::{Add, AddAssign, Mul, MulAssign}; +use vstd::arithmetic::power2::*; +use vstd::prelude::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// The modulus `M = x^64 + x^4 + x^3 + x + 1`, as a 65-bit polynomial. +pub open spec fn modulus() -> u128 { + 0x1_0000_0000_0000_001Bu128 +} + +/// `r` is the remainder of `p` modulo `M`: `p = q * M + r` for some polynomial `q`, and `deg r < 64`. +/// +/// A `p` of degree below 128 has a quotient of degree below 64, so `q: u64` loses nothing. +pub open spec fn is_remainder(p: u128, r: u64) -> bool { + exists|q: u64| p == clmul(q, modulus()) ^ (r as u128) +} + +/// `p mod M`, the unique remainder (unique by [`lemma_remainder_unique`]). +pub open spec fn k_mod(p: u128) -> u64 { + choose|r: u64| is_remainder(p, r) +} + +/// The product in `K`: `a * b mod M`. +pub open spec fn k_mul(a: u64, b: u64) -> u64 { + k_mod(clmul(a, b as u128)) +} + +/// `a^n` in `K`. +pub open spec fn k_pow(a: u64, n: nat) -> u64 + decreases n, +{ + if n == 0 { + 1 + } else { + k_mul(k_pow(a, (n - 1) as nat), a) + } +} + +/// `a` squared `n` times, `a^(2^n)` by [`lemma_k_sq_iter_pow`]. +pub open spec fn k_sq_iter(a: u64, n: nat) -> u64 + decreases n, +{ + if n == 0 { + a + } else { + let v = k_sq_iter(a, (n - 1) as nat); + k_mul(v, v) + } +} + +// --------------------------------------------------------------------------------------------- +// The reduction +// --------------------------------------------------------------------------------------------- +/// The closed form of the production `reduce`, as a spec function. +pub open spec fn reduce_formula(p: u128) -> u64 { + let (lo, hi) = (p as u64, (p >> 64u128) as u64); + let spill = (hi >> 63u64) ^ (hi >> 61u64) ^ (hi >> 60u64); + let v = hi ^ spill; + lo ^ v ^ (v << 1u64) ^ (v << 3u64) ^ (v << 4u64) +} + +/// Multiplication by `x` in `K`: shift, and fold `x^64` back as `x^4 + x^3 + x + 1`. +pub open spec fn mulx(a: u64) -> u64 { + (a << 1u64) ^ (if a >> 63u64 == 1 { + 0x1Bu64 + } else { + 0u64 + }) +} + +/// `a * x^k` in `K`. +pub open spec fn mulx_pow(a: u64, k: nat) -> u64 + decreases k, +{ + if k == 0 { + a + } else { + mulx(mulx_pow(a, (k - 1) as nat)) + } +} + +/// `q * (x^4 + x^3 + x + 1) = q ^ q<<1 ^ q<<3 ^ q<<4`, the product by the reduction constant `0x1B`. +pub open spec fn times_r64(q: u64) -> u128 { + let w = q as u128; + w ^ (w << 1u128) ^ (w << 3u128) ^ (w << 4u128) +} + +pub proof fn lemma_clmul_r64(q: u64) + ensures + clmul(q, 0x1Bu128) == times_r64(q), +{ + assert(0x1Bu128 == (1u128 << 0u128) ^ (1u128 << 1u128) ^ (1u128 << 3u128) ^ (1u128 << 4u128)) + by (bit_vector); + let (e0, e1, e3, e4) = (1u128 << 0u128, 1u128 << 1u128, 1u128 << 3u128, 1u128 << 4u128); + lemma_clmul_xor_right(q, e0 ^ e1 ^ e3, e4); + lemma_clmul_xor_right(q, e0 ^ e1, e3); + lemma_clmul_xor_right(q, e0, e1); + lemma_clmul_monomial(q, 0); + lemma_clmul_monomial(q, 1); + lemma_clmul_monomial(q, 3); + lemma_clmul_monomial(q, 4); + let w = q as u128; + assert(w << 0u128 == w) by (bit_vector); +} + +/// `q * M = q x^64 + q * 0x1B`. +pub proof fn lemma_clmul_modulus(q: u64) + ensures + clmul(q, modulus()) == ((q as u128) << 64u128) ^ times_r64(q), +{ + assert(modulus() == 0x1Bu128 ^ (1u128 << 64u128)) by (bit_vector); + lemma_clmul_xor_right(q, 0x1Bu128, 1u128 << 64u128); + lemma_clmul_r64(q); + lemma_clmul_monomial(q, 64); + let (a, b) = (times_r64(q), (q as u128) << 64u128); + assert(a ^ b == b ^ a) by (bit_vector); +} + +/// The production reduction returns a remainder of `p` modulo `M`, with quotient `hi ^ spill`. +pub proof fn lemma_reduce_formula_is_remainder(p: u128) + ensures + is_remainder(p, reduce_formula(p)), +{ + let hi = (p >> 64u128) as u64; + let q = hi ^ ((hi >> 63u64) ^ (hi >> 61u64) ^ (hi >> 60u64)); + lemma_clmul_modulus(q); + assert({ + let hi = (p >> 64u128) as u64; + let q = hi ^ ((hi >> 63u64) ^ (hi >> 61u64) ^ (hi >> 60u64)); + let w = q as u128; + p == ((w << 64u128) ^ (w ^ (w << 1u128) ^ (w << 3u128) ^ (w << 4u128))) ^ (reduce_formula(p) as u128) + }) by (bit_vector); +} + +/// The remainder modulo `M` is unique. +pub proof fn lemma_remainder_unique(p: u128, r1: u64, r2: u64) + requires + is_remainder(p, r1), + is_remainder(p, r2), + ensures + r1 == r2, +{ + let q1 = choose|q: u64| p == clmul(q, modulus()) ^ (r1 as u128); + let q2 = choose|q: u64| p == clmul(q, modulus()) ^ (r2 as u128); + lemma_clmul_modulus(q1); + lemma_clmul_modulus(q2); + assert({ + let (w1, w2) = (q1 as u128, q2 as u128); + ((w1 << 64u128) ^ (w1 ^ (w1 << 1u128) ^ (w1 << 3u128) ^ (w1 << 4u128))) ^ (r1 as u128) == ((w2 + << 64u128) ^ (w2 ^ (w2 << 1u128) ^ (w2 << 3u128) ^ (w2 << 4u128))) ^ (r2 as u128) ==> r1 == r2 + }) by (bit_vector); +} + +/// `p mod M` is the production reduction's closed form. +pub proof fn lemma_k_mod(p: u128) + ensures + k_mod(p) == reduce_formula(p), + is_remainder(p, k_mod(p)), +{ + lemma_reduce_formula_is_remainder(p); + lemma_remainder_unique(p, k_mod(p), reduce_formula(p)); +} + +/// Reduction is GF(2)-linear. +pub proof fn lemma_k_mod_xor(p1: u128, p2: u128) + ensures + k_mod(p1 ^ p2) == k_mod(p1) ^ k_mod(p2), +{ + lemma_k_mod(p1); + lemma_k_mod(p2); + lemma_k_mod(p1 ^ p2); + assert(reduce_formula(p1 ^ p2) == reduce_formula(p1) ^ reduce_formula(p2)) by (bit_vector); +} + +/// A polynomial of degree below 64 is its own remainder. +pub proof fn lemma_k_mod_small(p: u128) + requires + p >> 64u128 == 0, + ensures + k_mod(p) == p as u64, +{ + lemma_k_mod(p); + assert(p >> 64u128 == 0 ==> reduce_formula(p) == p as u64) by (bit_vector); +} + +/// `(p x) mod M = (p mod M) x mod M`. +pub proof fn lemma_k_mod_shl1(p: u128) + requires + p >> 127u128 == 0, + ensures + k_mod(p << 1u128) == mulx(k_mod(p)), +{ + lemma_k_mod(p); + lemma_k_mod(p << 1u128); + assert(p >> 127u128 == 0 ==> reduce_formula(p << 1u128) == mulx(reduce_formula(p))) by (bit_vector); +} + +/// A multiple of `M` reduces to zero. +pub proof fn lemma_k_mod_multiple(q: u64) + ensures + k_mod(clmul(q, modulus())) == 0, +{ + lemma_clmul_modulus(q); + lemma_k_mod(clmul(q, modulus())); + assert({ + let w = q as u128; + reduce_formula((w << 64u128) ^ (w ^ (w << 1u128) ^ (w << 3u128) ^ (w << 4u128))) == 0 + }) by (bit_vector); +} + +// --------------------------------------------------------------------------------------------- +// K is a commutative ring +// --------------------------------------------------------------------------------------------- +pub proof fn lemma_k_mul_comm(a: u64, b: u64) + ensures + k_mul(a, b) == k_mul(b, a), +{ + lemma_clmul_comm(a, b); +} + +pub proof fn lemma_k_mul_xor_right(a: u64, b: u64, c: u64) + ensures + k_mul(a, b ^ c) == k_mul(a, b) ^ k_mul(a, c), +{ + assert((b ^ c) as u128 == (b as u128) ^ (c as u128)) by (bit_vector); + lemma_clmul_xor_right(a, b as u128, c as u128); + lemma_k_mod_xor(clmul(a, b as u128), clmul(a, c as u128)); +} + +pub proof fn lemma_k_mul_xor_left(a: u64, b: u64, c: u64) + ensures + k_mul(a ^ b, c) == k_mul(a, c) ^ k_mul(b, c), +{ + lemma_k_mul_comm(a ^ b, c); + lemma_k_mul_comm(a, c); + lemma_k_mul_comm(b, c); + lemma_k_mul_xor_right(c, a, b); +} + +pub proof fn lemma_k_mul_one(a: u64) + ensures + k_mul(a, 1) == a, + k_mul(1, a) == a, +{ + assert(1u64 as u128 == 1u128 << 0u128) by (bit_vector); + lemma_clmul_monomial(a, 0); + assert((a as u128) << 0u128 == (a as u128) && (a as u128) >> 64u128 == 0) by (bit_vector); + lemma_k_mod_small(a as u128); + lemma_k_mul_comm(a, 1); +} + +pub proof fn lemma_k_mul_zero(a: u64) + ensures + k_mul(a, 0) == 0, + k_mul(0, a) == 0, +{ + lemma_clmul_zero(a); + assert(0u128 >> 64u128 == 0) by (bit_vector); + lemma_k_mod_small(0); + lemma_k_mul_comm(a, 0); +} + +/// `a * (x b) = x (a * b)`. +pub proof fn lemma_k_mul_mulx(a: u64, b: u64) + ensures + k_mul(a, mulx(b)) == mulx(k_mul(a, b)), +{ + let m: u128 = if b >> 63u64 == 1 { + modulus() + } else { + 0 + }; + // b x = mulx(b) + [b_63] M, as polynomials. + assert((b as u128) << 1u128 == (mulx(b) as u128) ^ (if b >> 63u64 == 1 { + 0x1_0000_0000_0000_001Bu128 + } else { + 0u128 + })) by (bit_vector); + lemma_clmul_shl1(a, b as u128); + lemma_clmul_xor_right(a, mulx(b) as u128, m); + lemma_clmul_bound(a, b); + lemma_k_mod_shl1(clmul(a, b as u128)); + lemma_k_mod_xor(clmul(a, mulx(b) as u128), clmul(a, m)); + if b >> 63u64 == 1 { + lemma_k_mod_multiple(a); + } else { + lemma_clmul_zero(a); + assert(0u128 >> 64u128 == 0) by (bit_vector); + lemma_k_mod_small(0); + } + let r = k_mod(clmul(a, mulx(b) as u128)); + assert(r ^ 0 == r) by (bit_vector); +} + +pub proof fn lemma_k_mul_mulx_pow(a: u64, b: u64, k: nat) + ensures + k_mul(a, mulx_pow(b, k)) == mulx_pow(k_mul(a, b), k), + decreases k, +{ + if k > 0 { + lemma_k_mul_mulx_pow(a, b, (k - 1) as nat); + lemma_k_mul_mulx(a, mulx_pow(b, (k - 1) as nat)); + } +} + +/// `x^k` is `1 << k` for `k < 64`. +pub proof fn lemma_mulx_pow_one(k: nat) + requires + k < 64, + ensures + mulx_pow(1, k) == 1u64 << (k as u64), + decreases k, +{ + if k == 0 { + assert(1u64 << 0u64 == 1) by (bit_vector); + } else { + lemma_mulx_pow_one((k - 1) as nat); + let s = (k - 1) as u64; + assert(s < 63 ==> mulx(1u64 << s) == 1u64 << (s + 1)) by (bit_vector); + } +} + +/// `t * x^k = mulx^k(t)`. +pub proof fn lemma_k_mul_monomial(t: u64, k: nat) + requires + k < 64, + ensures + k_mul(t, 1u64 << (k as u64)) == mulx_pow(t, k), +{ + lemma_mulx_pow_one(k); + lemma_k_mul_mulx_pow(t, 1, k); + lemma_k_mul_one(t); +} + +/// Associativity, by linearity in `c` and the monomial case `(a b) x^k = a (b x^k)`. +pub proof fn lemma_k_mul_assoc_upto(a: u64, b: u64, c: u64, n: nat) + requires + n <= 64, + ensures + k_mul(k_mul(a, b), low(c, n)) == k_mul(a, k_mul(b, low(c, n))), + decreases n, +{ + if n == 0 { + assert(low(c, 0) == 0) by { + assert(c & ((1u64 << 0u64) - 1) as u64 == 0) by (bit_vector); + } + lemma_k_mul_zero(k_mul(a, b)); + lemma_k_mul_zero(b); + lemma_k_mul_zero(a); + } else { + let i = (n - 1) as nat; + lemma_k_mul_assoc_upto(a, b, c, i); + lemma_low_step(c, i); + let e: u64 = if bit(c, i) { + 1u64 << (i as u64) + } else { + 0 + }; + let lo = low(c, i); + lemma_k_mul_xor_right(k_mul(a, b), lo, e); + lemma_k_mul_xor_right(b, lo, e); + lemma_k_mul_xor_right(a, k_mul(b, lo), k_mul(b, e)); + if bit(c, i) { + lemma_k_mul_monomial(k_mul(a, b), i); + lemma_k_mul_monomial(b, i); + lemma_k_mul_mulx_pow(a, b, i); + } else { + lemma_k_mul_zero(k_mul(a, b)); + lemma_k_mul_zero(b); + lemma_k_mul_zero(a); + } + } +} + +pub proof fn lemma_k_mul_assoc(a: u64, b: u64, c: u64) + ensures + k_mul(k_mul(a, b), c) == k_mul(a, k_mul(b, c)), +{ + lemma_k_mul_assoc_upto(a, b, c, 64); +} + +// --------------------------------------------------------------------------------------------- +// Powers +// --------------------------------------------------------------------------------------------- +pub proof fn lemma_k_pow_add(a: u64, m: nat, n: nat) + ensures + k_mul(k_pow(a, m), k_pow(a, n)) == k_pow(a, m + n), + decreases n, +{ + if n == 0 { + lemma_k_mul_one(k_pow(a, m)); + } else { + lemma_k_pow_add(a, m, (n - 1) as nat); + lemma_k_mul_assoc(k_pow(a, m), k_pow(a, (n - 1) as nat), a); + assert((m + n - 1) as nat + 1 == m + n); + } +} + +/// Squaring `n` times raises to the power `2^n`: `(a^m)^(2^n) = a^(m 2^n)`. +pub proof fn lemma_k_sq_iter_pow(a: u64, m: nat, n: nat) + ensures + k_sq_iter(k_pow(a, m), n) == k_pow(a, m * pow2(n)), + decreases n, +{ + if n == 0 { + lemma2_to64(); + assert(m * pow2(0) == m) by (nonlinear_arith) + requires + pow2(0) == 1, + ; + } else { + lemma_k_sq_iter_pow(a, m, (n - 1) as nat); + let e = m * pow2((n - 1) as nat); + lemma_k_pow_add(a, e, e); + lemma_pow2_unfold(n); + assert(e + e == m * pow2(n)) by (nonlinear_arith) + requires + e == m * pow2((n - 1) as nat), + pow2(n) == 2 * pow2((n - 1) as nat), + ; + } +} + +pub proof fn lemma_k_sq_iter_add(a: u64, m: nat, n: nat) + ensures + k_sq_iter(k_sq_iter(a, m), n) == k_sq_iter(a, m + n), + decreases n, +{ + if n > 0 { + lemma_k_sq_iter_add(a, m, (n - 1) as nat); + } +} + +// --------------------------------------------------------------------------------------------- +// Fermat: every element of K is fixed by the 64th power of squaring, a^(2^64) = a +// --------------------------------------------------------------------------------------------- +/// The carry-less square in closed form: bit `i` of `a` moves to bit `2i` (Morton spread with zeros). +pub open spec fn spread(a: u64) -> u128 { + let x = a as u128; + let x = (x | (x << 32u128)) & 0x0000_0000_FFFF_FFFF_0000_0000_FFFF_FFFFu128; + let x = (x | (x << 16u128)) & 0x0000_FFFF_0000_FFFF_0000_FFFF_0000_FFFFu128; + let x = (x | (x << 8u128)) & 0x00FF_00FF_00FF_00FF_00FF_00FF_00FF_00FFu128; + let x = (x | (x << 4u128)) & 0x0F0F_0F0F_0F0F_0F0F_0F0F_0F0F_0F0F_0F0Fu128; + let x = (x | (x << 2u128)) & 0x3333_3333_3333_3333_3333_3333_3333_3333u128; + (x | (x << 1u128)) & 0x5555_5555_5555_5555_5555_5555_5555_5555u128 +} + +/// The square of a polynomial over GF(2) is its bit spread: cross terms cancel in pairs. +pub proof fn lemma_clmul_square_upto(a: u64, n: nat) + requires + n <= 64, + ensures + clmul(low(a, n), low(a, n) as u128) == spread(low(a, n)), + decreases n, +{ + if n == 0 { + assert(low(a, 0) == 0) by { + assert(a & ((1u64 << 0u64) - 1) as u64 == 0) by (bit_vector); + } + lemma_clmul_zero(0); + assert(spread(0) == 0) by (bit_vector); + } else { + let i = (n - 1) as nat; + lemma_clmul_square_upto(a, i); + lemma_low_step(a, i); + let l = low(a, i); + let e: u64 = if bit(a, i) { + 1u64 << (i as u64) + } else { + 0 + }; + let (lw, ew) = (l as u128, e as u128); + assert((l ^ e) as u128 == (l as u128) ^ (e as u128)) by (bit_vector); + lemma_clmul_xor_left(l, e, lw ^ ew); + lemma_clmul_xor_right(l, lw, ew); + lemma_clmul_xor_right(e, lw, ew); + lemma_clmul_comm(l, e); + assert(spread(l ^ e) == spread(l) ^ spread(e)) by (bit_vector); + if bit(a, i) { + let s = i as u64; + assert(s < 64 ==> ((1u64 << s) as u128) == 1u128 << (s as u128)) by (bit_vector); + lemma_clmul_monomial(e, i as u128); + assert(s < 64 ==> ((1u64 << s) as u128) << (s as u128) == spread(1u64 << s)) by (bit_vector); + } else { + lemma_clmul_zero(0); + lemma_clmul_zero(l); + lemma_clmul_comm(l, 0); + assert(spread(0) == 0) by (bit_vector); + } + let (x, y, z) = (clmul(l, lw), clmul(l, ew), clmul(e, ew)); + assert((x ^ y) ^ (y ^ z) == x ^ z) by (bit_vector); + } +} + +/// Squaring in `K` in closed form. +pub proof fn lemma_k_square(a: u64) + ensures + k_mul(a, a) == reduce_formula(spread(a)), +{ + lemma_clmul_square_upto(a, 64); + lemma_k_mod(clmul(a, a as u128)); +} + +/// `(a + b)^2 = a^2 + b^2`. +pub proof fn lemma_k_square_xor(a: u64, b: u64) + ensures + k_mul(a ^ b, a ^ b) == k_mul(a, a) ^ k_mul(b, b), +{ + lemma_k_mul_xor_left(a, b, a ^ b); + lemma_k_mul_xor_right(a, a, b); + lemma_k_mul_xor_right(b, a, b); + lemma_k_mul_comm(a, b); + let (x, y, z) = (k_mul(a, a), k_mul(a, b), k_mul(b, b)); + assert((x ^ y) ^ (y ^ z) == x ^ z) by (bit_vector); +} + +pub proof fn lemma_k_sq_iter_xor(a: u64, b: u64, n: nat) + ensures + k_sq_iter(a ^ b, n) == k_sq_iter(a, n) ^ k_sq_iter(b, n), + decreases n, +{ + if n > 0 { + lemma_k_sq_iter_xor(a, b, (n - 1) as nat); + lemma_k_square_xor(k_sq_iter(a, (n - 1) as nat), k_sq_iter(b, (n - 1) as nat)); + } +} + +/// `(a b)^2 = a^2 b^2`. +pub proof fn lemma_k_square_mul(a: u64, b: u64) + ensures + k_mul(k_mul(a, b), k_mul(a, b)) == k_mul(k_mul(a, a), k_mul(b, b)), +{ + let ab = k_mul(a, b); + lemma_k_mul_assoc(a, b, ab); + lemma_k_mul_assoc(b, a, b); + lemma_k_mul_comm(a, b); + lemma_k_mul_assoc(a, b, b); + lemma_k_mul_assoc(a, a, k_mul(b, b)); + lemma_k_mul_comm(b, ab); +} + +pub proof fn lemma_k_sq_iter_mul(a: u64, b: u64, n: nat) + ensures + k_sq_iter(k_mul(a, b), n) == k_mul(k_sq_iter(a, n), k_sq_iter(b, n)), + decreases n, +{ + if n > 0 { + lemma_k_sq_iter_mul(a, b, (n - 1) as nat); + lemma_k_square_mul(k_sq_iter(a, (n - 1) as nat), k_sq_iter(b, (n - 1) as nat)); + } +} + +pub proof fn lemma_k_sq_iter_one(n: nat) + ensures + k_sq_iter(1, n) == 1, + decreases n, +{ + if n > 0 { + lemma_k_sq_iter_one((n - 1) as nat); + lemma_k_mul_one(1); + } +} + +pub proof fn lemma_k_sq_iter_zero(n: nat) + ensures + k_sq_iter(0, n) == 0, + decreases n, +{ + if n > 0 { + lemma_k_sq_iter_zero((n - 1) as nat); + lemma_k_mul_zero(0); + } +} + +/// Squaring commutes with powers: `(a^i)^(2^n) = (a^(2^n))^i`. +pub proof fn lemma_k_sq_iter_k_pow(a: u64, i: nat, n: nat) + ensures + k_sq_iter(k_pow(a, i), n) == k_pow(k_sq_iter(a, n), i), + decreases i, +{ + if i == 0 { + lemma_k_sq_iter_one(n); + } else { + lemma_k_sq_iter_k_pow(a, (i - 1) as nat, n); + lemma_k_sq_iter_mul(k_pow(a, (i - 1) as nat), a, n); + } +} + +/// The powers of the generator `x` below `x^64` are the monomials. +pub proof fn lemma_k_pow_x(i: nat) + requires + i < 64, + ensures + k_pow(2, i) == 1u64 << (i as u64), + decreases i, +{ + if i == 0 { + assert(1u64 << 0u64 == 1) by (bit_vector); + } else { + lemma_k_pow_x((i - 1) as nat); + let t = k_pow(2, (i - 1) as nat); + assert(2u64 == 1u64 << 1u64) by (bit_vector); + lemma_k_mul_monomial(t, 1); + assert(mulx_pow(t, 0) == t); + let s = (i - 1) as u64; + assert(s < 63 ==> mulx(1u64 << s) == 1u64 << (s + 1)) by (bit_vector); + } +} + +/// Squarings 0 to 8 of `x`. +proof fn lemma_k_frobenius_x_0() + ensures + k_sq_iter(0x2u64, 8) == 0x11011u64, +{ + assert(k_sq_iter(0x2u64, 0) == 0x2u64); + lemma_k_square(0x2u64); + assert(reduce_formula(spread(0x2u64)) == 0x4u64) by (bit_vector); + assert(k_sq_iter(0x2u64, 1) == 0x4u64); + lemma_k_square(0x4u64); + assert(reduce_formula(spread(0x4u64)) == 0x10u64) by (bit_vector); + assert(k_sq_iter(0x2u64, 2) == 0x10u64); + lemma_k_square(0x10u64); + assert(reduce_formula(spread(0x10u64)) == 0x100u64) by (bit_vector); + assert(k_sq_iter(0x2u64, 3) == 0x100u64); + lemma_k_square(0x100u64); + assert(reduce_formula(spread(0x100u64)) == 0x10000u64) by (bit_vector); + assert(k_sq_iter(0x2u64, 4) == 0x10000u64); + lemma_k_square(0x10000u64); + assert(reduce_formula(spread(0x10000u64)) == 0x100000000u64) by (bit_vector); + assert(k_sq_iter(0x2u64, 5) == 0x100000000u64); + lemma_k_square(0x100000000u64); + assert(reduce_formula(spread(0x100000000u64)) == 0x1bu64) by (bit_vector); + assert(k_sq_iter(0x2u64, 6) == 0x1bu64); + lemma_k_square(0x1bu64); + assert(reduce_formula(spread(0x1bu64)) == 0x145u64) by (bit_vector); + assert(k_sq_iter(0x2u64, 7) == 0x145u64); + lemma_k_square(0x145u64); + assert(reduce_formula(spread(0x145u64)) == 0x11011u64) by (bit_vector); + assert(k_sq_iter(0x2u64, 8) == 0x11011u64); +} + +/// Squarings 8 to 16 of `x`. +proof fn lemma_k_frobenius_x_1() + ensures + k_sq_iter(0x11011u64, 8) == 0x15f0144001a114fu64, +{ + assert(k_sq_iter(0x11011u64, 0) == 0x11011u64); + lemma_k_square(0x11011u64); + assert(reduce_formula(spread(0x11011u64)) == 0x101000101u64) by (bit_vector); + assert(k_sq_iter(0x11011u64, 1) == 0x101000101u64); + lemma_k_square(0x101000101u64); + assert(reduce_formula(spread(0x101000101u64)) == 0x100000001001au64) by (bit_vector); + assert(k_sq_iter(0x11011u64, 2) == 0x100000001001au64); + lemma_k_square(0x100000001001au64); + assert(reduce_formula(spread(0x100000001001au64)) == 0x1a00000144u64) by (bit_vector); + assert(k_sq_iter(0x11011u64, 3) == 0x1a00000144u64); + lemma_k_square(0x1a00000144u64); + assert(reduce_formula(spread(0x1a00000144u64)) == 0x10dbcu64) by (bit_vector); + assert(k_sq_iter(0x11011u64, 4) == 0x10dbcu64); + lemma_k_square(0x10dbcu64); + assert(reduce_formula(spread(0x10dbcu64)) == 0x100514550u64) by (bit_vector); + assert(k_sq_iter(0x11011u64, 5) == 0x100514550u64); + lemma_k_square(0x100514550u64); + assert(reduce_formula(spread(0x100514550u64)) == 0x11011011111bu64) by (bit_vector); + assert(k_sq_iter(0x11011u64, 6) == 0x11011011111bu64); + lemma_k_square(0x11011011111bu64); + assert(reduce_formula(spread(0x11011011111bu64)) == 0x10001011a1a015eu64) by (bit_vector); + assert(k_sq_iter(0x11011u64, 7) == 0x10001011a1a015eu64); + lemma_k_square(0x10001011a1a015eu64); + assert(reduce_formula(spread(0x10001011a1a015eu64)) == 0x15f0144001a114fu64) by (bit_vector); + assert(k_sq_iter(0x11011u64, 8) == 0x15f0144001a114fu64); +} + +/// Squarings 16 to 24 of `x`. +proof fn lemma_k_frobenius_x_2() + ensures + k_sq_iter(0x15f0144001a114fu64, 8) == 0xb6d535c542116f62u64, +{ + assert(k_sq_iter(0x15f0144001a114fu64, 0) == 0x15f0144001a114fu64); + lemma_k_square(0x15f0144001a114fu64); + assert(reduce_formula(spread(0x15f0144001a114fu64)) == 0x1aad43011ba1e5u64) by (bit_vector); + assert(k_sq_iter(0x15f0144001a114fu64, 1) == 0x1aad43011ba1e5u64); + lemma_k_square(0x1aad43011ba1e5u64); + assert(reduce_formula(spread(0x1aad43011ba1e5u64)) == 0x11cefef6be466u64) by (bit_vector); + assert(k_sq_iter(0x15f0144001a114fu64, 2) == 0x11cefef6be466u64); + lemma_k_square(0x11cefef6be466u64); + assert(reduce_formula(spread(0x11cefef6be466u64)) == 0x5455145e48670f13u64) by (bit_vector); + assert(k_sq_iter(0x15f0144001a114fu64, 3) == 0x5455145e48670f13u64); + lemma_k_square(0x5455145e48670f13u64); + assert(reduce_formula(spread(0x5455145e48670f13u64)) == 0xbaf1bebe1ae4ad02u64) by (bit_vector); + assert(k_sq_iter(0x15f0144001a114fu64, 4) == 0xbaf1bebe1ae4ad02u64); + lemma_k_square(0xbaf1bebe1ae4ad02u64); + assert(reduce_formula(spread(0xbaf1bebe1ae4ad02u64)) == 0xb0ef530df44bb042u64) by (bit_vector); + assert(k_sq_iter(0x15f0144001a114fu64, 5) == 0xb0ef530df44bb042u64); + lemma_k_square(0xb0ef530df44bb042u64); + assert(reduce_formula(spread(0xb0ef530df44bb042u64)) == 0xe2170b43ee771735u64) by (bit_vector); + assert(k_sq_iter(0x15f0144001a114fu64, 6) == 0xe2170b43ee771735u64); + lemma_k_square(0xe2170b43ee771735u64); + assert(reduce_formula(spread(0xe2170b43ee771735u64)) == 0x48380fd207a3b527u64) by (bit_vector); + assert(k_sq_iter(0x15f0144001a114fu64, 7) == 0x48380fd207a3b527u64); + lemma_k_square(0x48380fd207a3b527u64); + assert(reduce_formula(spread(0x48380fd207a3b527u64)) == 0xb6d535c542116f62u64) by (bit_vector); + assert(k_sq_iter(0x15f0144001a114fu64, 8) == 0xb6d535c542116f62u64); +} + +/// Squarings 24 to 32 of `x`. +proof fn lemma_k_frobenius_x_3() + ensures + k_sq_iter(0xb6d535c542116f62u64, 8) == 0x5c62236777028505u64, +{ + assert(k_sq_iter(0xb6d535c542116f62u64, 0) == 0xb6d535c542116f62u64); + lemma_k_square(0xb6d535c542116f62u64); + assert(reduce_formula(spread(0xb6d535c542116f62u64)) == 0xa6df6baa62f965f5u64) by (bit_vector); + assert(k_sq_iter(0xb6d535c542116f62u64, 1) == 0xa6df6baa62f965f5u64); + lemma_k_square(0xa6df6baa62f965f5u64); + assert(reduce_formula(spread(0xa6df6baa62f965f5u64)) == 0xb9df3947cea0ffe7u64) by (bit_vector); + assert(k_sq_iter(0xb6d535c542116f62u64, 2) == 0xb9df3947cea0ffe7u64); + lemma_k_square(0xb9df3947cea0ffe7u64); + assert(reduce_formula(spread(0xb9df3947cea0ffe7u64)) == 0xe1882807248fe588u64) by (bit_vector); + assert(k_sq_iter(0xb6d535c542116f62u64, 3) == 0xe1882807248fe588u64); + lemma_k_square(0xe1882807248fe588u64); + assert(reduce_formula(spread(0xe1882807248fe588u64)) == 0x180d86953ed141c6u64) by (bit_vector); + assert(k_sq_iter(0xb6d535c542116f62u64, 4) == 0x180d86953ed141c6u64); + lemma_k_square(0x180d86953ed141c6u64); + assert(reduce_formula(spread(0x180d86953ed141c6u64)) == 0x1894566cd1db8abfu64) by (bit_vector); + assert(k_sq_iter(0xb6d535c542116f62u64, 5) == 0x1894566cd1db8abfu64); + lemma_k_square(0x1894566cd1db8abfu64); + assert(reduce_formula(spread(0x1894566cd1db8abfu64)) == 0x4cc78bf4ea999e25u64) by (bit_vector); + assert(k_sq_iter(0xb6d535c542116f62u64, 6) == 0x4cc78bf4ea999e25u64); + lemma_k_square(0x4cc78bf4ea999e25u64); + assert(reduce_formula(spread(0x4cc78bf4ea999e25u64)) == 0xe333308087e402bau64) by (bit_vector); + assert(k_sq_iter(0xb6d535c542116f62u64, 7) == 0xe333308087e402bau64); + lemma_k_square(0xe333308087e402bau64); + assert(reduce_formula(spread(0xe333308087e402bau64)) == 0x5c62236777028505u64) by (bit_vector); + assert(k_sq_iter(0xb6d535c542116f62u64, 8) == 0x5c62236777028505u64); +} + +/// Squarings 32 to 40 of `x`. +proof fn lemma_k_frobenius_x_4() + ensures + k_sq_iter(0x5c62236777028505u64, 8) == 0x13846a66c22c75beu64, +{ + assert(k_sq_iter(0x5c62236777028505u64, 0) == 0x5c62236777028505u64); + lemma_k_square(0x5c62236777028505u64); + assert(reduce_formula(spread(0x5c62236777028505u64)) == 0xb964dc682c67ddcdu64) by (bit_vector); + assert(k_sq_iter(0x5c62236777028505u64, 1) == 0xb964dc682c67ddcdu64); + lemma_k_square(0xb964dc682c67ddcdu64); + assert(reduce_formula(spread(0xb964dc682c67ddcdu64)) == 0xb58ac9a23d208acbu64) by (bit_vector); + assert(k_sq_iter(0x5c62236777028505u64, 2) == 0xb58ac9a23d208acbu64); + lemma_k_square(0xb58ac9a23d208acbu64); + assert(reduce_formula(spread(0xb58ac9a23d208acbu64)) == 0xb3fcc2ab3699fc73u64) by (bit_vector); + assert(k_sq_iter(0x5c62236777028505u64, 3) == 0xb3fcc2ab3699fc73u64); + lemma_k_square(0xb3fcc2ab3699fc73u64); + assert(reduce_formula(spread(0xb3fcc2ab3699fc73u64)) == 0xb2644136253abfe8u64) by (bit_vector); + assert(k_sq_iter(0x5c62236777028505u64, 4) == 0xb2644136253abfe8u64); + lemma_k_square(0xb2644136253abfe8u64); + assert(reduce_formula(spread(0xb2644136253abfe8u64)) == 0xb37cd8f5f54e22c6u64) by (bit_vector); + assert(k_sq_iter(0x5c62236777028505u64, 5) == 0xb37cd8f5f54e22c6u64); + lemma_k_square(0xb37cd8f5f54e22c6u64); + assert(reduce_formula(spread(0xb37cd8f5f54e22c6u64)) == 0xe267d02369c356e5u64) by (bit_vector); + assert(k_sq_iter(0x5c62236777028505u64, 6) == 0xe267d02369c356e5u64); + lemma_k_square(0xe267d02369c356e5u64); + assert(reduce_formula(spread(0xe267d02369c356e5u64)) == 0x82c8dc57a143827u64) by (bit_vector); + assert(k_sq_iter(0x5c62236777028505u64, 7) == 0x82c8dc57a143827u64); + lemma_k_square(0x82c8dc57a143827u64); + assert(reduce_formula(spread(0x82c8dc57a143827u64)) == 0x13846a66c22c75beu64) by (bit_vector); + assert(k_sq_iter(0x5c62236777028505u64, 8) == 0x13846a66c22c75beu64); +} + +/// Squarings 40 to 48 of `x`. +proof fn lemma_k_frobenius_x_5() + ensures + k_sq_iter(0x13846a66c22c75beu64, 8) == 0xfa5ac498d20dd97eu64, +{ + assert(k_sq_iter(0x13846a66c22c75beu64, 0) == 0x13846a66c22c75beu64); + lemma_k_square(0x13846a66c22c75beu64); + assert(reduce_formula(spread(0x13846a66c22c75beu64)) == 0x4b75c5e1cfbc9888u64) by (bit_vector); + assert(k_sq_iter(0x13846a66c22c75beu64, 1) == 0x4b75c5e1cfbc9888u64); + lemma_k_square(0x4b75c5e1cfbc9888u64); + assert(reduce_formula(spread(0x4b75c5e1cfbc9888u64)) == 0xe6e383fc30ec5c40u64) by (bit_vector); + assert(k_sq_iter(0x13846a66c22c75beu64, 2) == 0xe6e383fc30ec5c40u64); + lemma_k_square(0xe6e383fc30ec5c40u64); + assert(reduce_formula(spread(0xe6e383fc30ec5c40u64)) == 0x18db4821d1201031u64) by (bit_vector); + assert(k_sq_iter(0x13846a66c22c75beu64, 3) == 0x18db4821d1201031u64); + lemma_k_square(0x18db4821d1201031u64); + assert(reduce_formula(spread(0x18db4821d1201031u64)) == 0x4cc669b6b7c0691au64) by (bit_vector); + assert(k_sq_iter(0x13846a66c22c75beu64, 4) == 0x4cc669b6b7c0691au64); + lemma_k_square(0x4cc669b6b7c0691au64); + assert(reduce_formula(spread(0x4cc669b6b7c0691au64)) == 0xf26221ddce9cb783u64) by (bit_vector); + assert(k_sq_iter(0x13846a66c22c75beu64, 5) == 0xf26221ddce9cb783u64); + lemma_k_square(0xf26221ddce9cb783u64); + assert(reduce_formula(spread(0xf26221ddce9cb783u64)) == 0x57399d3c29092c2fu64) by (bit_vector); + assert(k_sq_iter(0x13846a66c22c75beu64, 6) == 0x57399d3c29092c2fu64); + lemma_k_square(0x57399d3c29092c2fu64); + assert(reduce_formula(spread(0x57399d3c29092c2fu64)) == 0xae86719cd83b743eu64) by (bit_vector); + assert(k_sq_iter(0x13846a66c22c75beu64, 7) == 0xae86719cd83b743eu64); + lemma_k_square(0xae86719cd83b743eu64); + assert(reduce_formula(spread(0xae86719cd83b743eu64)) == 0xfa5ac498d20dd97eu64) by (bit_vector); + assert(k_sq_iter(0x13846a66c22c75beu64, 8) == 0xfa5ac498d20dd97eu64); +} + +/// Squarings 48 to 56 of `x`. +proof fn lemma_k_frobenius_x_6() + ensures + k_sq_iter(0xfa5ac498d20dd97eu64, 8) == 0x10a51a6f8e1e1acu64, +{ + assert(k_sq_iter(0xfa5ac498d20dd97eu64, 0) == 0xfa5ac498d20dd97eu64); + lemma_k_square(0xfa5ac498d20dd97eu64); + assert(reduce_formula(spread(0xfa5ac498d20dd97eu64)) == 0x50a9adfa20f7c8d5u64) by (bit_vector); + assert(k_sq_iter(0xfa5ac498d20dd97eu64, 1) == 0x50a9adfa20f7c8d5u64); + lemma_k_square(0x50a9adfa20f7c8d5u64); + assert(reduce_formula(spread(0x50a9adfa20f7c8d5u64)) == 0xaf06ffc8fb2c50a6u64) by (bit_vector); + assert(k_sq_iter(0xfa5ac498d20dd97eu64, 2) == 0xaf06ffc8fb2c50a6u64); + lemma_k_square(0xaf06ffc8fb2c50a6u64); + assert(reduce_formula(spread(0xaf06ffc8fb2c50a6u64)) == 0xfe42058b1100328eu64) by (bit_vector); + assert(k_sq_iter(0xfa5ac498d20dd97eu64, 3) == 0xfe42058b1100328eu64); + lemma_k_square(0xfe42058b1100328eu64); + assert(reduce_formula(spread(0xfe42058b1100328eu64)) == 0x11cb06c04a986a2u64) by (bit_vector); + assert(k_sq_iter(0xfa5ac498d20dd97eu64, 4) == 0x11cb06c04a986a2u64); + lemma_k_square(0x11cb06c04a986a2u64); + assert(reduce_formula(spread(0x11cb06c04a986a2u64)) == 0xb5837f7159f74u64) by (bit_vector); + assert(k_sq_iter(0xfa5ac498d20dd97eu64, 5) == 0xb5837f7159f74u64); + lemma_k_square(0xb5837f7159f74u64); + assert(reduce_formula(spread(0xb5837f7159f74u64)) == 0x551507a7ec9563d7u64) by (bit_vector); + assert(k_sq_iter(0xfa5ac498d20dd97eu64, 6) == 0x551507a7ec9563d7u64); + lemma_k_square(0x551507a7ec9563d7u64); + assert(reduce_formula(spread(0x551507a7ec9563d7u64)) == 0xfefb5bba15c4fcc9u64) by (bit_vector); + assert(k_sq_iter(0xfa5ac498d20dd97eu64, 7) == 0xfefb5bba15c4fcc9u64); + lemma_k_square(0xfefb5bba15c4fcc9u64); + assert(reduce_formula(spread(0xfefb5bba15c4fcc9u64)) == 0x10a51a6f8e1e1acu64) by (bit_vector); + assert(k_sq_iter(0xfa5ac498d20dd97eu64, 8) == 0x10a51a6f8e1e1acu64); +} + +/// Squarings 56 to 64 of `x`. +proof fn lemma_k_frobenius_x_7() + ensures + k_sq_iter(0x10a51a6f8e1e1acu64, 8) == 0x2u64, +{ + assert(k_sq_iter(0x10a51a6f8e1e1acu64, 0) == 0x10a51a6f8e1e1acu64); + lemma_k_square(0x10a51a6f8e1e1acu64); + assert(reduce_formula(spread(0x10a51a6f8e1e1acu64)) == 0x555b52acff1ce98cu64) by (bit_vector); + assert(k_sq_iter(0x10a51a6f8e1e1acu64, 1) == 0x555b52acff1ce98cu64); + lemma_k_square(0x555b52acff1ce98cu64); + assert(reduce_formula(spread(0x555b52acff1ce98cu64)) == 0xfffface6ff2beb3bu64) by (bit_vector); + assert(k_sq_iter(0x10a51a6f8e1e1acu64, 2) == 0xfffface6ff2beb3bu64); + lemma_k_square(0xfffface6ff2beb3bu64); + assert(reduce_formula(spread(0xfffface6ff2beb3bu64)) == 0x55550444ff3218d8u64) by (bit_vector); + assert(k_sq_iter(0x10a51a6f8e1e1acu64, 3) == 0x55550444ff3218d8u64); + lemma_k_square(0x55550444ff3218d8u64); + assert(reduce_formula(spread(0x55550444ff3218d8u64)) == 0xffffafaf00f1e0ebu64) by (bit_vector); + assert(k_sq_iter(0x10a51a6f8e1e1acu64, 4) == 0xffffafaf00f1e0ebu64); + lemma_k_square(0xffffafaf00f1e0ebu64); + assert(reduce_formula(spread(0xffffafaf00f1e0ebu64)) == 0x5500ff01ff03u64) by (bit_vector); + assert(k_sq_iter(0x10a51a6f8e1e1acu64, 5) == 0x5500ff01ff03u64); + lemma_k_square(0x5500ff01ff03u64); + assert(reduce_formula(spread(0x5500ff01ff03u64)) == 0x55550000fffe0005u64) by (bit_vector); + assert(k_sq_iter(0x10a51a6f8e1e1acu64, 6) == 0x55550000fffe0005u64); + lemma_k_square(0x55550000fffe0005u64); + assert(reduce_formula(spread(0x55550000fffe0005u64)) == 0xffffffff0000000au64) by (bit_vector); + assert(k_sq_iter(0x10a51a6f8e1e1acu64, 7) == 0xffffffff0000000au64); + lemma_k_square(0xffffffff0000000au64); + assert(reduce_formula(spread(0xffffffff0000000au64)) == 0x2u64) by (bit_vector); + assert(k_sq_iter(0x10a51a6f8e1e1acu64, 8) == 0x2u64); +} + +/// `x^(2^64) = x`, by 64 concrete squarings. +pub proof fn lemma_k_frobenius_x() + ensures + k_sq_iter(2, 64) == 2, +{ + lemma_k_frobenius_x_0(); + lemma_k_frobenius_x_1(); + lemma_k_frobenius_x_2(); + lemma_k_frobenius_x_3(); + lemma_k_frobenius_x_4(); + lemma_k_frobenius_x_5(); + lemma_k_frobenius_x_6(); + lemma_k_frobenius_x_7(); + lemma_k_sq_iter_add(2, 8, 8); + assert(k_sq_iter(2, 16) == 0x15f0144001a114fu64); + lemma_k_sq_iter_add(2, 16, 8); + assert(k_sq_iter(2, 24) == 0xb6d535c542116f62u64); + lemma_k_sq_iter_add(2, 24, 8); + assert(k_sq_iter(2, 32) == 0x5c62236777028505u64); + lemma_k_sq_iter_add(2, 32, 8); + assert(k_sq_iter(2, 40) == 0x13846a66c22c75beu64); + lemma_k_sq_iter_add(2, 40, 8); + assert(k_sq_iter(2, 48) == 0xfa5ac498d20dd97eu64); + lemma_k_sq_iter_add(2, 48, 8); + assert(k_sq_iter(2, 56) == 0x10a51a6f8e1e1acu64); + lemma_k_sq_iter_add(2, 56, 8); + assert(k_sq_iter(2, 64) == 0x2u64); +} + +/// Fermat in `K`: `a^(2^64) = a` for every `a`. +/// +/// Squaring is GF(2)-linear and multiplicative, so it suffices that it fixes `x`, whose powers +/// `x^i = 1 << i` span `K`. +pub proof fn lemma_k_fermat(a: u64) + ensures + k_sq_iter(a, 64) == a, +{ + lemma_k_fermat_upto(a, 64); +} + +pub proof fn lemma_k_fermat_upto(a: u64, n: nat) + requires + n <= 64, + ensures + k_sq_iter(low(a, n), 64) == low(a, n), + decreases n, +{ + if n == 0 { + assert(low(a, 0) == 0) by { + assert(a & ((1u64 << 0u64) - 1) as u64 == 0) by (bit_vector); + } + lemma_k_sq_iter_zero(64); + } else { + let i = (n - 1) as nat; + lemma_k_fermat_upto(a, i); + lemma_low_step(a, i); + let e: u64 = if bit(a, i) { + 1u64 << (i as u64) + } else { + 0 + }; + lemma_k_sq_iter_xor(low(a, i), e, 64); + if bit(a, i) { + lemma_k_pow_x(i); + lemma_k_sq_iter_k_pow(2, i, 64); + lemma_k_frobenius_x(); + } else { + lemma_k_sq_iter_zero(64); + } + } +} + +// --------------------------------------------------------------------------------------------- +// K is a field: the inverse is an inverse +// --------------------------------------------------------------------------------------------- +/// The only idempotents of `K` are 0 and 1 (Berlekamp: the Frobenius fixes only `GF(2)`, so `M` has a +/// single irreducible factor; with Fermat, `K` is a field). +pub proof fn lemma_k_idempotent(e: u64) + requires + k_mul(e, e) == e, + ensures + e == 0 || e == 1, +{ + lemma_k_square(e); + assert(reduce_formula(spread(e)) == e ==> e == 0 || e == 1) by (bit_vector); +} + +pub proof fn lemma_k_pow_zero(n: nat) + requires + n > 0, + ensures + k_pow(0, n) == 0, +{ + lemma_k_mul_zero(k_pow(0, (n - 1) as nat)); +} + +/// `a^(2^64 - 2)` is the inverse of every nonzero `a`, and zero's image is zero. +pub proof fn lemma_k_inverse(a: u64) + ensures + a != 0 ==> k_mul(a, k_pow(a, (pow2(64) - 2) as nat)) == 1, + a == 0 ==> k_pow(a, (pow2(64) - 2) as nat) == 0, +{ + lemma2_to64(); + let q = pow2(64); + let w = k_pow(a, (q - 2) as nat); + let u = k_pow(a, (q - 1) as nat); + if a == 0 { + lemma_k_pow_zero((q - 2) as nat); + } else { + // u = a^(q-1) = a * w. + assert(u == k_mul(w, a)); + lemma_k_mul_comm(w, a); + // a^q = a by Fermat. + lemma_k_sq_iter_pow(a, 1, 64); + assert(k_pow(a, 1) == a) by { + assert(k_pow(a, 0) == 1); + lemma_k_mul_one(a); + } + lemma_k_fermat(a); + assert(k_pow(a, q) == a); + // u^2 = a^(2q - 2) = a^q a^(q-2) = u, so u is 0 or 1. + lemma_k_pow_add(a, (q - 1) as nat, (q - 1) as nat); + lemma_k_pow_add(a, q, (q - 2) as nat); + assert((q - 1) as nat + (q - 1) as nat == q + (q - 2) as nat); + lemma_k_idempotent(u); + // u = 0 would give a = a^q = u a = 0. + lemma_k_pow_add(a, (q - 1) as nat, 1); + assert((q - 1) as nat + 1 == q); + if u == 0 { + lemma_k_mul_zero(a); + } + } +} + +// --------------------------------------------------------------------------------------------- +// Executable code: the portable paths of `crates/primitives/src/field/gf2_64.rs` +// --------------------------------------------------------------------------------------------- +/// Reduction constant of the base field: `x^64 = x^4 + x^3 + x + 1 = 0x1B`. +pub const R64: u64 = 0x1B; + +/// A GF(2^64) element; bit i = coefficient of x^i. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(transparent)] +pub struct F64(pub u64); + +impl F64 { + /// Degree over GF(2), the number of binary coefficients in one element. + pub const DEGREE: usize = 64; + + pub const ZERO: Self = Self(0); + + pub const ONE: Self = Self(1); + + /// `x`, with `ord(x) = 2^64 - 1`. + pub const G: Self = Self(2); + + #[inline] + pub const fn is_zero(self) -> (r: bool) + ensures + r == (self.0 == 0), + { + self.0 == 0 + } + + /// Squaring, as a bit spread followed by one reduction. + /// + /// Cross terms vanish in characteristic 2, so the square moves bit `i` to bit `2i`. + /// On aarch64 it is the product with itself instead: its PMULL folds stay in the vector register, + /// where [`reduce`] would cross to integer registers and back. + #[inline] + pub fn square(self) -> (r: Self) + ensures + r.0 == k_mul(self.0, self.0), + { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + self * self + } + #[cfg(not(all(target_arch = "aarch64", target_feature = "aes")))] + { + Self(reduce(square_wide(self.0))) + } + } + + /// Multiplicative inverse `x^(2^64 - 2)`, mapping zero to zero. + /// + /// Itoh-Tsujii: with `t_k = x^(2^k - 1)`, the inverse is `t_63^2`. + pub fn inv(self) -> (r: Self) + ensures + r.0 == k_pow(self.0, (pow2(64) - 2) as nat), + self.0 != 0 ==> k_mul(self.0, r.0) == 1, + self.0 == 0 ==> r.0 == 0, + { + proof { + lemma_k_inverse(self.0); + } + // Square `v` a total of `n` times. + let sq = |v: Self, n: u32| -> (r: Self) + ensures + r.0 == k_sq_iter(v.0, n as nat), + { + let ghost v0 = v; + let mut v = v; + for i in 0..n + invariant + v.0 == k_sq_iter(v0.0, i as nat), + { + v = v.square(); + } + v + }; + let ghost a = self.0; + proof { + lemma2_to64(); + lemma2_to64_rest(); + assert(k_pow(a, 1) == a) by { + assert(k_pow(a, 0) == 1); + lemma_k_mul_one(a); + lemma_k_mul_comm(1, a); + } + } + // Each step reads t_k = x^(2^k - 1). + let t1 = self; + let t2 = sq(t1, 1) * t1; + proof { step(a, 1, 1, 1) } + let t3 = sq(t2, 1) * t1; + proof { step(a, 2, 1, 1) } + let t6 = sq(t3, 3) * t3; + proof { step(a, 3, 3, 3) } + let t12 = sq(t6, 6) * t6; + proof { step(a, 6, 6, 6) } + let t24 = sq(t12, 12) * t12; + proof { step(a, 12, 12, 12) } + let t48 = sq(t24, 24) * t24; + proof { step(a, 24, 24, 24) } + let t60 = sq(t48, 12) * t12; + proof { step(a, 48, 12, 12) } + let t63 = sq(t60, 3) * t3; + proof { step(a, 60, 3, 3) } + // (x^(2^63 - 1))^2 = x^(2^64 - 2). + proof { + lemma_k_sq_iter_pow(a, (pow2(63) - 1) as nat, 1); + } + sq(t63, 1) + } +} + +/// One Itoh-Tsujii step: `t_k^(2^n) * t_n = t_(k + n)` with `t_k = a^(2^k - 1)`. +proof fn step(a: u64, k: nat, n: nat, n2: nat) + requires + n == n2, + k + n <= 64, + ensures + k_mul(k_sq_iter(k_pow(a, (pow2(k) - 1) as nat), n), k_pow(a, (pow2(n2) - 1) as nat)) == k_pow( + a, + (pow2(k + n) - 1) as nat, + ), +{ + lemma_k_sq_iter_pow(a, (pow2(k) - 1) as nat, n); + lemma_pow2_pos(k); + lemma_pow2_pos(n); + lemma_pow2_adds(k, n); + let e1 = ((pow2(k) - 1) as nat) * pow2(n); + let e2 = (pow2(n) - 1) as nat; + lemma_k_pow_add(a, e1, e2); + assert(e1 + e2 == (pow2(k + n) - 1) as nat) by (nonlinear_arith) + requires + e1 == ((pow2(k) - 1) as nat) * pow2(n), + e2 == (pow2(n) - 1) as nat, + pow2(k) * pow2(n) == pow2(k + n), + pow2(k) > 0, + pow2(n) > 0, + ; +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::AddSpecImpl for F64 { + open spec fn obeys_add_spec() -> bool { + true + } + + open spec fn add_req(self, rhs: F64) -> bool { + true + } + + open spec fn add_spec(self, rhs: F64) -> F64 { + F64(self.0 ^ rhs.0) + } +} + +impl Add for F64 { + type Output = Self; + + #[inline] + fn add(self, rhs: Self) -> (r: Self) + ensures + r.0 == self.0 ^ rhs.0, + { + Self(self.0 ^ rhs.0) + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::MulSpecImpl for F64 { + open spec fn obeys_mul_spec() -> bool { + true + } + + open spec fn mul_req(self, rhs: F64) -> bool { + true + } + + open spec fn mul_spec(self, rhs: F64) -> F64 { + F64(k_mul(self.0, rhs.0)) + } +} + +impl Mul for F64 { + type Output = Self; + + #[inline] + fn mul(self, rhs: Self) -> (r: Self) + ensures + r.0 == k_mul(self.0, rhs.0), + { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + // SAFETY: aes target feature is enabled at compile time. + unsafe { aarch64::mul_shift_tail(self, rhs) } + } + #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] + { + // SAFETY: pclmulqdq is enabled at compile time. + unsafe { Self(x86_64::mul(self.0, rhs.0)) } + } + #[cfg(not(any( + all(target_arch = "aarch64", target_feature = "aes"), + all(target_arch = "x86_64", target_feature = "pclmulqdq") + )))] + { + Self(reduce(mul_wide(self.0, rhs.0))) + } + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::AddAssignSpecImpl for F64 { + open spec fn obeys_add_assign_spec() -> bool { + true + } + + open spec fn add_assign_req(&self, rhs: F64) -> bool { + true + } + + open spec fn add_assign_spec(&self, rhs: F64) -> &F64 { + &F64(self.0 ^ rhs.0) + } +} + +impl AddAssign for F64 { + #[inline] + fn add_assign(&mut self, rhs: Self) + ensures + final(self).0 == old(self).0 ^ rhs.0, + { + self.0 ^= rhs.0; + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::MulAssignSpecImpl for F64 { + open spec fn obeys_mul_assign_spec() -> bool { + true + } + + open spec fn mul_assign_req(&self, rhs: F64) -> bool { + true + } + + open spec fn mul_assign_spec(&self, rhs: F64) -> &F64 { + &F64(k_mul(self.0, rhs.0)) + } +} + +impl MulAssign for F64 { + #[inline] + fn mul_assign(&mut self, rhs: Self) + ensures + final(self).0 == k_mul(old(self).0, rhs.0), + { + *self = *self * rhs; + } +} + +/// The carry-less product of two 64-bit polynomials, as a 128-bit polynomial. +#[inline] +pub fn mul_wide(a: u64, b: u64) -> (r: u128) + ensures + r == clmul(a, b as u128), +{ + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + proof { + broadcast use crate::intrinsics::aarch64::axiom_u64x2_as_u128; + let p = clmul(a, b as u128); + assert(((p as u64) as u128) | ((((p >> 64u128) as u64) as u128) << 64u128) == p) by (bit_vector); + } + // SAFETY: aes is enabled at compile time; the reinterpret is between 128-bit values. + unsafe { core::mem::transmute::(aarch64::pmull(a, b)) } + } + #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] + { + // SAFETY: pclmulqdq is enabled at compile time. + unsafe { x86_64::clmul(a, b) } + } + #[cfg(not(any( + all(target_arch = "aarch64", target_feature = "aes"), + all(target_arch = "x86_64", target_feature = "pclmulqdq") + )))] + { + software::clmul(a, b) + } +} + +/// The carry-less square of a 64-bit polynomial: bit `i` moves to bit `2i`. +#[inline] +pub fn square_wide(a: u64) -> (r: u128) + ensures + r == clmul(a, a as u128), +{ + #[cfg(all(target_arch = "x86_64", target_feature = "bmi2"))] + { + // SAFETY: bmi2 is enabled at compile time. + unsafe { x86_64::spread(a) } + } + #[cfg(not(all(target_arch = "x86_64", target_feature = "bmi2")))] + { + mul_wide(a, a) + } +} + +/// Reduce a 128-bit carry-less product modulo `x^64 + x^4 + x^3 + x + 1`. +#[inline] +pub const fn reduce(p: u128) -> (r: u64) + ensures + r == k_mod(p), + is_remainder(p, r), +{ + proof { + lemma_k_mod(p); + } + // Split the product into its low and high words. + let (lo, hi) = (p as u64, (p >> 64) as u64); + // The bits of `hi * 0x1B` shifted past x^63, which wrap around as `spill * 0x1B`. + let spill = (hi >> 63) ^ (hi >> 61) ^ (hi >> 60); + // Fold `hi` and `spill` together: both are multiplied by the same constant. + let v = hi ^ spill; + lo ^ v ^ (v << 1) ^ (v << 3) ^ (v << 4) +} + +pub mod software { + use super::*; + + /// Portable 64x64 carry-less product, used by fallback paths and as the reference. + pub const fn clmul(a: u64, b: u64) -> (r: u128) + ensures + r == crate::clmul::clmul(a, b as u128), + { + let mut acc = 0u128; + let mut i = 0; + // Schoolbook: XOR in `b * x^i` for every set bit `i` of `a`. + while i < 64 + invariant + 0 <= i <= 64, + acc == clmul_upto(a, b as u128, i as nat), + decreases 64 - i, + { + proof { + let s = i as u64; + let bb = b as u128; + assert(((a >> s) & 1 != 0) == ((a >> s) & 1 == 1)) by (bit_vector); + assert(acc ^ 0 == acc) by (bit_vector); + } + if (a >> i) & 1 != 0 { + acc ^= (b as u128) << i; + } + i += 1; + } + acc + } +} + +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +pub mod aarch64 { + use super::{F64, R64}; + use crate::neon::xor3_u64; + use core::arch::aarch64::*; + use core::mem::transmute; + #[cfg(verus_keep_ghost)] + use super::{k_mul, k_mod, lemma_clmul_fold_reduction}; + #[cfg(verus_keep_ghost)] + use crate::intrinsics::aarch64::*; + #[cfg(verus_keep_ghost)] + use crate::intrinsics::transmuted; + use vstd::prelude::*; + + /// 64x64 carry-less product as a 128-bit NEON vector. + /// + /// # Safety + /// Requires the `aes` target feature (compiles to PMULL); only call where + /// `aes` is statically enabled or has been runtime-detected. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn pmull(a: u64, b: u64) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == crate::clmul::clmul(a, b as u128) as u64, + u64x2(r)[1] == (crate::clmul::clmul(a, b as u128) >> 64u128) as u64, + { + proof { + broadcast use axiom_u128_as_u64x2; + } + // SAFETY: u128 and uint64x2_t are both 128-bit values. + unsafe { transmute::(vmull_p64(a, b)) } + } + + /// Carry-less product of the two *high* lanes: PMULL2 on the register + /// pair, no lane extraction (the lane-crossing-free way to fold a + /// product's high half). + /// + /// # Safety + /// Requires the `aes` target feature; see [`pmull`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn pmull_hi(a: uint64x2_t, b: uint64x2_t) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == crate::clmul::clmul(u64x2(a)[1], u64x2(b)[1] as u128) as u64, + u64x2(r)[1] == (crate::clmul::clmul(u64x2(a)[1], u64x2(b)[1] as u128) >> 64u128) as u64, + { + proof { + broadcast use axiom_u128_as_u64x2, axiom_u64x2_as_p64x2; + } + // SAFETY: bit-level reinterprets between 128-bit vector types. + unsafe { + transmute::(vmull_high_p64( + transmute::(a), + transmute::(b), + )) + } + } + + /// The two lanes of a register as one 128-bit polynomial, lane 0 the low word. + pub open spec fn u64x2_u128(v: uint64x2_t) -> u128 { + (u64x2(v)[0] as u128) | ((u64x2(v)[1] as u128) << 64u128) + } + + proof fn lemma_u64x2_u128(v: uint64x2_t) + ensures + u64x2(v)[0] == u64x2_u128(v) as u64, + u64x2(v)[1] == (u64x2_u128(v) >> 64u128) as u64, + { + let (a, b) = (u64x2(v)[0], u64x2(v)[1]); + assert(a == (((a as u128) | ((b as u128) << 64u128)) as u64) && b == ((((a as u128) | ((b as u128) << 64u128)) + >> 64u128) as u64)) by (bit_vector); + } + + /// The reduction of one product in a register: `p ^ t ^ u` with `t = hi(p) * 0x1B`, `u = hi(t) * 0x1B`, + /// whose low lane is `k_mod` of the product, given `p`'s lanes are the product's words. + proof fn lemma_fold_lanes(p: uint64x2_t, r: uint64x2_t, t: uint64x2_t, u: uint64x2_t, pp: u128) + requires + u64x2(p)[0] == pp as u64, + u64x2(p)[1] == (pp >> 64u128) as u64, + u64x2(r)[1] == R64, + u64x2(t)[0] == crate::clmul::clmul(u64x2(p)[1], u64x2(r)[1] as u128) as u64, + u64x2(t)[1] == (crate::clmul::clmul(u64x2(p)[1], u64x2(r)[1] as u128) >> 64u128) as u64, + u64x2(u)[0] == crate::clmul::clmul(u64x2(t)[1], u64x2(r)[1] as u128) as u64, + ensures + u64x2(p)[0] ^ u64x2(t)[0] ^ u64x2(u)[0] == k_mod(pp), + { + lemma_clmul_fold_reduction(pp); + let tt = crate::clmul::clmul((pp >> 64u128) as u64, 0x1Bu128); + let uu = crate::clmul::clmul((tt >> 64u128) as u64, 0x1Bu128); + assert((pp ^ tt ^ uu) as u64 == (pp as u64) ^ (tt as u64) ^ (uu as u64)) by (bit_vector); + } + + /// Reduce two 128-bit carry-less products into one lane pair: `[reduce(p0), reduce(p1)]`. + /// + /// Seven instructions for the pair. + /// + /// Rewritten for Verus: the empty `asm!` block that hides the lanes from LLVM is in [`hide_lanes`], whose + /// body is that block and whose specification is that it leaves both registers unchanged. + /// + /// # Safety + /// Requires the `aes` target feature; see [`pmull`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn reduce_pair_pmull4(p0: uint64x2_t, p1: uint64x2_t) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == k_mod(u64x2_u128(p0)), + u64x2(r)[1] == k_mod(u64x2_u128(p1)), + { + // SAFETY: function carries the aes target feature. + unsafe { + let r = vdupq_n_u64(R64); + let (t0, t1) = (pmull_hi(p0, r), pmull_hi(p1, r)); + let (u0, u1) = (pmull_hi(t0, r), pmull_hi(t1, r)); + let (e0, e1) = (xor3_u64(p0, t0, u0), xor3_u64(p1, t1, u1)); + // Why: only lane 0 of each sum survives the zip. + // LLVM would gather the six low lanes with three inserts and XOR once: one instruction more. + // An empty asm block hides the lanes from it, and emits nothing. + let (e0, e1) = hide_lanes(e0, e1); + proof { + lemma_u64x2_u128(p0); + lemma_u64x2_u128(p1); + lemma_fold_lanes(p0, r, t0, u0, u64x2_u128(p0)); + lemma_fold_lanes(p1, r, t1, u1, u64x2_u128(p1)); + } + vzip1q_u64(e0, e1) + } + } + + /// The empty `asm!` block of [`reduce_pair_pmull4`]: it emits nothing, so both registers keep their value. + #[verifier::external_body] + #[inline(always)] + fn hide_lanes(mut e0: uint64x2_t, mut e1: uint64x2_t) -> (r: (uint64x2_t, uint64x2_t)) + ensures + r == (e0, e1), + { + // SAFETY: an empty block, which reads and writes nothing. + unsafe { + core::arch::asm!( + "/* {0:v} {1:v} */", + inout(vreg) e0, + inout(vreg) e1, + options(pure, nomem, nostack, preserves_flags) + ); + } + (e0, e1) + } + + /// The default `Mul` kernel on aarch64. 3-PMULL multiply: product, then two + /// PMULL-by-0x1B folds, the second taking the first's <=4-bit overflow + /// exactly (`ov*0x1B` fits in 8 bits). + /// + /// # Safety + /// Requires the `aes` target feature; see [`pmull`]. + #[inline] + #[target_feature(enable = "aes")] + pub unsafe fn mul_shift_tail(a: F64, b: F64) -> (r: F64) + ensures + r.0 == k_mul(a.0, b.0), + { + // SAFETY: function carries the aes target feature. + unsafe { + let r = vdupq_n_u64(R64); + let p = pmull(a.0, b.0); + let t = pmull_hi(p, r); + let u = pmull_hi(t, r); + proof { + lemma_fold_lanes(p, r, t, u, crate::clmul::clmul(a.0, b.0 as u128)); + } + F64(vgetq_lane_u64::<0>(veorq_u64(veorq_u64(p, t), u))) + } + } +} + +/// The x86-64 products. +#[cfg(target_arch = "x86_64")] +pub mod x86_64 { + use super::R64; + use core::arch::x86_64::*; + #[cfg(verus_keep_ghost)] + use super::{k_mul, lemma_clmul_fold_reduction, lemma_clmul_square_upto, lemma_k_mod}; + #[cfg(verus_keep_ghost)] + use crate::clmul::low; + use vstd::prelude::*; + #[cfg(verus_keep_ghost)] + use crate::intrinsics::x86::*; + #[cfg(verus_keep_ghost)] + use crate::intrinsics::transmuted; + #[cfg(verus_keep_ghost)] + use crate::clmul::lemma_u64_bits_eq; + + /// One field product: the carry-less product, then two carry-less folds of its high word by `0x1B`. + /// + /// # Safety + /// + /// Requires the `pclmulqdq` target feature. + #[inline] + #[target_feature(enable = "pclmulqdq")] + pub unsafe fn mul(a: u64, b: u64) -> (r: u64) + ensures + r == k_mul(a, b), + { + proof { + lemma_clmul_sel(); + lemma_i64_round_trip(a); + lemma_i64_round_trip(b); + lemma_i64_round_trip(R64); + } + let r = _mm_cvtsi64_si128(R64 as i64); + let p = _mm_clmulepi64_si128::<0x00>(_mm_cvtsi64_si128(a as i64), _mm_cvtsi64_si128(b as i64)); + // hi * 0x1B, at most 68 bits. + let t = _mm_clmulepi64_si128::<0x01>(p, r); + // Its spill past x^63, times 0x1B: at most 8 bits. + let u = _mm_clmulepi64_si128::<0x01>(t, r); + proof { + lemma_clmul_words_split(m128(p), a, b); + lemma_clmul_words_split(m128(t), m128(p)[1], R64); + lemma_clmul_words_split(m128(u), m128(t)[1], R64); + let pp = crate::clmul::clmul(a, b as u128); + lemma_clmul_fold_reduction(pp); + let tt = crate::clmul::clmul((pp >> 64u128) as u64, 0x1Bu128); + let uu = crate::clmul::clmul((tt >> 64u128) as u64, 0x1Bu128); + assert((pp ^ tt ^ uu) as u64 == (pp as u64) ^ (tt as u64) ^ (uu as u64)) by (bit_vector); + } + _mm_cvtsi128_si64(_mm_xor_si128(_mm_xor_si128(p, t), u)) as u64 + } + + /// 64x64 carry-less product. + /// + /// # Safety + /// + /// Requires the `pclmulqdq` target feature. + #[inline] + #[target_feature(enable = "pclmulqdq")] + pub unsafe fn clmul(a: u64, b: u64) -> (r: u128) + ensures + r == crate::clmul::clmul(a, b as u128), + { + proof { + lemma_i64_round_trip(a); + lemma_i64_round_trip(b); + } + // Move both operands into the low qword of a vector register. + let (a, b) = (_mm_cvtsi64_si128(a as i64), _mm_cvtsi64_si128(b as i64)); + proof { + let (x, y) = (m128(a)[0], m128(b)[0]); + lemma_clmul_sel(); + let pp = crate::clmul::clmul(x, y as u128); + assert(((pp as u64) as u128) | ((((pp >> 64u128) as u64) as u128) << 64u128) == pp) by (bit_vector); + broadcast use axiom_m128_as_u128; + } + // SAFETY: both types are 128 bits wide with no invalid bit patterns. + unsafe { core::mem::transmute::<__m128i, u128>(_mm_clmulepi64_si128::<0x00>(a, b)) } + } + + /// Carry-less square by bit deposit: `pdep` spreads each 32-bit half onto the even bits of one word. + /// + /// # Safety + /// + /// Requires the `bmi2` target feature. + #[inline] + #[target_feature(enable = "bmi2")] + pub unsafe fn spread(a: u64) -> (r: u128) + ensures + r == crate::clmul::clmul(a, a as u128), + { + // Bits 0, 2, 4, ...: where a square puts the input bits. + const EVEN: u64 = 0x5555_5555_5555_5555; + // Low half of `a` into the low word, high half into the high word. + let lo = _pdep_u64(a, EVEN); + let hi = _pdep_u64(a >> 32, EVEN); + proof { + lemma_pdep_even(a, lo); + lemma_pdep_even(a >> 32, hi); + lemma_spread_halves(a, lo, hi); + assert(low(a, 64) == a); + lemma_clmul_square_upto(a, 64); + } + lo as u128 | (hi as u128) << 64 + } + + // ----------------------------------------------------------------------------------------- + // Proof helpers + // ----------------------------------------------------------------------------------------- + pub proof fn lemma_i64_round_trip(a: u64) + ensures + (a as i64) as u64 == a, + { + assert((a as i64) as u64 == a) by (bit_vector); + } + + /// PCLMULQDQ's two result words are the low and high words of the carry-less product. + pub proof fn lemma_clmul_words_split(w: [u64; 2], a: u64, b: u64) + requires + w[0] == clmul_words(a, b)[0], + w[1] == clmul_words(a, b)[1], + ensures + w[0] == crate::clmul::clmul(a, b as u128) as u64, + w[1] == (crate::clmul::clmul(a, b as u128) >> 64u128) as u64, + { + } + + /// The set bits of `0x5555...`: the even positions, and so `ones_below(EVEN, j) = ceil(j / 2)`. + pub proof fn lemma_ones_below_even(j: nat) + requires + j <= 64, + ensures + ones_below(0x5555_5555_5555_5555u64, j) == (j + 1) / 2, + decreases j, + { + if j > 0 { + lemma_ones_below_even((j - 1) as nat); + let s = (j - 1) as u64; + assert(s < 64 ==> (((0x5555_5555_5555_5555u64 >> s) & 1 == 1) == (s % 2 == 0))) by (bit_vector); + } + } + + /// Bit `j` of `pdep(a, EVEN)` is bit `j / 2` of `a` for even `j`, zero for odd `j`. + pub proof fn lemma_pdep_even(a: u64, d: u64) + requires + forall|j: nat| j < 64 ==> #[trigger] crate::clmul::bit(d, j) == pdep_bit(a, 0x5555_5555_5555_5555u64, j), + ensures + d == spread_word(a), + { + assert forall|j: nat| j < 64 implies #[trigger] crate::clmul::bit(d, j) == crate::clmul::bit(spread_word(a), j) by { + lemma_ones_below_even(j); + let s = j as u64; + assert(s < 64 ==> (((0x5555_5555_5555_5555u64 >> s) & 1 == 1) == (s % 2 == 0))) by (bit_vector); + assert(s < 64 ==> (((spread_word(a) >> s) & 1 == 1) == (s % 2 == 0 && (a >> ((s + 1) / 2)) & 1 == 1))) + by (bit_vector); + assert((s + 1) / 2 == ((j + 1) / 2) as u64); + } + lemma_u64_bits_eq(d, spread_word(a)); + } + + /// The low 32 bits of `a` spread onto the even bits of a word. + pub open spec fn spread_word(a: u64) -> u64 { + let x = a & 0xFFFF_FFFF; + let x = (x | (x << 16u64)) & 0x0000_FFFF_0000_FFFFu64; + let x = (x | (x << 8u64)) & 0x00FF_00FF_00FF_00FFu64; + let x = (x | (x << 4u64)) & 0x0F0F_0F0F_0F0F_0F0Fu64; + let x = (x | (x << 2u64)) & 0x3333_3333_3333_3333u64; + (x | (x << 1u64)) & 0x5555_5555_5555_5555u64 + } + + /// The two spread halves make the spread of the whole word. + pub proof fn lemma_spread_halves(a: u64, lo: u64, hi: u64) + requires + lo == spread_word(a), + hi == spread_word(a >> 32u64), + ensures + lo as u128 | (hi as u128) << 64u128 == super::spread(a), + { + assert(spread_word(a) as u128 | (spread_word(a >> 32u64) as u128) << 64u128 == super::spread(a)) by (bit_vector); + } +} + +// --------------------------------------------------------------------------------------------- +// The scalar SIMD products: their reduction, given the semantics of the carry-less multiply +// --------------------------------------------------------------------------------------------- +/// The reduction of `x86_64::mul` (PCLMULQDQ) and `aarch64::mul_shift_tail` / `reduce_pair_pmull4` (PMULL): +/// +/// ```text +/// t = hi(p) * 0x1B, u = hi(t) * 0x1B, result = lo(p ^ t ^ u) +/// ``` +/// +/// equals [`reduce`], so with the instruction computing [`clmul`] those kernels compute [`k_mul`]. +pub proof fn lemma_clmul_fold_reduction(p: u128) + ensures + ({ + let t = clmul((p >> 64u128) as u64, 0x1Bu128); + let u = clmul((t >> 64u128) as u64, 0x1Bu128); + (p ^ t ^ u) as u64 == k_mod(p) + }), +{ + let t = clmul((p >> 64u128) as u64, 0x1Bu128); + lemma_clmul_r64((p >> 64u128) as u64); + lemma_clmul_r64((t >> 64u128) as u64); + lemma_k_mod(p); + assert({ + let h = ((p >> 64u128) as u64) as u128; + let t = h ^ (h << 1u128) ^ (h << 3u128) ^ (h << 4u128); + let h2 = ((t >> 64u128) as u64) as u128; + let u = h2 ^ (h2 << 1u128) ^ (h2 << 3u128) ^ (h2 << 4u128); + (p ^ t ^ u) as u64 == reduce_formula(p) + }) by (bit_vector); +} + +} // verus! diff --git a/verification/verus/src/gf2_64x3.rs b/verification/verus/src/gf2_64x3.rs new file mode 100644 index 0000000000..64b1b3f6cd --- /dev/null +++ b/verification/verus/src/gf2_64x3.rs @@ -0,0 +1,1527 @@ +//! The extension field `E = K[y] / (y^3 + y + 1)` over the base field `K = GF(2^64)`. +//! +//! The executable functions are the portable paths of `crates/primitives/src/field/gf2_64x3.rs` and +//! `gf2_64x3/software.rs`, and the aarch64 kernels of `gf2_64x3/aarch64.rs` ([`aarch64`]), copied with the +//! same bodies where Verus accepts them; `tests/equivalence.rs` checks the two agree. +//! +//! Specification: an element is `c0 + c1 y + c2 y^2` with `c_i` in `K`. The product [`e_mul`] is the +//! product of polynomials in `y` over `K` (coefficients by [`k_mul`]), folded by `y^3 = y + 1` and +//! `y^4 = y^2 + y`. +use crate::clmul::*; +use crate::gf2_64::*; +use core::ops::{Add, AddAssign, BitXor, BitXorAssign, Mul, MulAssign}; +use vstd::arithmetic::power2::*; +use vstd::prelude::*; + +/// aarch64 kernels. +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +pub mod aarch64; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// The product in `E`. +/// +/// ```text +/// (a0 + a1 y + a2 y^2)(b0 + b1 y + b2 y^2) = d0 + d1 y + d2 y^2 + d3 y^3 + d4 y^4, +/// d_k = sum_{i + j = k} a_i b_j, and y^3 = y + 1, y^4 = y^2 + y. +/// ``` +pub open spec fn e_mul(a: F192, b: F192) -> F192 { + let d0 = k_mul(a.c0, b.c0); + let d1 = k_mul(a.c0, b.c1) ^ k_mul(a.c1, b.c0); + let d2 = k_mul(a.c0, b.c2) ^ k_mul(a.c1, b.c1) ^ k_mul(a.c2, b.c0); + let d3 = k_mul(a.c1, b.c2) ^ k_mul(a.c2, b.c1); + let d4 = k_mul(a.c2, b.c2); + F192 { c0: d0 ^ d3, c1: d1 ^ d3 ^ d4, c2: d2 ^ d4 } +} + +/// The sum in `E`, coefficient-wise in `K`. +pub open spec fn e_add(a: F192, b: F192) -> F192 { + F192 { c0: a.c0 ^ b.c0, c1: a.c1 ^ b.c1, c2: a.c2 ^ b.c2 } +} + +/// The embedding of `K` in `E`. +pub open spec fn e_from_k(k: u64) -> F192 { + F192 { c0: k, c1: 0, c2: 0 } +} + +/// A 128-bit coefficient from its `[low, high]` words. +pub open spec fn wide(w: [u64; 2]) -> u128 { + ((w[1] as u128) << 64u128) | (w[0] as u128) +} + +/// The element an unreduced value stands for: each 128-bit coefficient reduced modulo `M`. +pub open spec fn e_value(u: F192Unreduced) -> F192 { + F192 { c0: k_mod(wide(u.coeffs[0])), c1: k_mod(wide(u.coeffs[1])), c2: k_mod(wide(u.coeffs[2])) } +} + +/// Coefficient-wise XOR of unreduced values. +pub open spec fn u_xor(u: F192Unreduced, v: F192Unreduced) -> F192Unreduced { + F192Unreduced { + coeffs: [ + [u.coeffs[0][0] ^ v.coeffs[0][0], u.coeffs[0][1] ^ v.coeffs[0][1]], + [u.coeffs[1][0] ^ v.coeffs[1][0], u.coeffs[1][1] ^ v.coeffs[1][1]], + [u.coeffs[2][0] ^ v.coeffs[2][0], u.coeffs[2][1] ^ v.coeffs[2][1]], + ], + } +} + +/// The unreduced value with the given 128-bit coefficients. +pub open spec fn u_from_wide(c0: u128, c1: u128, c2: u128) -> F192Unreduced { + F192Unreduced { + coeffs: [[c0 as u64, (c0 >> 64u128) as u64], [c1 as u64, (c1 >> 64u128) as u64], [ + c2 as u64, + (c2 >> 64u128) as u64, + ]], + } +} + +pub proof fn lemma_wide_split(c: u128) + ensures + wide([c as u64, (c >> 64u128) as u64]) == c, +{ + assert(((((c >> 64u128) as u64) as u128) << 64u128) | ((c as u64) as u128) == c) by (bit_vector); +} + +pub proof fn lemma_u_from_wide(c0: u128, c1: u128, c2: u128) + ensures + e_value(u_from_wide(c0, c1, c2)) == (F192 { c0: k_mod(c0), c1: k_mod(c1), c2: k_mod(c2) }), +{ + lemma_wide_split(c0); + lemma_wide_split(c1); + lemma_wide_split(c2); +} + +/// Lazy reduction: reducing a XOR of unreduced values is the sum of their reductions. +pub proof fn lemma_e_value_xor(u: F192Unreduced, v: F192Unreduced) + ensures + e_value(u_xor(u, v)) == e_add(e_value(u), e_value(v)), +{ + let w = u_xor(u, v); + assert forall|k: int| 0 <= k < 3 implies #[trigger] wide(w.coeffs[k]) == wide(u.coeffs[k]) ^ wide(v.coeffs[k]) by { + let (a0, a1, b0, b1) = (u.coeffs[k][0], u.coeffs[k][1], v.coeffs[k][0], v.coeffs[k][1]); + assert((((a1 ^ b1) as u128) << 64u128) | ((a0 ^ b0) as u128) == (((a1 as u128) << 64u128) | ( + a0 as u128)) ^ (((b1 as u128) << 64u128) | (b0 as u128))) by (bit_vector); + } + lemma_k_mod_xor(wide(u.coeffs[0]), wide(v.coeffs[0])); + lemma_k_mod_xor(wide(u.coeffs[1]), wide(v.coeffs[1])); + lemma_k_mod_xor(wide(u.coeffs[2]), wide(v.coeffs[2])); +} + +/// XOR of a sequence of unreduced values. +pub open spec fn u_sum(us: Seq) -> F192Unreduced + decreases us.len(), +{ + if us.len() == 0 { + F192Unreduced::ZERO + } else { + u_xor(u_sum(us.drop_last()), us.last()) + } +} + +/// Sum of a sequence of elements. +pub open spec fn e_sum(es: Seq) -> F192 + decreases es.len(), +{ + if es.len() == 0 { + F192::ZERO + } else { + e_add(e_sum(es.drop_last()), es.last()) + } +} + +/// The identity the accumulating kernels rely on: reducing an XOR-accumulated sum of unreduced +/// values once equals summing the reduced values. +pub proof fn lemma_lazy_reduction(us: Seq) + ensures + e_value(u_sum(us)) == e_sum(us.map_values(|u: F192Unreduced| e_value(u))), + decreases us.len(), +{ + let es = us.map_values(|u: F192Unreduced| e_value(u)); + if us.len() == 0 { + lemma_u_zero(); + } else { + lemma_lazy_reduction(us.drop_last()); + assert(us.drop_last().map_values(|u: F192Unreduced| e_value(u)) =~= es.drop_last()); + lemma_e_value_xor(u_sum(us.drop_last()), us.last()); + } +} + +pub proof fn lemma_u_zero() + ensures + e_value(F192Unreduced::ZERO) == F192::ZERO, +{ + lemma_u_from_wide(0, 0, 0); + assert(u_from_wide(0, 0, 0) == F192Unreduced::ZERO) by { + assert((0u128 as u64) == 0 && ((0u128 >> 64u128) as u64) == 0) by (bit_vector); + } + assert(0u128 >> 64u128 == 0) by (bit_vector); + lemma_k_mod_small(0); +} + +// --------------------------------------------------------------------------------------------- +// The software product, unrolled: what the 3 x 3 loop accumulates +// --------------------------------------------------------------------------------------------- +/// The accumulator `e` after the first `n` steps of the schoolbook loop (row-major over `i, j < 3`). +pub open spec fn sched(a: Seq, b: Seq, n: nat) -> Seq + decreases n, +{ + if n == 0 { + seq![0u128, 0u128, 0u128, 0u128, 0u128] + } else { + let s = sched(a, b, (n - 1) as nat); + let (i, j) = (((n - 1) / 3) as int, ((n - 1) % 3) as int); + s.update(i + j, s[i + j] ^ clmul(a[i], b[j] as u128)) + } +} + +/// `p_ij = a_i * b_j` as polynomials. +pub open spec fn cp(a: Seq, b: Seq, i: int, j: int) -> u128 { + clmul(a[i], b[j] as u128) +} + +pub proof fn lemma_sched_9(a: Seq, b: Seq) + requires + a.len() == 3, + b.len() == 3, + ensures + sched(a, b, 9) == seq![ + 0 ^ cp(a, b, 0, 0), + 0 ^ cp(a, b, 0, 1) ^ cp(a, b, 1, 0), + 0 ^ cp(a, b, 0, 2) ^ cp(a, b, 1, 1) ^ cp(a, b, 2, 0), + 0 ^ cp(a, b, 1, 2) ^ cp(a, b, 2, 1), + 0 ^ cp(a, b, 2, 2), + ], +{ + reveal_with_fuel(sched, 10); + assert(sched(a, b, 9) =~= seq![ + 0 ^ cp(a, b, 0, 0), + 0 ^ cp(a, b, 0, 1) ^ cp(a, b, 1, 0), + 0 ^ cp(a, b, 0, 2) ^ cp(a, b, 1, 1) ^ cp(a, b, 2, 0), + 0 ^ cp(a, b, 1, 2) ^ cp(a, b, 2, 1), + 0 ^ cp(a, b, 2, 2), + ]); +} + +/// The y-folded coefficients reduce to the product in `E`. +pub proof fn lemma_mul_unreduced_value(a: F192, b: F192) + ensures + ({ + let (sa, sb) = (seq![a.c0, a.c1, a.c2], seq![b.c0, b.c1, b.c2]); + let e = sched(sa, sb, 9); + e_value(u_from_wide(e[0] ^ e[3], e[1] ^ e[3] ^ e[4], e[2] ^ e[4])) == e_mul(a, b) + }), +{ + let (sa, sb) = (seq![a.c0, a.c1, a.c2], seq![b.c0, b.c1, b.c2]); + lemma_sched_9(sa, sb); + let e = sched(sa, sb, 9); + lemma_u_from_wide(e[0] ^ e[3], e[1] ^ e[3] ^ e[4], e[2] ^ e[4]); + assert forall|x: u128| #![auto] 0 ^ x == x by { + assert(0 ^ x == x) by (bit_vector); + } + let p = |i: int, j: int| cp(sa, sb, i, j); + // Reduction is linear, so it distributes over every XOR. + lemma_k_mod_xor(e[0], e[3]); + lemma_k_mod_xor(e[1] ^ e[3], e[4]); + lemma_k_mod_xor(e[1], e[3]); + lemma_k_mod_xor(e[2], e[4]); + lemma_k_mod_xor(p(0, 1), p(1, 0)); + lemma_k_mod_xor(p(0, 2) ^ p(1, 1), p(2, 0)); + lemma_k_mod_xor(p(0, 2), p(1, 1)); + lemma_k_mod_xor(p(1, 2), p(2, 1)); +} + +// --------------------------------------------------------------------------------------------- +// E is a commutative ring, and its Frobenius +// --------------------------------------------------------------------------------------------- +/// `a^n` in `E`. +pub open spec fn e_pow(a: F192, n: nat) -> F192 + decreases n, +{ + if n == 0 { + F192::ONE + } else { + e_mul(e_pow(a, (n - 1) as nat), a) + } +} + +/// `a` squared `n` times in `E`. +pub open spec fn e_sq_iter(a: F192, n: nat) -> F192 + decreases n, +{ + if n == 0 { + a + } else { + let v = e_sq_iter(a, (n - 1) as nat); + e_mul(v, v) + } +} + +/// The Frobenius shuffle `c0 + c2 y + (c1 + c2) y^2`. +pub open spec fn e_frobenius(a: F192) -> F192 { + F192 { c0: a.c0, c1: a.c2, c2: a.c1 ^ a.c2 } +} + +pub proof fn lemma_e_mul_one(a: F192) + ensures + e_mul(a, F192::ONE) == a, + e_mul(F192::ONE, a) == a, +{ + lemma_k_mul_one(a.c0); + lemma_k_mul_one(a.c1); + lemma_k_mul_one(a.c2); + lemma_k_mul_zero(a.c0); + lemma_k_mul_zero(a.c1); + lemma_k_mul_zero(a.c2); + let (x0, x1, x2) = (a.c0, a.c1, a.c2); + assert(x0 ^ (0u64 ^ 0u64) == x0 && (0u64 ^ x1) ^ (0u64 ^ 0u64) ^ 0u64 == x1 && (0u64 ^ 0u64 ^ x2) ^ 0u64 + == x2) by (bit_vector); + assert(x0 ^ (0u64 ^ 0u64) == x0 && (0u64 ^ x1) ^ (0u64 ^ 0u64) ^ 0u64 == x1 && (0u64 ^ 0u64 ^ x2) ^ 0u64 + == x2) by (bit_vector); + assert(x0 ^ (0u64 ^ 0u64) == x0 && (x1 ^ 0u64) ^ (0u64 ^ 0u64) ^ 0u64 == x1 && (x2 ^ 0u64 ^ 0u64) ^ 0u64 + == x2) by (bit_vector); +} + +pub proof fn lemma_e_mul_comm(a: F192, b: F192) + ensures + e_mul(a, b) == e_mul(b, a), +{ + lemma_k_mul_comm(a.c0, b.c0); + lemma_k_mul_comm(a.c0, b.c1); + lemma_k_mul_comm(a.c0, b.c2); + lemma_k_mul_comm(a.c1, b.c0); + lemma_k_mul_comm(a.c1, b.c1); + lemma_k_mul_comm(a.c1, b.c2); + lemma_k_mul_comm(a.c2, b.c0); + lemma_k_mul_comm(a.c2, b.c1); + lemma_k_mul_comm(a.c2, b.c2); + let (p00, p01, p02, p10, p11, p12, p20, p21, p22) = ( + k_mul(a.c0, b.c0), + k_mul(a.c0, b.c1), + k_mul(a.c0, b.c2), + k_mul(a.c1, b.c0), + k_mul(a.c1, b.c1), + k_mul(a.c1, b.c2), + k_mul(a.c2, b.c0), + k_mul(a.c2, b.c1), + k_mul(a.c2, b.c2), + ); + assert(p00 ^ (p12 ^ p21) == p00 ^ (p21 ^ p12) && (p01 ^ p10) ^ (p12 ^ p21) ^ p22 == (p10 ^ p01) ^ (p21 ^ p12) + ^ p22 && (p02 ^ p11 ^ p20) ^ p22 == (p20 ^ p11 ^ p02) ^ p22) by (bit_vector); +} + +/// Associativity, by expanding both sides into the 27 triple products of coefficients. +pub proof fn lemma_e_mul_assoc(a: F192, b: F192, c: F192) + ensures + e_mul(e_mul(a, b), c) == e_mul(a, e_mul(b, c)), +{ + let t000 = k_mul(k_mul(a.c0, b.c0), c.c0); + lemma_k_mul_assoc(a.c0, b.c0, c.c0); + let t001 = k_mul(k_mul(a.c0, b.c0), c.c1); + lemma_k_mul_assoc(a.c0, b.c0, c.c1); + let t002 = k_mul(k_mul(a.c0, b.c0), c.c2); + lemma_k_mul_assoc(a.c0, b.c0, c.c2); + let t010 = k_mul(k_mul(a.c0, b.c1), c.c0); + lemma_k_mul_assoc(a.c0, b.c1, c.c0); + let t011 = k_mul(k_mul(a.c0, b.c1), c.c1); + lemma_k_mul_assoc(a.c0, b.c1, c.c1); + let t012 = k_mul(k_mul(a.c0, b.c1), c.c2); + lemma_k_mul_assoc(a.c0, b.c1, c.c2); + let t020 = k_mul(k_mul(a.c0, b.c2), c.c0); + lemma_k_mul_assoc(a.c0, b.c2, c.c0); + let t021 = k_mul(k_mul(a.c0, b.c2), c.c1); + lemma_k_mul_assoc(a.c0, b.c2, c.c1); + let t022 = k_mul(k_mul(a.c0, b.c2), c.c2); + lemma_k_mul_assoc(a.c0, b.c2, c.c2); + let t100 = k_mul(k_mul(a.c1, b.c0), c.c0); + lemma_k_mul_assoc(a.c1, b.c0, c.c0); + let t101 = k_mul(k_mul(a.c1, b.c0), c.c1); + lemma_k_mul_assoc(a.c1, b.c0, c.c1); + let t102 = k_mul(k_mul(a.c1, b.c0), c.c2); + lemma_k_mul_assoc(a.c1, b.c0, c.c2); + let t110 = k_mul(k_mul(a.c1, b.c1), c.c0); + lemma_k_mul_assoc(a.c1, b.c1, c.c0); + let t111 = k_mul(k_mul(a.c1, b.c1), c.c1); + lemma_k_mul_assoc(a.c1, b.c1, c.c1); + let t112 = k_mul(k_mul(a.c1, b.c1), c.c2); + lemma_k_mul_assoc(a.c1, b.c1, c.c2); + let t120 = k_mul(k_mul(a.c1, b.c2), c.c0); + lemma_k_mul_assoc(a.c1, b.c2, c.c0); + let t121 = k_mul(k_mul(a.c1, b.c2), c.c1); + lemma_k_mul_assoc(a.c1, b.c2, c.c1); + let t122 = k_mul(k_mul(a.c1, b.c2), c.c2); + lemma_k_mul_assoc(a.c1, b.c2, c.c2); + let t200 = k_mul(k_mul(a.c2, b.c0), c.c0); + lemma_k_mul_assoc(a.c2, b.c0, c.c0); + let t201 = k_mul(k_mul(a.c2, b.c0), c.c1); + lemma_k_mul_assoc(a.c2, b.c0, c.c1); + let t202 = k_mul(k_mul(a.c2, b.c0), c.c2); + lemma_k_mul_assoc(a.c2, b.c0, c.c2); + let t210 = k_mul(k_mul(a.c2, b.c1), c.c0); + lemma_k_mul_assoc(a.c2, b.c1, c.c0); + let t211 = k_mul(k_mul(a.c2, b.c1), c.c1); + lemma_k_mul_assoc(a.c2, b.c1, c.c1); + let t212 = k_mul(k_mul(a.c2, b.c1), c.c2); + lemma_k_mul_assoc(a.c2, b.c1, c.c2); + let t220 = k_mul(k_mul(a.c2, b.c2), c.c0); + lemma_k_mul_assoc(a.c2, b.c2, c.c0); + let t221 = k_mul(k_mul(a.c2, b.c2), c.c1); + lemma_k_mul_assoc(a.c2, b.c2, c.c1); + let t222 = k_mul(k_mul(a.c2, b.c2), c.c2); + lemma_k_mul_assoc(a.c2, b.c2, c.c2); + lemma_k_mul_xor_left(k_mul(a.c0, b.c0), (k_mul(a.c1, b.c2) ^ k_mul(a.c2, b.c1)), c.c0); + lemma_k_mul_xor_left(k_mul(a.c1, b.c2), k_mul(a.c2, b.c1), c.c0); + lemma_k_mul_xor_left(k_mul(a.c0, b.c0), (k_mul(a.c1, b.c2) ^ k_mul(a.c2, b.c1)), c.c1); + lemma_k_mul_xor_left(k_mul(a.c1, b.c2), k_mul(a.c2, b.c1), c.c1); + lemma_k_mul_xor_left(k_mul(a.c0, b.c0), (k_mul(a.c1, b.c2) ^ k_mul(a.c2, b.c1)), c.c2); + lemma_k_mul_xor_left(k_mul(a.c1, b.c2), k_mul(a.c2, b.c1), c.c2); + lemma_k_mul_xor_left(((k_mul(a.c0, b.c1) ^ k_mul(a.c1, b.c0)) ^ (k_mul(a.c1, b.c2) ^ k_mul(a.c2, b.c1))), k_mul(a.c2, b.c2), c.c0); + lemma_k_mul_xor_left((k_mul(a.c0, b.c1) ^ k_mul(a.c1, b.c0)), (k_mul(a.c1, b.c2) ^ k_mul(a.c2, b.c1)), c.c0); + lemma_k_mul_xor_left(k_mul(a.c0, b.c1), k_mul(a.c1, b.c0), c.c0); + lemma_k_mul_xor_left(k_mul(a.c1, b.c2), k_mul(a.c2, b.c1), c.c0); + lemma_k_mul_xor_left(((k_mul(a.c0, b.c1) ^ k_mul(a.c1, b.c0)) ^ (k_mul(a.c1, b.c2) ^ k_mul(a.c2, b.c1))), k_mul(a.c2, b.c2), c.c1); + lemma_k_mul_xor_left((k_mul(a.c0, b.c1) ^ k_mul(a.c1, b.c0)), (k_mul(a.c1, b.c2) ^ k_mul(a.c2, b.c1)), c.c1); + lemma_k_mul_xor_left(k_mul(a.c0, b.c1), k_mul(a.c1, b.c0), c.c1); + lemma_k_mul_xor_left(k_mul(a.c1, b.c2), k_mul(a.c2, b.c1), c.c1); + lemma_k_mul_xor_left(((k_mul(a.c0, b.c1) ^ k_mul(a.c1, b.c0)) ^ (k_mul(a.c1, b.c2) ^ k_mul(a.c2, b.c1))), k_mul(a.c2, b.c2), c.c2); + lemma_k_mul_xor_left((k_mul(a.c0, b.c1) ^ k_mul(a.c1, b.c0)), (k_mul(a.c1, b.c2) ^ k_mul(a.c2, b.c1)), c.c2); + lemma_k_mul_xor_left(k_mul(a.c0, b.c1), k_mul(a.c1, b.c0), c.c2); + lemma_k_mul_xor_left(k_mul(a.c1, b.c2), k_mul(a.c2, b.c1), c.c2); + lemma_k_mul_xor_left(((k_mul(a.c0, b.c2) ^ k_mul(a.c1, b.c1)) ^ k_mul(a.c2, b.c0)), k_mul(a.c2, b.c2), c.c0); + lemma_k_mul_xor_left((k_mul(a.c0, b.c2) ^ k_mul(a.c1, b.c1)), k_mul(a.c2, b.c0), c.c0); + lemma_k_mul_xor_left(k_mul(a.c0, b.c2), k_mul(a.c1, b.c1), c.c0); + lemma_k_mul_xor_left(((k_mul(a.c0, b.c2) ^ k_mul(a.c1, b.c1)) ^ k_mul(a.c2, b.c0)), k_mul(a.c2, b.c2), c.c1); + lemma_k_mul_xor_left((k_mul(a.c0, b.c2) ^ k_mul(a.c1, b.c1)), k_mul(a.c2, b.c0), c.c1); + lemma_k_mul_xor_left(k_mul(a.c0, b.c2), k_mul(a.c1, b.c1), c.c1); + lemma_k_mul_xor_left(((k_mul(a.c0, b.c2) ^ k_mul(a.c1, b.c1)) ^ k_mul(a.c2, b.c0)), k_mul(a.c2, b.c2), c.c2); + lemma_k_mul_xor_left((k_mul(a.c0, b.c2) ^ k_mul(a.c1, b.c1)), k_mul(a.c2, b.c0), c.c2); + lemma_k_mul_xor_left(k_mul(a.c0, b.c2), k_mul(a.c1, b.c1), c.c2); + lemma_k_mul_xor_right(a.c0, k_mul(b.c0, c.c0), (k_mul(b.c1, c.c2) ^ k_mul(b.c2, c.c1))); + lemma_k_mul_xor_right(a.c0, k_mul(b.c1, c.c2), k_mul(b.c2, c.c1)); + lemma_k_mul_xor_right(a.c0, ((k_mul(b.c0, c.c1) ^ k_mul(b.c1, c.c0)) ^ (k_mul(b.c1, c.c2) ^ k_mul(b.c2, c.c1))), k_mul(b.c2, c.c2)); + lemma_k_mul_xor_right(a.c0, (k_mul(b.c0, c.c1) ^ k_mul(b.c1, c.c0)), (k_mul(b.c1, c.c2) ^ k_mul(b.c2, c.c1))); + lemma_k_mul_xor_right(a.c0, k_mul(b.c0, c.c1), k_mul(b.c1, c.c0)); + lemma_k_mul_xor_right(a.c0, k_mul(b.c1, c.c2), k_mul(b.c2, c.c1)); + lemma_k_mul_xor_right(a.c0, ((k_mul(b.c0, c.c2) ^ k_mul(b.c1, c.c1)) ^ k_mul(b.c2, c.c0)), k_mul(b.c2, c.c2)); + lemma_k_mul_xor_right(a.c0, (k_mul(b.c0, c.c2) ^ k_mul(b.c1, c.c1)), k_mul(b.c2, c.c0)); + lemma_k_mul_xor_right(a.c0, k_mul(b.c0, c.c2), k_mul(b.c1, c.c1)); + lemma_k_mul_xor_right(a.c1, k_mul(b.c0, c.c0), (k_mul(b.c1, c.c2) ^ k_mul(b.c2, c.c1))); + lemma_k_mul_xor_right(a.c1, k_mul(b.c1, c.c2), k_mul(b.c2, c.c1)); + lemma_k_mul_xor_right(a.c1, ((k_mul(b.c0, c.c1) ^ k_mul(b.c1, c.c0)) ^ (k_mul(b.c1, c.c2) ^ k_mul(b.c2, c.c1))), k_mul(b.c2, c.c2)); + lemma_k_mul_xor_right(a.c1, (k_mul(b.c0, c.c1) ^ k_mul(b.c1, c.c0)), (k_mul(b.c1, c.c2) ^ k_mul(b.c2, c.c1))); + lemma_k_mul_xor_right(a.c1, k_mul(b.c0, c.c1), k_mul(b.c1, c.c0)); + lemma_k_mul_xor_right(a.c1, k_mul(b.c1, c.c2), k_mul(b.c2, c.c1)); + lemma_k_mul_xor_right(a.c1, ((k_mul(b.c0, c.c2) ^ k_mul(b.c1, c.c1)) ^ k_mul(b.c2, c.c0)), k_mul(b.c2, c.c2)); + lemma_k_mul_xor_right(a.c1, (k_mul(b.c0, c.c2) ^ k_mul(b.c1, c.c1)), k_mul(b.c2, c.c0)); + lemma_k_mul_xor_right(a.c1, k_mul(b.c0, c.c2), k_mul(b.c1, c.c1)); + lemma_k_mul_xor_right(a.c2, k_mul(b.c0, c.c0), (k_mul(b.c1, c.c2) ^ k_mul(b.c2, c.c1))); + lemma_k_mul_xor_right(a.c2, k_mul(b.c1, c.c2), k_mul(b.c2, c.c1)); + lemma_k_mul_xor_right(a.c2, ((k_mul(b.c0, c.c1) ^ k_mul(b.c1, c.c0)) ^ (k_mul(b.c1, c.c2) ^ k_mul(b.c2, c.c1))), k_mul(b.c2, c.c2)); + lemma_k_mul_xor_right(a.c2, (k_mul(b.c0, c.c1) ^ k_mul(b.c1, c.c0)), (k_mul(b.c1, c.c2) ^ k_mul(b.c2, c.c1))); + lemma_k_mul_xor_right(a.c2, k_mul(b.c0, c.c1), k_mul(b.c1, c.c0)); + lemma_k_mul_xor_right(a.c2, k_mul(b.c1, c.c2), k_mul(b.c2, c.c1)); + lemma_k_mul_xor_right(a.c2, ((k_mul(b.c0, c.c2) ^ k_mul(b.c1, c.c1)) ^ k_mul(b.c2, c.c0)), k_mul(b.c2, c.c2)); + lemma_k_mul_xor_right(a.c2, (k_mul(b.c0, c.c2) ^ k_mul(b.c1, c.c1)), k_mul(b.c2, c.c0)); + lemma_k_mul_xor_right(a.c2, k_mul(b.c0, c.c2), k_mul(b.c1, c.c1)); + assert(((t000 ^ (t120 ^ t210)) ^ ((((t012 ^ t102) ^ (t122 ^ t212)) ^ t222) ^ (((t021 ^ t111) ^ t201) ^ t221))) == ((t000 ^ (t012 ^ t021)) ^ ((((t102 ^ t111) ^ t120) ^ t122) ^ (((t201 ^ t210) ^ (t212 ^ t221)) ^ t222))) && ((((t001 ^ (t121 ^ t211)) ^ (((t010 ^ t100) ^ (t120 ^ t210)) ^ t220)) ^ ((((t012 ^ t102) ^ (t122 ^ t212)) ^ t222) ^ (((t021 ^ t111) ^ t201) ^ t221))) ^ (((t022 ^ t112) ^ t202) ^ t222)) == ((((((t001 ^ t010) ^ (t012 ^ t021)) ^ t022) ^ (t100 ^ (t112 ^ t121))) ^ ((((t102 ^ t111) ^ t120) ^ t122) ^ (((t201 ^ t210) ^ (t212 ^ t221)) ^ t222))) ^ (((t202 ^ t211) ^ t220) ^ t222)) && ((((t002 ^ (t122 ^ t212)) ^ (((t011 ^ t101) ^ (t121 ^ t211)) ^ t221)) ^ (((t020 ^ t110) ^ t200) ^ t220)) ^ (((t022 ^ t112) ^ t202) ^ t222)) == ((((((t002 ^ t011) ^ t020) ^ t022) ^ (((t101 ^ t110) ^ (t112 ^ t121)) ^ t122)) ^ (t200 ^ (t212 ^ t221))) ^ (((t202 ^ t211) ^ t220) ^ t222))) by (bit_vector); +} + +pub proof fn lemma_e_pow_add(a: F192, m: nat, n: nat) + ensures + e_mul(e_pow(a, m), e_pow(a, n)) == e_pow(a, m + n), + decreases n, +{ + if n == 0 { + lemma_e_mul_one(e_pow(a, m)); + } else { + lemma_e_pow_add(a, m, (n - 1) as nat); + lemma_e_mul_assoc(e_pow(a, m), e_pow(a, (n - 1) as nat), a); + assert((m + n - 1) as nat + 1 == m + n); + } +} + +pub proof fn lemma_e_pow_mul(a: F192, m: nat, n: nat) + ensures + e_pow(e_pow(a, m), n) == e_pow(a, m * n), + decreases n, +{ + if n == 0 { + assert(m * 0 == 0) by (nonlinear_arith); + } else { + lemma_e_pow_mul(a, m, (n - 1) as nat); + lemma_e_pow_add(a, m * (n - 1) as nat, m); + assert(m * (n - 1) as nat + m == m * n) by (nonlinear_arith) + requires + n > 0, + ; + } +} + +pub proof fn lemma_e_pow_one(a: F192) + ensures + e_pow(a, 1) == a, +{ + assert(e_pow(a, 0) == F192::ONE); + lemma_e_mul_one(a); +} + +pub proof fn lemma_e_sq_iter_pow(a: F192, n: nat) + ensures + e_sq_iter(a, n) == e_pow(a, pow2(n)), + decreases n, +{ + if n == 0 { + lemma2_to64(); + lemma_e_pow_one(a); + } else { + lemma_e_sq_iter_pow(a, (n - 1) as nat); + let e = pow2((n - 1) as nat); + lemma_e_pow_add(a, e, e); + lemma_pow2_unfold(n); + } +} + +/// The Galois action on the y-coefficients after `n` squarings: `(c1, c2) -> (c2, c1 + c2)` has order 3. +pub open spec fn rot(n: nat, x1: u64, x2: u64) -> (u64, u64) { + if n % 3 == 0 { + (x1, x2) + } else if n % 3 == 1 { + (x2, x1 ^ x2) + } else { + (x1 ^ x2, x1) + } +} + +/// `n` squarings in `E` square each coefficient `n` times and rotate the y-coefficients. +pub proof fn lemma_e_sq_iter_shape(a: F192, n: nat) + ensures + ({ + let r = rot(n, k_sq_iter(a.c1, n), k_sq_iter(a.c2, n)); + e_sq_iter(a, n) == (F192 { c0: k_sq_iter(a.c0, n), c1: r.0, c2: r.1 }) + }), + decreases n, +{ + if n > 0 { + let i = (n - 1) as nat; + lemma_e_sq_iter_shape(a, i); + let v = e_sq_iter(a, i); + lemma_square_spec(v); + let (x, y) = (k_sq_iter(a.c1, i), k_sq_iter(a.c2, i)); + lemma_k_square_xor(x, y); + let (sx, sy) = (k_mul(x, x), k_mul(y, y)); + assert(sy ^ (sx ^ sy) == sx && (sx ^ sy) ^ sx == sy) by (bit_vector); + } +} + +/// The Frobenius shuffle is the 64th squaring, `a^(2^64)`, by Fermat in `K`. +pub proof fn lemma_e_frobenius(a: F192) + ensures + e_frobenius(a) == e_sq_iter(a, 64), + e_frobenius(a) == e_pow(a, pow2(64)), +{ + lemma_e_sq_iter_shape(a, 64); + lemma_k_fermat(a.c0); + lemma_k_fermat(a.c1); + lemma_k_fermat(a.c2); + lemma_e_sq_iter_pow(a, 64); +} + +/// The embedding of `K` is multiplicative. +pub proof fn lemma_e_from_k_mul(x: u64, y: u64) + ensures + e_mul(e_from_k(x), e_from_k(y)) == e_from_k(k_mul(x, y)), +{ + lemma_k_mul_zero(x); + lemma_k_mul_zero(y); + lemma_k_mul_zero(0); + let p = k_mul(x, y); + assert(p ^ (0u64 ^ 0u64) == p && (0u64 ^ 0u64) ^ (0u64 ^ 0u64) ^ 0u64 == 0u64 && (0u64 ^ 0u64 ^ 0u64) + ^ 0u64 == 0u64) by (bit_vector); +} + +pub proof fn lemma_e_from_k_pow(x: u64, n: nat) + ensures + e_from_k(k_pow(x, n)) == e_pow(e_from_k(x), n), + decreases n, +{ + if n > 0 { + lemma_e_from_k_pow(x, (n - 1) as nat); + lemma_e_from_k_mul(k_pow(x, (n - 1) as nat), x); + } +} + +/// The norm `a φ(a) φ²(a) = a^(1 + q + q^2)`, `q = 2^64`, lies in `K`: it is fixed by the Frobenius. +pub proof fn lemma_norm(a: F192) + ensures + ({ + let m = e_mul(e_frobenius(a), e_frobenius(e_frobenius(a))); + let q = pow2(64); + &&& m == e_pow(a, q + q * q) + &&& e_mul(a, m) == e_pow(a, 1 + q + q * q) + &&& e_mul(a, m).c1 == 0 + &&& e_mul(a, m).c2 == 0 + }), +{ + let q = pow2(64); + let fa = e_frobenius(a); + lemma_e_frobenius(a); + lemma_e_frobenius(fa); + lemma_e_pow_mul(a, q, q); + lemma_e_pow_add(a, q, q * q); + let m = e_mul(fa, e_frobenius(fa)); + lemma_e_pow_one(a); + lemma_e_pow_add(a, 1, q + q * q); + assert(1 + (q + q * q) == 1 + q + q * q); + let norm = e_mul(a, m); + // φ(norm) = a^(q + q^2 + q^3), and a^(q^3) = φ³(a) = a. + lemma_e_frobenius(norm); + lemma_e_pow_mul(a, 1 + q + q * q, q); + lemma_e_frobenius(e_frobenius(fa)); + lemma_e_pow_mul(a, q * q, q); + assert(e_frobenius(e_frobenius(fa)) == a) by { + let (x1, x2) = (a.c1, a.c2); + assert(x2 ^ (x1 ^ x2) == x1 && (x1 ^ x2) ^ (x2 ^ (x1 ^ x2)) == x2) by (bit_vector); + } + lemma_e_pow_add(a, q + q * q, q * q * q); + assert((1 + q + q * q) * q == (q + q * q) + q * q * q) by (nonlinear_arith); + lemma_e_mul_comm(m, a); + let (n1, n2) = (norm.c1, norm.c2); + assert(n2 == n1 && (n1 ^ n2) == n2 ==> n1 == 0 && n2 == 0) by (bit_vector); +} + +/// `m N^(q-2) = a^(q^3 - 2)`, the exponent bookkeeping of `F192::inv`. +pub proof fn lemma_inv_exponent(a: F192) + ensures + ({ + let m = e_mul(e_frobenius(a), e_frobenius(e_frobenius(a))); + let norm = e_mul(a, m); + e_mul(m, e_from_k(k_pow(norm.c0, (pow2(64) - 2) as nat))) == e_pow(a, (pow2(192) - 2) as nat) + }), +{ + let q = pow2(64); + lemma2_to64(); + lemma_pow2_adds(64, 64); + lemma_pow2_adds(128, 64); + lemma_norm(a); + let m = e_mul(e_frobenius(a), e_frobenius(e_frobenius(a))); + let norm = e_mul(a, m); + assert(e_from_k(norm.c0) == norm); + lemma_e_from_k_pow(norm.c0, (q - 2) as nat); + lemma_e_pow_mul(a, 1 + q + q * q, (q - 2) as nat); + lemma_e_pow_add(a, q + q * q, ((1 + q + q * q) * (q - 2)) as nat); + assert(pow2(192) == q * q * q); + assert((q + q * q) + (1 + q + q * q) * (q - 2) == q * q * q - 2) by (nonlinear_arith); +} + +// --------------------------------------------------------------------------------------------- +// E is a field: the inverse is an inverse +// --------------------------------------------------------------------------------------------- +/// The only idempotents of `E` are 0 and 1. +pub proof fn lemma_e_idempotent(e: F192) + requires + e_mul(e, e) == e, + ensures + e == F192::ZERO || e == F192::ONE, +{ + lemma_square_spec(e); + lemma_k_idempotent(e.c0); + lemma_k_square(e.c1); + lemma_k_square(e.c2); + let (c1, c2) = (e.c1, e.c2); + assert(reduce_formula(spread(c2)) == c1 && reduce_formula(spread(c1)) ^ reduce_formula(spread(c2)) == c2 + ==> c1 == 0 && c2 == 0) by (bit_vector); +} + +pub proof fn lemma_e_pow_zero(n: nat) + requires + n > 0, + ensures + e_pow(F192::ZERO, n) == F192::ZERO, +{ + let p = e_pow(F192::ZERO, (n - 1) as nat); + lemma_k_mul_zero(p.c0); + lemma_k_mul_zero(p.c1); + lemma_k_mul_zero(p.c2); + assert(0u64 ^ (0u64 ^ 0u64) == 0u64 && (0u64 ^ 0u64) ^ (0u64 ^ 0u64) ^ 0u64 == 0u64 && (0u64 ^ 0u64 ^ 0u64) + ^ 0u64 == 0u64) by (bit_vector); +} + +/// `a^(2^192 - 2)` is the inverse of every nonzero `a` of `E`, and zero's image is zero. +pub proof fn lemma_e_inverse(a: F192) + ensures + a != F192::ZERO ==> e_mul(a, e_pow(a, (pow2(192) - 2) as nat)) == F192::ONE, + a == F192::ZERO ==> e_pow(a, (pow2(192) - 2) as nat) == F192::ZERO, +{ + lemma2_to64(); + lemma_pow2_adds(64, 64); + lemma_pow2_adds(128, 64); + let q = pow2(64); + let big = pow2(192); + assert(big == q * q * q); + let w = e_pow(a, (big - 2) as nat); + let u = e_pow(a, (big - 1) as nat); + if a == F192::ZERO { + lemma_e_pow_zero((big - 2) as nat); + } else { + assert(u == e_mul(w, a)); + lemma_e_mul_comm(w, a); + // a^(q^3) = φ³(a) = a. + let fa = e_frobenius(a); + lemma_e_frobenius(a); + lemma_e_frobenius(fa); + lemma_e_frobenius(e_frobenius(fa)); + lemma_e_pow_mul(a, q, q); + lemma_e_pow_mul(a, q * q, q); + assert(e_frobenius(e_frobenius(fa)) == a) by { + let (x1, x2) = (a.c1, a.c2); + assert(x2 ^ (x1 ^ x2) == x1 && (x1 ^ x2) ^ (x2 ^ (x1 ^ x2)) == x2) by (bit_vector); + } + assert(e_pow(a, big) == a); + // u^2 = u, so u is 0 or 1; u = 0 would give a = 0. + lemma_e_pow_add(a, (big - 1) as nat, (big - 1) as nat); + lemma_e_pow_add(a, big, (big - 2) as nat); + assert((big - 1) as nat + (big - 1) as nat == big + (big - 2) as nat); + lemma_e_idempotent(u); + lemma_e_pow_add(a, (big - 1) as nat, 1); + lemma_e_pow_one(a); + assert((big - 1) as nat + 1 == big); + if u == F192::ZERO { + lemma_k_mul_zero(a.c0); + lemma_k_mul_zero(a.c1); + lemma_k_mul_zero(a.c2); + assert(0u64 ^ (0u64 ^ 0u64) == 0u64 && (0u64 ^ 0u64) ^ (0u64 ^ 0u64) ^ 0u64 == 0u64 && (0u64 ^ 0u64 + ^ 0u64) ^ 0u64 == 0u64) by (bit_vector); + } + } +} + +// --------------------------------------------------------------------------------------------- +// Executable code: the portable paths +// --------------------------------------------------------------------------------------------- +/// An element `c0 + c1*y + c2*y^2`; bit `i` of each coefficient is its coefficient of `x^i`. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(C)] +pub struct F192 { + /// The coefficient of `y^0`. + pub c0: u64, + /// The coefficient of `y^1`. + pub c1: u64, + /// The coefficient of `y^2`. + pub c2: u64, +} + +impl F192 { + pub const ZERO: Self = Self { c0: 0, c1: 0, c2: 0 }; + + pub const ONE: Self = Self { c0: 1, c1: 0, c2: 0 }; + + /// The element `y` (root of y^3 + y + 1 over the base field). + pub const Y: Self = Self { c0: 0, c1: 1, c2: 0 }; + + #[inline] + pub const fn new(c0: u64, c1: u64, c2: u64) -> (r: Self) + ensures + r == (F192 { c0, c1, c2 }), + { + Self { c0, c1, c2 } + } + + #[inline] + pub const fn is_zero(self) -> (r: bool) + ensures + r == (self == F192::ZERO), + { + self.c0 == 0 && self.c1 == 0 && self.c2 == 0 + } + + /// Product without the base-field reduction, for XOR accumulation. + #[inline] + pub fn mul_unreduced(self, rhs: Self) -> (r: F192Unreduced) + ensures + e_value(r) == e_mul(self, rhs), + { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + return aarch64::mul_unreduced(self, rhs); + #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] + return unsafe { x86_64::mul_unreduced(self, rhs) }; + #[cfg(not(any(all(target_arch = "aarch64", target_feature = "aes"), + all(target_arch = "x86_64", target_feature = "pclmulqdq"))))] + software::mul_unreduced(self, rhs) + } + + /// Mixed product by a base-field scalar. + /// + /// The scalar has no `y` component, so the three coefficients multiply independently in `K`. + #[inline] + pub fn mul_base(self, k: F64) -> (r: Self) + ensures + r == e_mul(self, e_from_k(k.0)), + r == (F192 { c0: k_mul(self.c0, k.0), c1: k_mul(self.c1, k.0), c2: k_mul(self.c2, k.0) }), + { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + proof { + lemma_mul_base_spec(self, k.0); + } + aarch64::mul_base(self, k) + } + #[cfg(not(all(target_arch = "aarch64", target_feature = "aes")))] + { + self.mul_base_unreduced(k).reduce() + } + } + + /// Mixed product by a base-field scalar without the reduction, for XOR accumulation. + #[inline] + pub fn mul_base_unreduced(self, k: F64) -> (r: F192Unreduced) + ensures + e_value(r) == e_mul(self, e_from_k(k.0)), + e_value(r) == (F192 { + c0: k_mul(self.c0, k.0), + c1: k_mul(self.c1, k.0), + c2: k_mul(self.c2, k.0), + }), + { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + proof { lemma_mul_base_spec(self, k.0); } + return aarch64::mul_base_unreduced(self, k); + } + #[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] + { + proof { lemma_mul_base_spec(self, k.0); } + return unsafe { x86_64::mul_base_unreduced(self, k) }; + } + #[cfg(not(any(all(target_arch = "aarch64", target_feature = "aes"), + all(target_arch = "x86_64", target_feature = "pclmulqdq"))))] + { + let (w0, w1, w2) = (mul_wide(self.c0, k.0), mul_wide(self.c1, k.0), mul_wide(self.c2, k.0)); + proof { + lemma_u_from_wide(w0, w1, w2); + lemma_mul_base_spec(self, k.0); + } + F192Unreduced::from_wide([w0, w1, w2]) + } + } + + /// Squaring, with 3 base-field squarings instead of 6 products. + /// + /// Cross terms vanish in characteristic 2: + /// + /// ```text + /// (c0 + c1*y + c2*y^2)^2 = c0^2 + c1^2*y^2 + c2^2*y^4 + /// = c0^2 + c2^2*y + (c1^2 + c2^2)*y^2 (y^4 = y^2 + y) + /// ``` + #[inline] + pub fn square(self) -> (r: Self) + ensures + r == e_mul(self, self), + { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + aarch64::square(self) + } + #[cfg(not(all(target_arch = "aarch64", target_feature = "aes")))] + { + // Square each coefficient as a 128-bit polynomial. + let (s0, s1, s2) = (square_wide(self.c0), square_wide(self.c1), square_wide(self.c2)); + proof { + lemma_u_from_wide(s0, s2, s1 ^ s2); + lemma_k_mod_xor(s1, s2); + lemma_square_spec(self); + } + // Fold y^4 back onto y^2 and y, then reduce each coefficient once. + F192Unreduced::from_wide([s0, s2, s1 ^ s2]).reduce() + } + } + + /// The Frobenius `self^(2^64)`, as the coefficient shuffle `c0 + c2·y + (c1 + c2)·y²`. + #[inline] + /// + /// `r == e_pow(self, 2^64)` is [`lemma_e_frobenius`]. + pub const fn frobenius(self) -> (r: Self) + ensures + r == e_frobenius(self), + { + Self { c0: self.c0, c1: self.c2, c2: self.c1 ^ self.c2 } + } + + /// Multiplicative inverse: `self^(2^192 − 2)`. `ZERO.inv() == ZERO`. + /// + /// Via the norm to the base field. With `φ` the Frobenius and + /// `m = φ(self)·φ²(self)`, the product `self·m` is the norm `N(self) ∈ K`, + /// so `self⁻¹ = m·N(self)⁻¹` needs two extension multiplies, one base-field + /// inverse and one base-field scaling. + /// + /// Production checks `N(self) ∈ K` with a `debug_assert!`; here it is proven ([`lemma_norm`]). + pub fn inv(self) -> (r: Self) + ensures + r == e_pow(self, (pow2(192) - 2) as nat), + self != F192::ZERO ==> e_mul(self, r) == F192::ONE, + self == F192::ZERO ==> r == F192::ZERO, + { + proof { + lemma_e_inverse(self); + } + let m = self.frobenius() * self.frobenius().frobenius(); + let norm = self * m; + proof { + lemma_norm(self); + assert(norm.c1 == 0 && norm.c2 == 0); + lemma_inv_exponent(self); + } + m.mul_base(F64(norm.c0).inv()) + } +} + +/// The mixed product is the product by an element of `K`. +pub proof fn lemma_mul_base_spec(a: F192, k: u64) + ensures + e_mul(a, e_from_k(k)) == (F192 { c0: k_mul(a.c0, k), c1: k_mul(a.c1, k), c2: k_mul(a.c2, k) }), +{ + lemma_k_mul_zero(a.c0); + lemma_k_mul_zero(a.c1); + lemma_k_mul_zero(a.c2); + let (x0, x1, x2) = (k_mul(a.c0, k), k_mul(a.c1, k), k_mul(a.c2, k)); + assert(x0 ^ (0u64 ^ 0u64) == x0 && (0u64 ^ x1) ^ (0u64 ^ 0u64) ^ 0u64 == x1 && (0u64 ^ 0u64 ^ x2) ^ 0u64 + == x2) by (bit_vector); +} + +/// Squaring in `E`: the cross terms cancel. +pub proof fn lemma_square_spec(a: F192) + ensures + e_mul(a, a) == (F192 { + c0: k_mul(a.c0, a.c0), + c1: k_mul(a.c2, a.c2), + c2: k_mul(a.c1, a.c1) ^ k_mul(a.c2, a.c2), + }), +{ + lemma_k_mul_comm(a.c0, a.c1); + lemma_k_mul_comm(a.c0, a.c2); + lemma_k_mul_comm(a.c1, a.c2); + let (s0, s1, s2) = (k_mul(a.c0, a.c0), k_mul(a.c1, a.c1), k_mul(a.c2, a.c2)); + let (x01, x02, x12) = (k_mul(a.c0, a.c1), k_mul(a.c0, a.c2), k_mul(a.c1, a.c2)); + assert(s0 ^ (x12 ^ x12) == s0 && (x01 ^ x01) ^ (x12 ^ x12) ^ s2 == s2 && (x02 ^ s1 ^ x02) ^ s2 == s1 ^ s2) + by (bit_vector); +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::AddSpecImpl for F192 { + open spec fn obeys_add_spec() -> bool { + true + } + + open spec fn add_req(self, rhs: F192) -> bool { + true + } + + open spec fn add_spec(self, rhs: F192) -> F192 { + e_add(self, rhs) + } +} + +impl Add for F192 { + type Output = Self; + + #[inline] + fn add(self, rhs: Self) -> (r: Self) + ensures + r == e_add(self, rhs), + { + Self { c0: self.c0 ^ rhs.c0, c1: self.c1 ^ rhs.c1, c2: self.c2 ^ rhs.c2 } + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::MulSpecImpl for F192 { + open spec fn obeys_mul_spec() -> bool { + true + } + + open spec fn mul_req(self, rhs: F192) -> bool { + true + } + + open spec fn mul_spec(self, rhs: F192) -> F192 { + e_mul(self, rhs) + } +} + +impl Mul for F192 { + type Output = Self; + + #[inline] + fn mul(self, rhs: Self) -> (r: Self) + ensures + r == e_mul(self, rhs), + { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + aarch64::mul(self, rhs) + } + #[cfg(not(all(target_arch = "aarch64", target_feature = "aes")))] + { + self.mul_unreduced(rhs).reduce() + } + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::AddAssignSpecImpl for F192 { + open spec fn obeys_add_assign_spec() -> bool { + true + } + + open spec fn add_assign_req(&self, rhs: F192) -> bool { + true + } + + open spec fn add_assign_spec(&self, rhs: F192) -> &F192 { + &e_add(*self, rhs) + } +} + +impl AddAssign for F192 { + #[inline] + fn add_assign(&mut self, rhs: Self) + ensures + *final(self) == e_add(*old(self), rhs), + { + self.c0 ^= rhs.c0; + self.c1 ^= rhs.c1; + self.c2 ^= rhs.c2; + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::MulAssignSpecImpl for F192 { + open spec fn obeys_mul_assign_spec() -> bool { + true + } + + open spec fn mul_assign_req(&self, rhs: F192) -> bool { + true + } + + open spec fn mul_assign_spec(&self, rhs: F192) -> &F192 { + &e_mul(*self, rhs) + } +} + +impl MulAssign for F192 { + #[inline] + fn mul_assign(&mut self, rhs: Self) + ensures + *final(self) == e_mul(*old(self), rhs), + { + *self = *self * rhs; + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::convert::FromSpecImpl for F192 { + open spec fn obeys_from_spec() -> bool { + true + } + + open spec fn from_spec(k: F64) -> F192 { + e_from_k(k.0) + } +} + +impl From for F192 { + #[inline] + fn from(k: F64) -> Self { + Self { c0: k.0, c1: 0, c2: 0 } + } +} + +/// An F192 value whose coefficients are not yet reduced modulo the base polynomial. +/// +/// Coefficient `k` is the 128-bit carry-less polynomial multiplying `y^k`, for `k < 3`. +/// Products land here already folded by `y^3 = y + 1`, so a sum of them is a plain XOR. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct F192Unreduced { + /// The 128-bit coefficients of `y^0`, `y^1`, `y^2`, each as its `[low, high]` words. + pub coeffs: [[u64; 2]; 3], +} + +impl F192Unreduced { + pub const ZERO: Self = Self { coeffs: [[0, 0], [0, 0], [0, 0]] }; + + /// Build from three 128-bit coefficients. + #[inline] + pub fn from_wide(coeffs: [u128; 3]) -> (r: Self) + ensures + r == u_from_wide(coeffs[0], coeffs[1], coeffs[2]), + { + Self { + coeffs: [ + [coeffs[0] as u64, (coeffs[0] >> 64) as u64], + [coeffs[1] as u64, (coeffs[1] >> 64) as u64], + [coeffs[2] as u64, (coeffs[2] >> 64) as u64], + ], + } + } + + /// Reduce each coefficient modulo the base polynomial. + #[inline] + pub fn reduce(self) -> (r: F192) + ensures + r == e_value(self), + { + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + aarch64::reduce(self) + } + #[cfg(not(all(target_arch = "aarch64", target_feature = "aes")))] + { + // Production maps this over the three coefficients. + let reduce_one = |w: [u64; 2]| -> (r: u64) + ensures + r == k_mod(wide(w)), + { reduce(u128::from(w[1]) << 64 | u128::from(w[0])) }; + let (c0, c1, c2) = (reduce_one(self.coeffs[0]), reduce_one(self.coeffs[1]), reduce_one(self.coeffs[2])); + F192 { c0, c1, c2 } + } + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::convert::FromSpecImpl for F192Unreduced { + open spec fn obeys_from_spec() -> bool { + true + } + + open spec fn from_spec(e: F192) -> F192Unreduced { + F192Unreduced { coeffs: [[e.c0, 0], [e.c1, 0], [e.c2, 0]] } + } +} + +impl From for F192Unreduced { + /// Embed a reduced element: each coefficient is its own 128-bit polynomial. + #[inline] + fn from(e: F192) -> Self { + Self { coeffs: [[e.c0, 0], [e.c1, 0], [e.c2, 0]] } + } +} + +/// Embedding a reduced element and reducing it gives the element back. +pub proof fn lemma_from_f192(e: F192) + ensures + e_value(F192Unreduced { coeffs: [[e.c0, 0], [e.c1, 0], [e.c2, 0]] }) == e, +{ + assert(wide([e.c0, 0]) == e.c0 as u128 && (e.c0 as u128) >> 64u128 == 0) by { + let c = e.c0; + assert((((0u64 as u128) << 64u128) | (c as u128)) == c as u128 && (c as u128) >> 64u128 == 0) by (bit_vector); + } + assert(wide([e.c1, 0]) == e.c1 as u128 && (e.c1 as u128) >> 64u128 == 0) by { + let c = e.c1; + assert((((0u64 as u128) << 64u128) | (c as u128)) == c as u128 && (c as u128) >> 64u128 == 0) by (bit_vector); + } + assert(wide([e.c2, 0]) == e.c2 as u128 && (e.c2 as u128) >> 64u128 == 0) by { + let c = e.c2; + assert((((0u64 as u128) << 64u128) | (c as u128)) == c as u128 && (c as u128) >> 64u128 == 0) by (bit_vector); + } + lemma_k_mod_small(e.c0 as u128); + lemma_k_mod_small(e.c1 as u128); + lemma_k_mod_small(e.c2 as u128); +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::BitXorSpecImpl for F192Unreduced { + open spec fn obeys_bitxor_spec() -> bool { + true + } + + open spec fn bitxor_req(self, rhs: F192Unreduced) -> bool { + true + } + + open spec fn bitxor_spec(self, rhs: F192Unreduced) -> F192Unreduced { + u_xor(self, rhs) + } +} + +impl BitXor for F192Unreduced { + type Output = Self; + + #[inline] + /// Production takes `mut self`, which Verus does not accept; the copy rebinds it. + fn bitxor(self, rhs: Self) -> (r: Self) + ensures + r == u_xor(self, rhs), + e_value(r) == e_add(e_value(self), e_value(rhs)), + { + proof { + lemma_e_value_xor(self, rhs); + } + let mut s = self; + s ^= rhs; + s + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::BitXorAssignSpecImpl for F192Unreduced { + open spec fn obeys_bitxor_assign_spec() -> bool { + true + } + + open spec fn bitxor_assign_req(&self, rhs: F192Unreduced) -> bool { + true + } + + open spec fn bitxor_assign_spec(&self, rhs: F192Unreduced) -> &F192Unreduced { + &u_xor(*self, rhs) + } +} + +impl BitXorAssign for F192Unreduced { + /// Production: one XOR per word of the flattened coefficients. + #[inline] + fn bitxor_assign(&mut self, rhs: Self) + ensures + *final(self) == u_xor(*old(self), rhs), + { + let mut k = 0; + while k < 3 + invariant + 0 <= k <= 3, + forall|i: int| 0 <= i < k ==> #[trigger] self.coeffs[i] == u_xor(*old(self), rhs).coeffs[i], + forall|i: int| k <= i < 3 ==> #[trigger] self.coeffs[i] == old(self).coeffs[i], + decreases 3 - k, + { + let mut row = self.coeffs[k]; + row[0] ^= rhs.coeffs[k][0]; + row[1] ^= rhs.coeffs[k][1]; + self.coeffs[k] = row; + k += 1; + } + assert(self.coeffs =~= u_xor(*old(self), rhs).coeffs); + } +} + +// Batched products: the portable arms. +/// Two independent products. +#[inline(always)] +pub fn mul2(a: [F192; 2], b: [F192; 2]) -> (r: [F192; 2]) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] r[i] == e_mul(a[i], b[i]), +{ + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2"))] + return unsafe { x86_64::mul_vec2(a, b) }; + #[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2")))] + [a[0] * b[0], a[1] * b[1]] +} + +/// Four independent products. +#[inline(always)] +pub fn mul4(a: [F192; 4], b: [F192; 4]) -> (r: [F192; 4]) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] r[i] == e_mul(a[i], b[i]), +{ + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] + return unsafe { x86_64::mul_vec4(a, b) }; + #[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f")))] + { + let lo = mul2([a[0], a[1]], [b[0], b[1]]); + let hi = mul2([a[2], a[3]], [b[2], b[3]]); + [lo[0], lo[1], hi[0], hi[1]] + } +} + +/// Four independent products without the reduction, for a caller XOR-accumulating many products. +/// +/// Production builds the array with `std::array::from_fn`. +#[inline(always)] +pub fn mul_unreduced4(a: [F192; 4], b: [F192; 4]) -> (r: [F192Unreduced; 4]) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] e_value(r[i]) == e_mul(a[i], b[i]), +{ + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] + return unsafe { x86_64::mul_unreduced_vec4(a, b) }; + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2", not(target_feature = "avx512f")))] + return unsafe { + let lo = x86_64::mul_unreduced_vec2([a[0], a[1]], [b[0], b[1]]); + let hi = x86_64::mul_unreduced_vec2([a[2], a[3]], [b[2], b[3]]); + [lo[0], lo[1], hi[0], hi[1]] + }; + #[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2")))] + [a[0].mul_unreduced(b[0]), a[1].mul_unreduced(b[1]), a[2].mul_unreduced(b[2]), a[3].mul_unreduced(b[3])] +} + +/// Eight mixed products `t * k[i]` by one shared `E` scalar. +/// +/// Production maps `t.mul_base` over `k`. +#[inline(always)] +pub fn mul_base8(t: F192, k: [F64; 8]) -> (r: [F192; 8]) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] r[i] == e_mul(t, e_from_k(k[i].0)), +{ + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] + return unsafe { x86_64::mul_base8(t, k) }; + #[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f")))] + [ + t.mul_base(k[0]), + t.mul_base(k[1]), + t.mul_base(k[2]), + t.mul_base(k[3]), + t.mul_base(k[4]), + t.mul_base(k[5]), + t.mul_base(k[6]), + t.mul_base(k[7]), + ] +} + +/// Eight `E` weights, laid out once for [`dot_base`]. +/// +/// ```text +/// lo lane j = [w_2j.c0, w_2j.c1 ] +/// hi lane j = [w_2j+1.c0, w_2j+1.c1] +/// c2 lane j = [w_2j.c2, w_2j+1.c2] +/// ``` +#[derive(Clone, Copy, Debug, Default)] +#[repr(C, align(64))] +pub struct Weights8 { + pub lo: [u64; 8], + pub hi: [u64; 8], + pub c2: [u64; 8], +} + +/// Weight `i` as [`Weights8::get`] reads it. +pub open spec fn w8_get(w: Weights8, i: int) -> F192 { + let j = i / 2; + let pair = if i % 2 == 0 { + w.lo + } else { + w.hi + }; + F192 { c0: pair[2 * j], c1: pair[2 * j + 1], c2: w.c2[2 * j + i % 2] } +} + +impl Weights8 { + /// Pack eight weights. + /// + /// Production starts from `Self::default()` and copies two words at a time with `copy_from_slice`. + pub fn new(w: &[F192; 8]) -> (out: Self) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] w8_get(out, i) == w[i], + { + let mut out = Self { lo: [0, 0, 0, 0, 0, 0, 0, 0], hi: [0, 0, 0, 0, 0, 0, 0, 0], c2: [0, 0, 0, 0, 0, 0, 0, 0] }; + for j in 0..4usize + invariant + forall|i: int| 0 <= i < 2 * j ==> #[trigger] w8_get(out, i) == w[i], + { + let (e, o) = (w[2 * j], w[2 * j + 1]); + let ghost before = out; + out.lo[2 * j] = e.c0; + out.lo[2 * j + 1] = e.c1; + out.hi[2 * j] = o.c0; + out.hi[2 * j + 1] = o.c1; + out.c2[2 * j] = e.c2; + out.c2[2 * j + 1] = o.c2; + assert forall|i: int| 0 <= i < 2 * j + 2 implies #[trigger] w8_get(out, i) == w[i] by { + if i >= 2 * j { + assert(i / 2 == j); + } else { + assert(i / 2 < j); + assert(2 * (i / 2) + 1 < 2 * j && 2 * (i / 2) + i % 2 < 2 * j); + assert(w8_get(out, i) == w8_get(before, i)); + } + } + } + out + } + + /// Weight `i`, unpacked. + #[inline] + pub const fn get(&self, i: usize) -> (r: F192) + requires + i < 8, + ensures + r == w8_get(*self, i as int), + { + let j = i / 2; + let pair = if i % 2 == 0 { &self.lo } else { &self.hi }; + F192::new(pair[2 * j], pair[2 * j + 1], self.c2[2 * j + i % 2]) + } +} + +/// `sum_{m < n} w_m * k_m`, weight `m` being `w8_get(w[m / 8], m % 8)`. +pub open spec fn dot_spec(w: Seq, k: Seq, n: nat) -> F192 + decreases n, +{ + if n == 0 { + F192::ZERO + } else { + let m = (n - 1) as int; + e_add(dot_spec(w, k, (n - 1) as nat), e_mul(w8_get(w[m / 8], m % 8), e_from_k(k[m].0))) + } +} + +/// The mixed inner product `sum_i w_i * k_i`, unreduced. +/// +/// Production checks `k.len() == 8 * w.len()` with `assert_eq!` and folds over `as_chunks::<8>`. +#[inline] +pub fn dot_base(w: &[Weights8], k: &[F64]) -> (r: F192Unreduced) + requires + k.len() == 8 * w.len(), + ensures + e_value(r) == dot_spec(w@, k@, k.len() as nat), +{ + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] + return unsafe { x86_64::dot_base(w, k) }; + #[cfg(not(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f")))] + { + let mut acc = F192Unreduced::ZERO; + proof { + lemma_u_zero(); + } + for c in 0..w.len() + invariant + k.len() == 8 * w.len(), + e_value(acc) == dot_spec(w@, k@, (8 * c) as nat), + { + for i in 0..8usize + invariant + k.len() == 8 * w.len(), + c < w.len(), + e_value(acc) == dot_spec(w@, k@, (8 * c + i) as nat), + { + proof { + assert((8 * c + i) / 8 == c && (8 * c + i) % 8 == i as int) by (nonlinear_arith) + requires + i < 8, + ; + } + acc = acc ^ w[c].get(i).mul_base_unreduced(k[8 * c + i]); + } + } + acc + } +} + +pub mod software { + use super::*; + use crate::gf2_64::software::clmul; + + /// Schoolbook: nine base products into the five coefficients of y^0..y^4, then the y-fold. + pub fn mul_unreduced(a: F192, b: F192) -> (r: F192Unreduced) + ensures + e_value(r) == e_mul(a, b), + { + let ghost (ga, gb) = (a, b); + let (a, b) = ([a.c0, a.c1, a.c2], [b.c0, b.c1, b.c2]); + let mut e = [0u128, 0, 0, 0, 0]; + // a_i * b_j lands on y^(i + j). + for i in 0..3usize + invariant + e@ == sched(a@, b@, (3 * i) as nat), + { + for j in 0..3usize + invariant + 0 <= i < 3, + e@ == sched(a@, b@, (3 * i + j) as nat), + { + proof { + assert(((3 * i + j) / 3) as int == i && ((3 * i + j) % 3) as int == j); + } + e[i + j] ^= clmul(a[i], b[j]); + proof { + assert(e@ =~= sched(a@, b@, (3 * i + j + 1) as nat)); + } + } + } + proof { + assert(a@ =~= seq![ga.c0, ga.c1, ga.c2]); + assert(b@ =~= seq![gb.c0, gb.c1, gb.c2]); + lemma_mul_unreduced_value(ga, gb); + } + // y^3 = y + 1 and y^4 = y^2 + y. + F192Unreduced::from_wide([e[0] ^ e[3], e[1] ^ e[3] ^ e[4], e[2] ^ e[4]]) + } + + pub fn mul(a: F192, b: F192) -> (r: F192) + ensures + r == e_mul(a, b), + { + mul_unreduced(a, b).reduce() + } +} + +// --------------------------------------------------------------------------------------------- +// The x86 Karatsuba product, given the semantics of the carry-less multiply +// --------------------------------------------------------------------------------------------- +/// `x86_64::mul_unreduced` (six PCLMULQDQs, Karatsuba, then `fold`) yields the same three 128-bit +/// coefficients as the schoolbook product above, with each PCLMULQDQ computing [`clmul`]: +/// +/// ```text +/// p0 = a0 b0, p1 = a1 b1, p2 = a2 b2, p01 = (a0+a1)(b0+b1), p02 = (a0+a2)(b0+b2), p12 = (a1+a2)(b1+b2) +/// d0 = p0 ^ p12 ^ p1 ^ p2, d1 = p0 ^ p12 ^ p01, d2 = p0 ^ p1 ^ p02 +/// ``` +pub proof fn lemma_karatsuba_fold(a: F192, b: F192) + ensures + ({ + let c = |x: u64, y: u64| clmul(x, y as u128); + let (p0, p1, p2) = (c(a.c0, b.c0), c(a.c1, b.c1), c(a.c2, b.c2)); + let p01 = c(a.c0 ^ a.c1, b.c0 ^ b.c1); + let p02 = c(a.c0 ^ a.c2, b.c0 ^ b.c2); + let p12 = c(a.c1 ^ a.c2, b.c1 ^ b.c2); + let q = p0 ^ p12; + let e = sched(seq![a.c0, a.c1, a.c2], seq![b.c0, b.c1, b.c2], 9); + &&& q ^ p1 ^ p2 == e[0] ^ e[3] + &&& q ^ p01 == e[1] ^ e[3] ^ e[4] + &&& p0 ^ p1 ^ p02 == e[2] ^ e[4] + }), +{ + let (sa, sb) = (seq![a.c0, a.c1, a.c2], seq![b.c0, b.c1, b.c2]); + lemma_sched_9(sa, sb); + // Expand each Karatsuba product bilinearly into the nine schoolbook products. + assert forall|x1: u64, x2: u64, y1: u64, y2: u64| + #[trigger] clmul(x1 ^ x2, (y1 ^ y2) as u128) == clmul(x1, y1 as u128) ^ clmul(x1, y2 as u128) ^ clmul( + x2, + y1 as u128, + ) ^ clmul(x2, y2 as u128) by { + assert((y1 ^ y2) as u128 == (y1 as u128) ^ (y2 as u128)) by (bit_vector); + lemma_clmul_xor_left(x1, x2, (y1 ^ y2) as u128); + lemma_clmul_xor_right(x1, y1 as u128, y2 as u128); + lemma_clmul_xor_right(x2, y1 as u128, y2 as u128); + let (a1, a2, b1, b2) = (clmul(x1, y1 as u128), clmul(x1, y2 as u128), clmul(x2, y1 as u128), clmul( + x2, + y2 as u128, + )); + assert((a1 ^ a2) ^ (b1 ^ b2) == a1 ^ a2 ^ b1 ^ b2) by (bit_vector); + } + let c = |x: u64, y: u64| clmul(x, y as u128); + let (c00, c01, c02, c10, c11, c12, c20, c21, c22) = ( + c(a.c0, b.c0), + c(a.c0, b.c1), + c(a.c0, b.c2), + c(a.c1, b.c0), + c(a.c1, b.c1), + c(a.c1, b.c2), + c(a.c2, b.c0), + c(a.c2, b.c1), + c(a.c2, b.c2), + ); + assert(c00 ^ (c11 ^ c12 ^ c21 ^ c22) ^ c11 ^ c22 == (0 ^ c00) ^ (0 ^ c12 ^ c21)) by (bit_vector); + assert(c00 ^ (c11 ^ c12 ^ c21 ^ c22) ^ (c00 ^ c01 ^ c10 ^ c11) == (0 ^ c01 ^ c10) ^ (0 ^ c12 ^ c21) ^ (0 + ^ c22)) by (bit_vector); + assert(c00 ^ c11 ^ (c00 ^ c02 ^ c20 ^ c22) == (0 ^ c02 ^ c11 ^ c20) ^ (0 ^ c22)) by (bit_vector); +} + +} // verus! + +/// x86-64 kernels (production's `gf2_64x3/x86_64.rs`). +#[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] +#[path = "gf2_64x3_x86.rs"] +pub mod x86_64; + +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2"))] +#[path = "gf2_64x3_x86_wrappers.rs"] +mod x86_wrappers; +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2"))] +pub use x86_wrappers::{F192x4, F192x4Unreduced, MixedSums8}; diff --git a/verification/verus/src/gf2_64x3/aarch64.rs b/verification/verus/src/gf2_64x3/aarch64.rs new file mode 100644 index 0000000000..a80232e018 --- /dev/null +++ b/verification/verus/src/gf2_64x3/aarch64.rs @@ -0,0 +1,797 @@ +//! The aarch64 kernels of `E`, `crates/primitives/src/field/gf2_64x3/aarch64.rs`, copied with the same bodies +//! except where noted at the function. +//! +//! A register holding a 128-bit coefficient is read through [`v128`], its two lanes as one polynomial. Given +//! the intrinsic specifications of `crate::intrinsics` (PMULL computes [`clmul`], ...), each kernel is proven to +//! compute the portable specification: [`mul`] is [`e_mul`], [`mul_unreduced`] stands for it ([`e_value`]), +//! the reductions are `k_mod` of each coefficient, and [`F192x1`] / [`F192x1Unreduced`] carry an element and +//! an unreduced value through sums and products. +#[cfg(verus_keep_ghost)] +use super::{ + e_add, e_from_k, e_mul, e_value, lemma_mul_base_spec, lemma_mul_unreduced_value, lemma_square_spec, + lemma_u_from_wide, sched, u_from_wide, u_xor, wide, +}; +use super::{F192Unreduced, F192}; +#[cfg(verus_keep_ghost)] +use crate::clmul::clmul; +#[cfg(verus_keep_ghost)] +use crate::gf2_64::{k_mod, k_mul, lemma_clmul_fold_reduction, lemma_k_mod_xor}; +use crate::gf2_64::{F64, R64}; +#[cfg(verus_keep_ghost)] +use crate::intrinsics::aarch64::*; +#[cfg(verus_keep_ghost)] +use crate::intrinsics::aarch64_gfneon::*; +use crate::neon::xor3_u64; +use core::arch::aarch64::*; +use core::mem::transmute; +use core::mem::MaybeUninit; +use core::ops::{Add, BitXor, BitXorAssign, Mul}; +use vstd::prelude::*; +#[cfg(verus_keep_ghost)] +use vstd::std_specs::maybe_uninit::MaybeUninitAdditionalSpecFns; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Registers as 128-bit polynomials +// --------------------------------------------------------------------------------------------- +/// The two lanes of a register as one 128-bit polynomial, lane 0 the low word. +pub open spec fn v128(v: uint64x2_t) -> u128 { + wide(u64x2(v)) +} + +/// The unreduced value whose three coefficients are the registers `d`. +pub open spec fn x1_unreduced(d: [uint64x2_t; 3]) -> F192Unreduced { + F192Unreduced { coeffs: [u64x2(d[0]), u64x2(d[1]), u64x2(d[2])] } +} + +/// The three y-folded schoolbook coefficients of `a * b`, as the portable `software::mul_unreduced` builds them. +pub open spec fn folded(a: F192, b: F192) -> (u128, u128, u128) { + let e = sched(seq![a.c0, a.c1, a.c2], seq![b.c0, b.c1, b.c2], 9); + (e[0] ^ e[3], e[1] ^ e[3] ^ e[4], e[2] ^ e[4]) +} + +/// The words of a 128-bit polynomial, and the polynomial of two words. +proof fn lemma_wide_words(x: u128, w: [u64; 2]) + ensures + wide(w) as u64 == w[0], + (wide(w) >> 64u128) as u64 == w[1], + wide([x as u64, (x >> 64u128) as u64]) == x, +{ + let (w0, w1) = (w[0], w[1]); + assert((((w1 as u128) << 64u128) | (w0 as u128)) as u64 == w0 && ((((w1 as u128) << 64u128) | (w0 as u128)) + >> 64u128) as u64 == w1) by (bit_vector); + assert((((((x >> 64u128) as u64) as u128) << 64u128) | ((x as u64) as u128)) == x) by (bit_vector); +} + +/// A lane-wise XOR is the XOR of the polynomials. +proof fn lemma_wide_xor(a: [u64; 2], b: [u64; 2], r: [u64; 2]) + requires + r[0] == a[0] ^ b[0], + r[1] == a[1] ^ b[1], + ensures + wide(r) == wide(a) ^ wide(b), +{ + let (a0, a1, b0, b1) = (a[0], a[1], b[0], b[1]); + assert((((a1 ^ b1) as u128) << 64u128) | ((a0 ^ b0) as u128) == ((((a1 as u128) << 64u128) | (a0 as u128)) ^ ( + ((b1 as u128) << 64u128) | (b0 as u128)))) by (bit_vector); +} + +/// The y-folded coefficients reduce to the product. +proof fn lemma_folded_value(a: F192, b: F192) + ensures + ({ + let (d0, d1, d2) = folded(a, b); + F192 { c0: k_mod(d0), c1: k_mod(d1), c2: k_mod(d2) } == e_mul(a, b) + }), +{ + let (d0, d1, d2) = folded(a, b); + lemma_mul_unreduced_value(a, b); + lemma_u_from_wide(d0, d1, d2); +} + +/// The nine products, folded as [`products`] does, are [`folded`]. +proof fn lemma_products_fold(a: F192, b: F192) + ensures + ({ + let c = |x: u64, y: u64| clmul(x, y as u128); + let y3 = c(a.c1, b.c2) ^ c(a.c2, b.c1); + let (d0, d1, d2) = folded(a, b); + &&& c(a.c0, b.c0) ^ y3 == d0 + &&& c(a.c0, b.c1) ^ c(a.c1, b.c0) ^ (y3 ^ c(a.c2, b.c2)) == d1 + &&& c(a.c0, b.c2) ^ c(a.c1, b.c1) ^ (c(a.c2, b.c0) ^ c(a.c2, b.c2)) == d2 + }), +{ + let (sa, sb) = (seq![a.c0, a.c1, a.c2], seq![b.c0, b.c1, b.c2]); + super::lemma_sched_9(sa, sb); + let c = |x: u64, y: u64| clmul(x, y as u128); + let (c00, c01, c02, c10, c11, c12, c20, c21, c22) = ( + c(a.c0, b.c0), + c(a.c0, b.c1), + c(a.c0, b.c2), + c(a.c1, b.c0), + c(a.c1, b.c1), + c(a.c1, b.c2), + c(a.c2, b.c0), + c(a.c2, b.c1), + c(a.c2, b.c2), + ); + assert(c00 ^ (c12 ^ c21) == (0 ^ c00) ^ (0 ^ c12 ^ c21)) by (bit_vector); + assert(c01 ^ c10 ^ ((c12 ^ c21) ^ c22) == (0 ^ c01 ^ c10) ^ (0 ^ c12 ^ c21) ^ (0 ^ c22)) by (bit_vector); + assert(c02 ^ c11 ^ (c20 ^ c22) == (0 ^ c02 ^ c11 ^ c20) ^ (0 ^ c22)) by (bit_vector); +} + +// --------------------------------------------------------------------------------------------- +// The kernels +// --------------------------------------------------------------------------------------------- +/// Carry-less product of the low qwords. +#[inline(always)] +fn lo(a: uint64x2_t, b: uint64x2_t) -> (r: uint64x2_t) + ensures + v128(r) == clmul(u64x2(a)[0], u64x2(b)[0] as u128), +{ + proof { + broadcast use axiom_u128_as_u64x2; + lemma_wide_words(clmul(u64x2(a)[0], u64x2(b)[0] as u128), u64x2(a)); + } + // SAFETY: the module's cfg enables aes, and both sides of the reinterpret are 128 bits. + unsafe { transmute::(vmull_p64(vgetq_lane_u64::<0>(a), vgetq_lane_u64::<0>(b))) } +} + +/// Carry-less product of the high qwords. +#[inline(always)] +fn hi(a: uint64x2_t, b: uint64x2_t) -> (r: uint64x2_t) + ensures + v128(r) == clmul(u64x2(a)[1], u64x2(b)[1] as u128), +{ + proof { + broadcast use axiom_u128_as_u64x2; + lemma_wide_words(clmul(u64x2(a)[1], u64x2(b)[1] as u128), u64x2(a)); + } + // SAFETY: the module's cfg enables aes, and both sides of the reinterpret are 128 bits. + unsafe { transmute::(vmull_high_p64(vreinterpretq_p64_u64(a), vreinterpretq_p64_u64(b))) } +} + +/// Two-way XOR. +#[inline(always)] +fn xor(a: uint64x2_t, b: uint64x2_t) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == u64x2(a)[0] ^ u64x2(b)[0], + u64x2(r)[1] == u64x2(a)[1] ^ u64x2(b)[1], + v128(r) == v128(a) ^ v128(b), +{ + // SAFETY: NEON is part of the aarch64 baseline. + let r = unsafe { veorq_u64(a, b) }; + proof { + lemma_wide_xor(u64x2(a), u64x2(b), u64x2(r)); + } + r +} + +/// Three-way XOR. +/// +/// Rewritten for Verus: the result is bound to a name, for the proof. +#[inline(always)] +fn xor3(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> (r: uint64x2_t) + ensures + v128(r) == v128(a) ^ v128(b) ^ v128(c), +{ + // SAFETY: the helper issues `EOR3` only where the cfg enables it. + let r = unsafe { xor3_u64(a, b, c) }; + proof { + let (a0, a1, b0, b1) = (u64x2(a)[0], u64x2(a)[1], u64x2(b)[0], u64x2(b)[1]); + let ab = [a0 ^ b0, a1 ^ b1]; + lemma_wide_xor(u64x2(a), u64x2(b), ab); + lemma_wide_xor(ab, u64x2(c), u64x2(r)); + } + r +} + +/// An element as its two registers. +#[inline(always)] +fn split(e: F192) -> (r: (uint64x2_t, uint64x2_t)) + ensures + u64x2(r.0)[0] == e.c0, + u64x2(r.0)[1] == e.c1, + u64x2(r.1)[0] == e.c2, + u64x2(r.1)[1] == e.c2, +{ + // SAFETY: NEON is part of the aarch64 baseline. + unsafe { (vcombine_u64(vcreate_u64(e.c0), vcreate_u64(e.c1)), vdupq_n_u64(e.c2)) } +} + +/// The element whose coefficients are the low qwords of `c0`, `c1`, `c2`. +#[inline(always)] +fn join(c0: uint64x2_t, c1: uint64x2_t, c2: uint64x2_t) -> (r: F192) + ensures + r == (F192 { c0: u64x2(c0)[0], c1: u64x2(c1)[0], c2: u64x2(c2)[0] }), +{ + // SAFETY: NEON is part of the aarch64 baseline. + unsafe { + // One zip puts c0 and c1 side by side, so a store of the element is two stores. + let c01 = vzip1q_u64(c0, c1); + F192::new( + vgetq_lane_u64::<0>(c01), + vgetq_lane_u64::<1>(c01), + vgetq_lane_u64::<0>(c2), + ) + } +} + +/// Reduce one 128-bit coefficient into its low qword. +/// +/// Write the coefficient as `lo + hi * x^64`, with `x^64 = 0x1B`: +/// +/// ```text +/// hi * 0x1B = [a, s] s has at most 4 bits +/// s * 0x1B = [b, 0] at most 8 bits, so the fold ends here +/// result = lo ^ a ^ b +/// ``` +/// +/// Two high-lane products and one three-way XOR. +#[inline(always)] +fn reduce_lane(d: uint64x2_t, r: uint64x2_t) -> (res: uint64x2_t) + requires + u64x2(r)[1] == R64, + ensures + u64x2(res)[0] == k_mod(v128(d)), +{ + let t = hi(d, r); + proof { + lemma_wide_words(0, u64x2(d)); + lemma_wide_words(0, u64x2(t)); + lemma_clmul_fold_reduction(v128(d)); + } + let res = xor3(d, t, hi(t, r)); + proof { + lemma_wide_words(0, u64x2(res)); + } + res +} + +/// Reduce the three coefficients of an unreduced element. +/// +/// Rewritten for Verus: production destructures the array in the parameter (`[d0, d1, d2]: [uint64x2_t; 3]`); +/// the copy indexes it. +#[inline(always)] +fn reduce3(d: [uint64x2_t; 3]) -> (r: F192) + ensures + r == (F192 { c0: k_mod(v128(d[0])), c1: k_mod(v128(d[1])), c2: k_mod(v128(d[2])) }), + r == e_value(x1_unreduced(d)), +{ + let (d0, d1, d2) = (d[0], d[1], d[2]); + // SAFETY: NEON is part of the aarch64 baseline. + let r = unsafe { vdupq_n_u64(R64) }; + join(reduce_lane(d0, r), reduce_lane(d1, r), reduce_lane(d2, r)) +} + +/// The y-folded schoolbook product: nine base products, each read straight from the operand registers. +/// +/// Karatsuba saves three products but needs shuffles to build its operand sums. +/// Here a product costs an XOR, so the nine products are cheaper. +/// +/// The one shuffle swaps the halves of the first operand. +/// A loop with a shared first factor hoists it. +/// +/// ```text +/// y^0: a0 b0 + a1 b2 + a2 b1 (y^3 = y + 1) +/// y^1: a0 b1 + a1 b0 + a1 b2 + a2 b1 + a2 b2 (y^4 = y^2 + y) +/// y^2: a0 b2 + a1 b1 + a2 b0 + a2 b2 +/// ``` +#[inline(always)] +fn schoolbook(a: F192, b: F192) -> (r: [uint64x2_t; 3]) + ensures + v128(r[0]) == folded(a, b).0, + v128(r[1]) == folded(a, b).1, + v128(r[2]) == folded(a, b).2, +{ + let ((a01, a22), (b01, b22)) = (split(a), split(b)); + products(a01, a22, b01, b22) +} + +/// The element a pair of registers holds: `c0, c1` in the first, `c2` in the low lane of the second. +pub open spec fn regs_value(c01: uint64x2_t, c22: uint64x2_t) -> F192 { + F192 { c0: u64x2(c01)[0], c1: u64x2(c01)[1], c2: u64x2(c22)[0] } +} + +/// [`schoolbook`] of two elements already in their registers. +#[inline(always)] +fn products(a01: uint64x2_t, a22: uint64x2_t, b01: uint64x2_t, b22: uint64x2_t) -> (r: [uint64x2_t; 3]) + requires + u64x2(a22)[1] == u64x2(a22)[0], + u64x2(b22)[1] == u64x2(b22)[0], + ensures + v128(r[0]) == folded(regs_value(a01, a22), regs_value(b01, b22)).0, + v128(r[1]) == folded(regs_value(a01, a22), regs_value(b01, b22)).1, + v128(r[2]) == folded(regs_value(a01, a22), regs_value(b01, b22)).2, +{ + // SAFETY: NEON is part of the aarch64 baseline. + let a10 = unsafe { vextq_u64::<1>(a01, a01) }; + // a_i b_j for every pair, named by (i, j). + let (m00, m11) = (lo(a01, b01), hi(a01, b01)); + let (m10, m01) = (lo(a10, b01), hi(a10, b01)); + let (m02, m12) = (lo(a01, b22), hi(a01, b22)); + let (m20, m21) = (lo(a22, b01), hi(a22, b01)); + let m22 = lo(a22, b22); + proof { + assert(u64x2(a10)[0] == u64x2(a01)[1] && u64x2(a10)[1] == u64x2(a01)[0]); + lemma_products_fold(regs_value(a01, a22), regs_value(b01, b22)); + } + // Both y^3 terms land on y^0 and y^1. + let y3 = xor(m12, m21); + [ + xor(m00, y3), + xor3(m01, m10, xor(y3, m22)), + xor3(m02, m11, xor(m20, m22)), + ] +} + +/// The product, reduced. +#[inline(always)] +pub fn mul(a: F192, b: F192) -> (r: F192) + ensures + r == e_mul(a, b), +{ + proof { + lemma_folded_value(a, b); + } + reduce3(schoolbook(a, b)) +} + +/// Three 128-bit coefficients as an unreduced element. +/// +/// Rewritten for Verus: production maps the transmute over the array (`d.map(..)`), which Verus does not +/// support; the copy applies it to each coefficient. +#[inline(always)] +fn unreduced(d: [uint64x2_t; 3]) -> (r: F192Unreduced) + ensures + r == x1_unreduced(d), +{ + F192Unreduced { + coeffs: [ + // SAFETY: `uint64x2_t` and `[u64; 2]` are both 16 plain bytes, valid for every bit pattern. + unsafe { transmute::(d[0]) }, + unsafe { transmute::(d[1]) }, + unsafe { transmute::(d[2]) }, + ], + } +} + +/// The product without the base-field reduction. +#[inline(always)] +pub fn mul_unreduced(a: F192, b: F192) -> (r: F192Unreduced) + ensures + e_value(r) == e_mul(a, b), +{ + proof { + lemma_folded_value(a, b); + } + unreduced(schoolbook(a, b)) +} + +/// The mixed product by a base-field scalar, without the reduction: three base products. +#[inline(always)] +fn mul_base_lanes(a: F192, k: F64) -> (r: [uint64x2_t; 3]) + ensures + v128(r[0]) == clmul(a.c0, k.0 as u128), + v128(r[1]) == clmul(a.c1, k.0 as u128), + v128(r[2]) == clmul(a.c2, k.0 as u128), +{ + let (a01, a22) = split(a); + // SAFETY: NEON is part of the aarch64 baseline. + let kk = unsafe { vdupq_n_u64(k.0) }; + [lo(a01, kk), hi(a01, kk), lo(a22, kk)] +} + +/// The mixed product by a base-field scalar, reduced. +#[inline(always)] +pub fn mul_base(a: F192, k: F64) -> (r: F192) + ensures + r == e_mul(a, e_from_k(k.0)), +{ + proof { + lemma_mul_base_spec(a, k.0); + } + reduce3(mul_base_lanes(a, k)) +} + +/// The mixed product by a base-field scalar, without the reduction. +#[inline(always)] +pub fn mul_base_unreduced(a: F192, k: F64) -> (r: F192Unreduced) + ensures + e_value(r) == e_mul(a, e_from_k(k.0)), +{ + proof { + lemma_mul_base_spec(a, k.0); + } + unreduced(mul_base_lanes(a, k)) +} + +/// The square: three base squares, then the y-fold `y^4 = y^2 + y`. +#[inline(always)] +pub fn square(a: F192) -> (r: F192) + ensures + r == e_mul(a, a), +{ + let (a01, a22) = split(a); + let (s0, s1, s2) = (lo(a01, a01), hi(a01, a01), lo(a22, a22)); + proof { + lemma_square_spec(a); + lemma_k_mod_xor(v128(s1), v128(s2)); + } + reduce3([s0, s2, xor(s1, s2)]) +} + +/// Reduce an unreduced element. +/// +/// Rewritten for Verus: production maps the transmute over the array (`u.coeffs.map(..)`); the copy applies +/// it to each coefficient. +#[inline(always)] +pub fn reduce(u: F192Unreduced) -> (r: F192) + ensures + r == e_value(u), +{ + proof { + broadcast use axiom_words_as_u64x2; + } + // SAFETY: a `[u64; 2]` and a 128-bit register hold the same bits. + reduce3( + [ + unsafe { transmute::<[u64; 2], uint64x2_t>(u.coeffs[0]) }, + unsafe { transmute::<[u64; 2], uint64x2_t>(u.coeffs[1]) }, + unsafe { transmute::<[u64; 2], uint64x2_t>(u.coeffs[2]) }, + ], + ) +} + +// --------------------------------------------------------------------------------------------- +// Memory helpers (trusted, tested in `tests/equivalence/intrinsics_aarch64_gfneon.rs`) +// --------------------------------------------------------------------------------------------- +/// The loads of [`F192x1::load`], production's expression: `c0, c1` into one register, `c2` into both lanes of +/// the other. +#[verifier::external_body] +#[inline(always)] +pub fn load_f192(e: &F192) -> (r: (uint64x2_t, uint64x2_t)) + ensures + u64x2(r.0)[0] == e.c0, + u64x2(r.0)[1] == e.c1, + u64x2(r.1)[0] == e.c2, + u64x2(r.1)[1] == e.c2, +{ + let words = (e as *const F192).cast::(); + // SAFETY: `e` is three words, `c0` and `c1` adjacent under `repr(C)`. + unsafe { (vld1q_u64(words), vld1q_dup_u64(words.add(2))) } +} + +/// The stores of [`F192x1::store`], production's statements: `c0, c1` from the first register, `c2` from the +/// low lane of the second. All three words are written, so `out` is initialized afterwards. +#[verifier::external_body] +#[inline(always)] +pub fn store_f192(out: &mut MaybeUninit, c01: uint64x2_t, c22: uint64x2_t) + ensures + final(out).as_option() == Some(regs_value(c01, c22)), +{ + let words = out.as_mut_ptr().cast::(); + // SAFETY: `out` is three words, `c0` and `c1` adjacent under `repr(C)`. + unsafe { + vst1q_u64(words, c01); + vst1q_lane_u64::<0>(words.add(2), c22); + } +} + +// --------------------------------------------------------------------------------------------- +// Register-resident elements +// --------------------------------------------------------------------------------------------- +/// One element in its two registers, for a kernel whose values stay there from load to store. +/// +/// An [`F192`] is three integer words: every sum of two runs on the integer side, and every +/// product moves its operands over and its result back, which costs more than the product. +/// These sums and products stay in vector registers. Keep them as named values or tuples: an +/// array of them is a memory object. +/// +/// Its type invariant (a Verus addition, no runtime cost): both lanes of `c22` hold `c2`, which the high-lane +/// products of [`products`] read. +#[derive(Clone, Copy)] +pub struct F192x1 { + c01: uint64x2_t, + c22: uint64x2_t, +} + +/// An unreduced product of [`F192x1`] values, or a sum of them: its three 128-bit coefficients +/// in registers. +#[derive(Clone, Copy)] +pub struct F192x1Unreduced([uint64x2_t; 3]); + +impl F192x1 { + /// The element the registers hold. + pub closed spec fn value(self) -> F192 { + regs_value(self.c01, self.c22) + } + + #[verifier::type_invariant] + spec fn inv(self) -> bool { + u64x2(self.c22)[1] == u64x2(self.c22)[0] + } + + #[inline(always)] + pub fn new(e: F192) -> (r: Self) + ensures + r.value() == e, + { + let (c01, c22) = split(e); + Self { c01, c22 } + } + + /// The element at `e`, loaded into its registers. + /// + /// Rewritten for Verus: the two loads through the cast pointer are in [`load_f192`], whose body is + /// production's expression. + #[inline(always)] + pub fn load(e: &F192) -> (r: Self) + ensures + r.value() == *e, + { + let (c01, c22) = load_f192(e); + Self { c01, c22 } + } + + /// Write the element to `out`, which need not be initialized. + /// + /// Rewritten for Verus: the two stores through the cast pointer are in [`store_f192`], whose body is + /// production's statements. + #[inline(always)] + pub fn store(self, out: &mut MaybeUninit) + ensures + final(out).as_option() == Some(self.value()), + { + store_f192(out, self.c01, self.c22) + } + + /// The product without the reduction. + #[inline(always)] + pub fn mul_unreduced(self, rhs: Self) -> (r: F192x1Unreduced) + ensures + e_value(r.value()) == e_mul(self.value(), rhs.value()), + { + proof { + use_type_invariant(&self); + use_type_invariant(&rhs); + lemma_folded_value(self.value(), rhs.value()); + } + F192x1Unreduced(products(self.c01, self.c22, rhs.c01, rhs.c22)) + } + + /// The mixed product by a base-field scalar, without the reduction. + #[inline(always)] + pub fn mul_base_unreduced(self, k: F64) -> (r: F192x1Unreduced) + ensures + e_value(r.value()) == e_mul(self.value(), e_from_k(k.0)), + { + proof { + lemma_mul_base_spec(self.value(), k.0); + } + // SAFETY: NEON is part of the aarch64 baseline. + let kk = unsafe { vdupq_n_u64(k.0) }; + F192x1Unreduced([lo(self.c01, kk), hi(self.c01, kk), lo(self.c22, kk)]) + } +} + +// The operator traits' specifications: the results are stated by each impl's `ensures`, through the closed +// `value` views, so the generic `*_spec` functions are left unused. +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::AddSpecImpl for F192x1 { + open spec fn obeys_add_spec() -> bool { + false + } + + open spec fn add_req(self, rhs: F192x1) -> bool { + true + } + + open spec fn add_spec(self, rhs: F192x1) -> F192x1 { + arbitrary() + } +} + +impl Add for F192x1 { + type Output = Self; + + #[inline(always)] + fn add(self, rhs: Self) -> (r: Self) + ensures + r.value() == e_add(self.value(), rhs.value()), + { + proof { + use_type_invariant(&self); + use_type_invariant(&rhs); + } + Self { c01: xor(self.c01, rhs.c01), c22: xor(self.c22, rhs.c22) } + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::MulSpecImpl for F192x1 { + open spec fn obeys_mul_spec() -> bool { + false + } + + open spec fn mul_req(self, rhs: F192x1) -> bool { + true + } + + open spec fn mul_spec(self, rhs: F192x1) -> F192x1 { + arbitrary() + } +} + +impl Mul for F192x1 { + type Output = Self; + + #[inline(always)] + fn mul(self, rhs: Self) -> (r: Self) + ensures + r.value() == e_mul(self.value(), rhs.value()), + { + self.mul_unreduced(rhs).reduce() + } +} + +impl F192x1Unreduced { + /// The unreduced value the registers hold. + pub closed spec fn value(self) -> F192Unreduced { + x1_unreduced(self.0) + } + + /// Rewritten for Verus: production fills the array with a repeat expression (`[vdupq_n_u64(0); 3]`); the + /// copy lists the three registers. + #[inline(always)] + pub fn zero() -> (r: Self) + ensures + r.value() == F192Unreduced::ZERO, + { + // SAFETY: NEON is part of the aarch64 baseline. + let z = unsafe { vdupq_n_u64(0) }; + proof { + assert(u64x2(z) =~= [0u64, 0u64]); + } + let r = Self([z, z, z]); + proof { + assert(r.value().coeffs =~= F192Unreduced::ZERO.coeffs); + } + r + } + + /// Reduce each coefficient, into the element's two registers. + /// + /// Rewritten for Verus: production destructures the array (`let [d0, d1, d2] = self.0`); the copy indexes it. + #[inline(always)] + pub fn reduce(self) -> (r: F192x1) + ensures + r.value() == e_value(self.value()), + { + let (d0, d1, d2) = (self.0[0], self.0[1], self.0[2]); + // SAFETY: NEON is part of the aarch64 baseline. + unsafe { + let r = vdupq_n_u64(R64); + let (c0, c1, c2) = (reduce_lane(d0, r), reduce_lane(d1, r), reduce_lane(d2, r)); + F192x1 { + c01: vzip1q_u64(c0, c1), + c22: vdupq_laneq_u64::<0>(c2), + } + } + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::BitXorSpecImpl for F192x1Unreduced { + open spec fn obeys_bitxor_spec() -> bool { + false + } + + open spec fn bitxor_req(self, rhs: F192x1Unreduced) -> bool { + true + } + + open spec fn bitxor_spec(self, rhs: F192x1Unreduced) -> F192x1Unreduced { + arbitrary() + } +} + +impl BitXor for F192x1Unreduced { + type Output = Self; + + /// Rewritten for Verus: production destructures both arrays (`let ([a0, a1, a2], [b0, b1, b2]) = ..`); the + /// copy indexes them. + #[inline(always)] + fn bitxor(self, rhs: Self) -> (r: Self) + ensures + r.value() == u_xor(self.value(), rhs.value()), + { + let (a0, a1, a2, b0, b1, b2) = (self.0[0], self.0[1], self.0[2], rhs.0[0], rhs.0[1], rhs.0[2]); + let r = Self([xor(a0, b0), xor(a1, b1), xor(a2, b2)]); + proof { + let u = u_xor(self.value(), rhs.value()); + assert(u64x2(r.0[0]) =~= u.coeffs[0]); + assert(u64x2(r.0[1]) =~= u.coeffs[1]); + assert(u64x2(r.0[2]) =~= u.coeffs[2]); + assert(r.value().coeffs =~= u.coeffs); + } + r + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::BitXorAssignSpecImpl for F192x1Unreduced { + open spec fn obeys_bitxor_assign_spec() -> bool { + false + } + + open spec fn bitxor_assign_req(&self, rhs: F192x1Unreduced) -> bool { + true + } + + open spec fn bitxor_assign_spec(&self, rhs: F192x1Unreduced) -> &F192x1Unreduced { + arbitrary() + } +} + +impl BitXorAssign for F192x1Unreduced { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) + ensures + final(self).value() == u_xor(old(self).value(), rhs.value()), + { + *self = *self ^ rhs; + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::convert::FromSpecImpl for F192Unreduced { + open spec fn obeys_from_spec() -> bool { + true + } + + open spec fn from_spec(u: F192x1Unreduced) -> F192Unreduced { + u.value() + } +} + +impl From for F192Unreduced { + #[inline(always)] + fn from(u: F192x1Unreduced) -> (r: F192Unreduced) + ensures + r == u.value(), + { + unreduced(u.0) + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::convert::FromSpecImpl for F192 { + open spec fn obeys_from_spec() -> bool { + true + } + + open spec fn from_spec(e: F192x1) -> F192 { + e.value() + } +} + +impl From for F192 { + #[inline(always)] + fn from(e: F192x1) -> (r: F192) + ensures + r == e.value(), + { + // SAFETY: NEON is part of the aarch64 baseline. + unsafe { + Self::new( + vgetq_lane_u64::<0>(e.c01), + vgetq_lane_u64::<1>(e.c01), + vgetq_lane_u64::<0>(e.c22), + ) + } + } +} + +} // verus! diff --git a/verification/verus/src/gf2_64x3_x86.rs b/verification/verus/src/gf2_64x3_x86.rs new file mode 100644 index 0000000000..45146455eb --- /dev/null +++ b/verification/verus/src/gf2_64x3_x86.rs @@ -0,0 +1,3057 @@ +//! The x86-64 kernels of `E = GF(2^192)`: a copy of `crates/primitives/src/field/gf2_64x3/x86_64.rs`. +//! +//! Every kernel is proven to compute what the portable path computes, given the intrinsic specifications of +//! `crate::intrinsics::x86` and `crate::intrinsics::x86_gfx86`: a product's three 128-bit coefficients are the +//! schoolbook's ([`prod_coeffs`]), so they reduce to [`e_mul`]; a lane-wise reduction is [`k_mod`] of each +//! coefficient; a sum of products in registers reduces to the sum of the products. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +use super::Weights8; +#[cfg(verus_keep_ghost)] +use super::{ + dot_spec, e_add, e_from_k, e_mul, e_value, lemma_e_value_xor, lemma_karatsuba_fold, lemma_mul_base_spec, + lemma_mul_unreduced_value, lemma_u_from_wide, lemma_u_zero, lemma_wide_split, sched, u_from_wide, u_xor, w8_get, + wide, +}; +use super::{F192Unreduced, F192}; +#[cfg(verus_keep_ghost)] +use crate::clmul::clmul; +use crate::gf2_64::F64; +#[cfg(verus_keep_ghost)] +use crate::gf2_64::{k_mod, k_mul, lemma_k_mod, lemma_k_mod_xor, reduce_formula}; +#[cfg(verus_keep_ghost)] +use crate::intrinsics::x86::*; +#[cfg(verus_keep_ghost)] +use crate::intrinsics::x86_gfx86::*; +use core::arch::x86_64::*; +use core::mem::transmute; +use core::mem::MaybeUninit; +use core::ops::{Add, BitXor, BitXorAssign, Mul}; +use vstd::prelude::*; +#[cfg(verus_keep_ghost)] +use vstd::raw_ptr::MemContents; +#[cfg(verus_keep_ghost)] +use vstd::std_specs::maybe_uninit::MaybeUninitAdditionalSpecFns; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specifications +// --------------------------------------------------------------------------------------------- +/// A 128-bit coefficient as its `[low, high]` words. +pub open spec fn split(c: u128) -> [u64; 2] { + [c as u64, (c >> 64u128) as u64] +} + +/// The three y-folded 128-bit coefficients of `a * b`: what the schoolbook `software::mul_unreduced` builds. +pub open spec fn prod_coeffs(a: F192, b: F192) -> [u128; 3] { + let e = sched(seq![a.c0, a.c1, a.c2], seq![b.c0, b.c1, b.c2], 9); + [e[0] ^ e[3], e[1] ^ e[3] ^ e[4], e[2] ^ e[4]] +} + +/// The unreduced product as [`F192Unreduced`] holds it. +pub open spec fn prod_unreduced(a: F192, b: F192) -> F192Unreduced { + u_from_wide(prod_coeffs(a, b)[0], prod_coeffs(a, b)[1], prod_coeffs(a, b)[2]) +} + +/// The unreduced mixed product by `k`: the three coefficients times `k`. +pub open spec fn base_unreduced(a: F192, k: u64) -> F192Unreduced { + u_from_wide(clmul(a.c0, k as u128), clmul(a.c1, k as u128), clmul(a.c2, k as u128)) +} + +/// The base-field reduction of `hi * x^64 + lo`. +pub open spec fn reduce_word(lo: u64, hi: u64) -> u64 { + k_mod(wide([lo, hi])) +} + +/// The kernels' shift network on one word pair, with the shifts as the intrinsics specify them: +/// `lo ^ f(hi ^ spill)`, `spill = hi >> 63 ^ hi >> 61 ^ hi >> 60`, `f(v) = v ^ v << 1 ^ v << 3 ^ v << 4`. +pub open spec fn reduce_net(lo: u64, hi: u64) -> u64 { + let spill = shr_imm(hi, 63) ^ shr_imm(hi, 61) ^ shr_imm(hi, 60); + let v = hi ^ spill; + lo ^ ((v ^ shl_imm(v, 1)) ^ (shl_imm(v, 3) ^ shl_imm(v, 4))) +} + +// --------------------------------------------------------------------------------------------- +// Lemmas +// --------------------------------------------------------------------------------------------- +pub proof fn lemma_i64_round_trip(a: u64) + ensures + (a as i64) as u64 == a, +{ + assert((a as i64) as u64 == a) by (bit_vector); +} + +/// The words of a XOR are the XORs of the words. +pub proof fn lemma_split_xor(x: u128, y: u128) + ensures + split(x ^ y)[0] == split(x)[0] ^ split(y)[0], + split(x ^ y)[1] == split(x)[1] ^ split(y)[1], +{ + assert((x ^ y) as u64 == (x as u64) ^ (y as u64) && ((x ^ y) >> 64u128) as u64 == ((x >> 64u128) as u64) ^ ((y + >> 64u128) as u64)) by (bit_vector); +} + +/// The shift network is the reduction. +pub proof fn lemma_reduce_net(lo: u64, hi: u64) + ensures + reduce_net(lo, hi) == reduce_word(lo, hi), +{ + let p = wide([lo, hi]); + lemma_k_mod(p); + assert(p == ((hi as u128) << 64u128) | (lo as u128)); + assert(p == ((hi as u128) << 64u128) | (lo as u128) ==> { + let spill = (hi >> 63u64) ^ (hi >> 61u64) ^ (hi >> 60u64); + let v = hi ^ spill; + reduce_formula(p) == lo ^ ((v ^ (v << 1u64)) ^ ((v << 3u64) ^ (v << 4u64))) + }) by (bit_vector); +} + +/// The schoolbook coefficients reduce to the product. +pub proof fn lemma_prod_unreduced(a: F192, b: F192) + ensures + e_value(prod_unreduced(a, b)) == e_mul(a, b), +{ + lemma_mul_unreduced_value(a, b); +} + +/// The mixed product's coefficients reduce to the product by `k`. +pub proof fn lemma_base_unreduced(a: F192, k: u64) + ensures + e_value(base_unreduced(a, k)) == e_mul(a, e_from_k(k)), +{ + lemma_u_from_wide(clmul(a.c0, k as u128), clmul(a.c1, k as u128), clmul(a.c2, k as u128)); + lemma_mul_base_spec(a, k); +} + +/// Word `w` of the carry-less product `x * y`. +pub open spec fn cw(x: u64, y: u64, w: int) -> u64 { + split(clmul(x, y as u128))[w] +} + +/// Word `w` of coefficient `k` as Karatsuba's six products and `fold` give it. +pub open spec fn kara_word(a: F192, b: F192, k: int, w: int) -> u64 { + let p0 = cw(a.c0, b.c0, w); + let p1 = cw(a.c1, b.c1, w); + let p2 = cw(a.c2, b.c2, w); + let p01 = cw(a.c0 ^ a.c1, b.c0 ^ b.c1, w); + let p02 = cw(a.c0 ^ a.c2, b.c0 ^ b.c2, w); + let p12 = cw(a.c1 ^ a.c2, b.c1 ^ b.c2, w); + if k == 0 { + p0 ^ p12 ^ p1 ^ p2 + } else if k == 1 { + p0 ^ p12 ^ p01 + } else { + p0 ^ p1 ^ p02 + } +} + +/// Karatsuba's six products give the schoolbook coefficients, word by word. +pub proof fn lemma_karatsuba_words(a: F192, b: F192) + ensures + forall|k: int, w: int| + 0 <= k < 3 && 0 <= w < 2 ==> #[trigger] split(prod_coeffs(a, b)[k])[w] == kara_word(a, b, k, w), +{ + lemma_karatsuba_fold(a, b); + let c = |x: u64, y: u64| clmul(x, y as u128); + let (p0, p1, p2) = (c(a.c0, b.c0), c(a.c1, b.c1), c(a.c2, b.c2)); + let p01 = c(a.c0 ^ a.c1, b.c0 ^ b.c1); + let p02 = c(a.c0 ^ a.c2, b.c0 ^ b.c2); + let p12 = c(a.c1 ^ a.c2, b.c1 ^ b.c2); + lemma_split_xor(p0, p12); + lemma_split_xor(p0 ^ p12, p1); + lemma_split_xor(p0 ^ p12 ^ p1, p2); + lemma_split_xor(p0 ^ p12, p01); + lemma_split_xor(p0, p1); + lemma_split_xor(p0 ^ p1, p02); +} + +/// `clmul_words` is the split of the product. +pub proof fn lemma_clmul_words(a: u64, b: u64) + ensures + clmul_words(a, b) == split(clmul(a, b as u128)), +{ +} + +// --------------------------------------------------------------------------------------------- +// The kernels +// --------------------------------------------------------------------------------------------- +/// Two 64-bit words as one register, `lo` in the low qword. +#[inline(always)] +fn pair(lo: u64, hi: u64) -> (r: __m128i) + ensures + m128(r)[0] == lo, + m128(r)[1] == hi, +{ + proof { + lemma_i64_round_trip(lo); + lemma_i64_round_trip(hi); + } + // SAFETY: SSE2 is part of the x86-64 baseline. + unsafe { _mm_set_epi64x(hi as i64, lo as i64) } +} + +/// The y-folded Karatsuba product of one pair. +/// +/// # Safety +/// +/// Requires the `pclmulqdq` target feature. +#[inline] +#[target_feature(enable = "pclmulqdq")] +unsafe fn karatsuba(a: F192, b: F192) -> (r: [__m128i; 3]) + ensures + forall|k: int, w: int| 0 <= k < 3 && 0 <= w < 2 ==> #[trigger] m128(r[k])[w] == split(prod_coeffs(a, b)[k])[w], +{ + proof { + lemma_clmul_sel(); + } + // Operand registers, as in the module table. + let (a01, b01) = (pair(a.c0, a.c1), pair(b.c0, b.c1)); + let (at, bt) = (pair(a.c2, a.c0 ^ a.c1), pair(b.c2, b.c0 ^ b.c1)); + // XOR the broadcast c2 into both qwords of r01. + let a_s = _mm_xor_si128(a01, _mm_unpacklo_epi64(at, at)); + let b_s = _mm_xor_si128(b01, _mm_unpacklo_epi64(bt, bt)); + // The six base products, two per register pair. + let p0 = _mm_clmulepi64_si128::<0x00>(a01, b01); + let p1 = _mm_clmulepi64_si128::<0x11>(a01, b01); + let p02 = _mm_clmulepi64_si128::<0x00>(a_s, b_s); + let p12 = _mm_clmulepi64_si128::<0x11>(a_s, b_s); + let p2 = _mm_clmulepi64_si128::<0x00>(at, bt); + let p01 = _mm_clmulepi64_si128::<0x11>(at, bt); + proof { + lemma_karatsuba_words(a, b); + assert(m128(a_s)[0] == a.c0 ^ a.c2 && m128(a_s)[1] == a.c1 ^ a.c2); + assert(m128(b_s)[0] == b.c0 ^ b.c2 && m128(b_s)[1] == b.c1 ^ b.c2); + lemma_clmul_words(a.c0, b.c0); + lemma_clmul_words(a.c1, b.c1); + lemma_clmul_words(a.c2, b.c2); + lemma_clmul_words(a.c0 ^ a.c1, b.c0 ^ b.c1); + lemma_clmul_words(a.c0 ^ a.c2, b.c0 ^ b.c2); + lemma_clmul_words(a.c1 ^ a.c2, b.c1 ^ b.c2); + } + let xor = |x: __m128i, y: __m128i| -> (z: __m128i) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] m128(z)[i] == m128(x)[i] ^ m128(y)[i], + { _mm_xor_si128(x, y) }; + let r = fold(p0, p1, p2, p01, p02, p12, xor); + proof { + lemma_fold(xor, |v: __m128i| m128(v)@, 2, p0, p1, p2, p01, p02, p12, r); + assert forall|k: int, w: int| 0 <= k < 3 && 0 <= w < 2 implies #[trigger] m128(r[k])[w] == split( + prod_coeffs(a, b)[k], + )[w] by { + assert(m128(p0)[w] == cw(a.c0, b.c0, w)); + assert(m128(p1)[w] == cw(a.c1, b.c1, w)); + assert(m128(p2)[w] == cw(a.c2, b.c2, w)); + assert(m128(p01)[w] == cw(a.c0 ^ a.c1, b.c0 ^ b.c1, w)); + assert(m128(p02)[w] == cw(a.c0 ^ a.c2, b.c0 ^ b.c2, w)); + assert(m128(p12)[w] == cw(a.c1 ^ a.c2, b.c1 ^ b.c2, w)); + assert(m128(r[k])[w] == kara_word(a, b, k, w)); + } + } + r +} + +/// Karatsuba recombination and y-fold in one step, for any register width. +/// +/// Composing the two linear maps leaves a short XOR network: +/// +/// ```text +/// d0 = p0 ^ p1 ^ p2 ^ p12 +/// d1 = p0 ^ p01 ^ p12 +/// d2 = p0 ^ p1 ^ p02 +/// ``` +/// +/// Rewritten for Verus: the two inner XORs are bound to names (`t`, `u`), which the specification quantifies over. +#[inline(always)] +fn fold(p0: V, p1: V, p2: V, p01: V, p02: V, p12: V, xor: impl Fn(V, V) -> V) -> (r: [V; 3]) + requires + forall|x: V, y: V| #[trigger] xor.requires((x, y)), + ensures + fold_post(xor, p0, p1, p2, p01, p02, p12, r), +{ + // Shared by d0 and d1. + let q = xor(p0, p12); + let t = xor(q, p1); + let u = xor(p0, p1); + [xor(t, p2), xor(q, p01), xor(u, p02)] +} + +/// [`fold`]'s result: its six XORs, as `xor` specifies them. +pub open spec fn fold_post V>(xor: F, p0: V, p1: V, p2: V, p01: V, p02: V, p12: V, r: [V; 3]) -> bool { + exists|q: V, t: V, u: V| + { + &&& #[trigger] xor.ensures((p0, p12), q) + &&& #[trigger] xor.ensures((q, p1), t) + &&& xor.ensures((t, p2), r[0]) + &&& xor.ensures((q, p01), r[1]) + &&& #[trigger] xor.ensures((p0, p1), u) + &&& xor.ensures((u, p02), r[2]) + } +} + +/// `xor` XORs the first `n` words of the `view` of its operands. +pub open spec fn xors_words V>(xor: F, view: spec_fn(V) -> Seq, n: int) -> bool { + forall|x: V, y: V, z: V, i: int| + #![trigger xor.ensures((x, y), z), view(z)[i]] + xor.ensures((x, y), z) && 0 <= i < n ==> view(z)[i] == view(x)[i] ^ view(y)[i] +} + +/// [`fold`] word by word, for a word-wise XOR. +pub proof fn lemma_fold V>( + xor: F, + view: spec_fn(V) -> Seq, + n: int, + p0: V, + p1: V, + p2: V, + p01: V, + p02: V, + p12: V, + r: [V; 3], +) + requires + xors_words(xor, view, n), + fold_post(xor, p0, p1, p2, p01, p02, p12, r), + ensures + forall|i: int| 0 <= i < n ==> #[trigger] view(r[0])[i] == view(p0)[i] ^ view(p12)[i] ^ view(p1)[i] ^ view(p2)[i], + forall|i: int| 0 <= i < n ==> #[trigger] view(r[1])[i] == view(p0)[i] ^ view(p12)[i] ^ view(p01)[i], + forall|i: int| 0 <= i < n ==> #[trigger] view(r[2])[i] == view(p0)[i] ^ view(p1)[i] ^ view(p02)[i], +{ + let (q, t, u) = choose|q: V, t: V, u: V| + { + &&& #[trigger] xor.ensures((p0, p12), q) + &&& #[trigger] xor.ensures((q, p1), t) + &&& xor.ensures((t, p2), r[0]) + &&& xor.ensures((q, p01), r[1]) + &&& #[trigger] xor.ensures((p0, p1), u) + &&& xor.ensures((u, p02), r[2]) + }; + assert forall|i: int| 0 <= i < n implies { + &&& #[trigger] view(r[0])[i] == view(p0)[i] ^ view(p12)[i] ^ view(p1)[i] ^ view(p2)[i] + &&& view(r[1])[i] == view(p0)[i] ^ view(p12)[i] ^ view(p01)[i] + &&& view(r[2])[i] == view(p0)[i] ^ view(p1)[i] ^ view(p02)[i] + } by { + assert(view(q)[i] == view(p0)[i] ^ view(p12)[i]); + assert(view(t)[i] == view(q)[i] ^ view(p1)[i]); + assert(view(u)[i] == view(p0)[i] ^ view(p1)[i]); + } +} + +/// One unreduced product. +/// +/// Rewritten for Verus: production maps the reinterpret over the three registers. +/// +/// # Safety +/// +/// Requires the `pclmulqdq` target feature. +#[inline] +#[target_feature(enable = "pclmulqdq")] +pub unsafe fn mul_unreduced(a: F192, b: F192) -> (r: F192Unreduced) + ensures + r == prod_unreduced(a, b), + e_value(r) == e_mul(a, b), +{ + // SAFETY: the function carries pclmulqdq; the reinterprets are between 128-bit values. + unsafe { + let d = karatsuba(a, b); + let r = F192Unreduced { + coeffs: [ + transmute::<__m128i, [u64; 2]>(d[0]), + transmute::<__m128i, [u64; 2]>(d[1]), + transmute::<__m128i, [u64; 2]>(d[2]), + ], + }; + proof { + assert(r == regs128(d)); + lemma_regs128(d, prod_coeffs(a, b)); + lemma_prod_unreduced(a, b); + } + r + } +} + +/// The unreduced value three 128-bit registers hold, one coefficient each. +pub open spec fn regs128(d: [__m128i; 3]) -> F192Unreduced { + F192Unreduced { coeffs: [m128(d[0]), m128(d[1]), m128(d[2])] } +} + +/// Registers holding the words of three coefficients hold their unreduced value. +pub proof fn lemma_regs128(d: [__m128i; 3], c: [u128; 3]) + requires + forall|k: int, w: int| 0 <= k < 3 && 0 <= w < 2 ==> #[trigger] m128(d[k])[w] == split(c[k])[w], + ensures + regs128(d) == u_from_wide(c[0], c[1], c[2]), +{ + let u = u_from_wide(c[0], c[1], c[2]); + assert forall|k: int| 0 <= k < 3 implies #[trigger] regs128(d).coeffs[k] == u.coeffs[k] by { + assert(m128(d[k])[0] == split(c[k])[0]); + assert(m128(d[k])[1] == split(c[k])[1]); + assert(regs128(d).coeffs[k] =~= u.coeffs[k]); + } + assert(regs128(d).coeffs =~= u.coeffs); +} + +/// One unreduced mixed product: three base products from two operand registers. +/// +/// Rewritten for Verus: production maps the reinterpret over the three registers. +/// +/// # Safety +/// +/// Requires the `pclmulqdq` target feature. +#[inline] +#[target_feature(enable = "pclmulqdq")] +pub unsafe fn mul_base_unreduced(a: F192, k: F64) -> (r: F192Unreduced) + ensures + r == base_unreduced(a, k.0), + e_value(r) == e_mul(a, e_from_k(k.0)), +{ + let ghost k0 = k.0; + proof { + lemma_clmul_sel(); + lemma_i64_round_trip(a.c2); + lemma_i64_round_trip(k.0); + } + // [c0, c1] and [c2, 0] against [k, 0]. + let a01 = pair(a.c0, a.c1); + let a2 = _mm_cvtsi64_si128(a.c2 as i64); + let k = _mm_cvtsi64_si128(k.0 as i64); + // Immediate 0x01 multiplies the high qword of the first operand by the low qword of the second. + let products = [ + _mm_clmulepi64_si128::<0x00>(a01, k), + _mm_clmulepi64_si128::<0x01>(a01, k), + _mm_clmulepi64_si128::<0x00>(a2, k), + ]; + // SAFETY: the reinterprets are between 128-bit values. + unsafe { + let r = F192Unreduced { + coeffs: [ + transmute::<__m128i, [u64; 2]>(products[0]), + transmute::<__m128i, [u64; 2]>(products[1]), + transmute::<__m128i, [u64; 2]>(products[2]), + ], + }; + proof { + lemma_clmul_words(a.c0, k0); + lemma_clmul_words(a.c1, k0); + lemma_clmul_words(a.c2, k0); + let c = [clmul(a.c0, k0 as u128), clmul(a.c1, k0 as u128), clmul(a.c2, k0 as u128)]; + assert forall|j: int, w: int| 0 <= j < 3 && 0 <= w < 2 implies #[trigger] m128(products[j])[w] == split( + c[j], + )[w] by {} + assert(r == regs128(products)); + lemma_regs128(products, c); + lemma_base_unreduced(a, k0); + } + r + } +} + +/// The batched kernels' lane-wise reduction on one register: qword `i` of the result is the +/// reduction of `hi[i] * x^64 + lo[i]`. +#[inline(always)] +fn reduce_lanes128(lo: __m128i, hi: __m128i) -> (r: __m128i) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] m128(r)[i] == reduce_word(m128(lo)[i], m128(hi)[i]), +{ + // SAFETY: SSE2 is part of the x86-64 baseline. + unsafe { + let spill = _mm_xor_si128( + _mm_xor_si128(_mm_srli_epi64::<63>(hi), _mm_srli_epi64::<61>(hi)), + _mm_srli_epi64::<60>(hi), + ); + let v = _mm_xor_si128(hi, spill); + let f = _mm_xor_si128( + _mm_xor_si128(v, _mm_slli_epi64::<1>(v)), + _mm_xor_si128(_mm_slli_epi64::<3>(v), _mm_slli_epi64::<4>(v)), + ); + let r = _mm_xor_si128(lo, f); + proof { + assert forall|i: int| 0 <= i < 2 implies #[trigger] m128(r)[i] == reduce_word(m128(lo)[i], m128(hi)[i]) by { + lemma_reduce_net(m128(lo)[i], m128(hi)[i]); + } + } + r + } +} + +/// `_mm*_shuffle_epi32::<0x4E>` swaps the two words of every 128-bit lane. +pub proof fn lemma_swap_words(a: Seq, i: int) + requires + 0 <= i < a.len(), + a.len() % 2 == 0, + ensures + shuffle_epi32_lane(a, 0x4Ei32 as u8, i) == if i % 2 == 0 { + a[i + 1] + } else { + a[i - 1] + }, +{ + assert(((0x4Eu8 >> 0u8) & 3) == 2 && ((0x4Eu8 >> 2u8) & 3) == 3 && ((0x4Eu8 >> 4u8) & 3) == 0 && ((0x4Eu8 >> 6u8) + & 3) == 1) by (bit_vector); + let (x, y) = (a[2 * (i / 2)], a[2 * (i / 2) + 1]); + assert(((x >> 0u64) & 0xFFFF_FFFFu64) | (((x >> 32u64) & 0xFFFF_FFFFu64) << 32u64) == x) by (bit_vector); + assert(((y >> 0u64) & 0xFFFF_FFFFu64) | (((y >> 32u64) & 0xFFFF_FFFFu64) << 32u64) == y) by (bit_vector); +} + +/// One element in two registers, `[c0, c1]` and `[c2, c2]`, for a kernel whose values stay +/// there from load to store, as the aarch64 kernels' `F192x1`; for x86 without the wide +/// CLMUL, where the batched products leave no lanes to fill. +/// +/// An [`F192`] is three integer words: its sums run on the integer side, and every product +/// moves its operands over and reduces its result there. These sums, products and reductions +/// stay in vector registers. +#[derive(Clone, Copy)] +pub struct F192x1 { + c01: __m128i, + c22: __m128i, +} + +/// An unreduced product of [`F192x1`] values, or a sum of them: its three 128-bit coefficients +/// in registers. +#[derive(Clone, Copy)] +pub struct F192x1Unreduced([__m128i; 3]); + +impl F192x1 { + /// The two words of `c22` are both `c2`. + #[verifier::type_invariant] + spec fn inv(self) -> bool { + m128(self.c22)[0] == m128(self.c22)[1] + } + + /// The element held. + pub closed spec fn elem(self) -> F192 { + F192 { c0: m128(self.c01)[0], c1: m128(self.c01)[1], c2: m128(self.c22)[0] } + } + + #[inline(always)] + pub fn new(e: F192) -> (r: Self) + ensures + r.elem() == e, + { + proof { + lemma_i64_round_trip(e.c2); + } + // SAFETY: SSE2 is part of the x86-64 baseline. + Self { c01: pair(e.c0, e.c1), c22: unsafe { _mm_set1_epi64x(e.c2 as i64) } } + } + + /// The element at `e`, loaded into its registers. + /// + /// Rewritten for Verus: the loads through the cast pointer are the helpers [`load_c01`] and [`load_c2`]. + #[inline(always)] + pub fn load(e: &F192) -> (r: Self) + ensures + r.elem() == *e, + { + let w2 = load_c2(e); + proof { + lemma_i64_round_trip(w2); + } + // SAFETY: `e` is three words, `c0` and `c1` adjacent under `repr(C)`. + unsafe { Self { c01: load_c01(e), c22: _mm_set1_epi64x(w2 as i64) } } + } + + /// Write the element to `out`, which need not be initialized. + /// + /// Rewritten for Verus: the two stores through the cast pointer are the helper [`store_c01_c2`]. + #[inline(always)] + pub fn store(self, out: &mut MaybeUninit) + ensures + final(out).mem_contents() == MemContents::Init(self.elem()), + { + store_c01_c2(self.c01, self.c22, out); + } + + /// The product without the reduction, Karatsuba's six from the registers. + /// + /// Rewritten for Verus: the six products and the XOR closure are bound to names before `fold`. + #[inline(always)] + pub fn mul_unreduced(self, rhs: Self) -> (r: F192x1Unreduced) + ensures + r.unreduced() == prod_unreduced(self.elem(), rhs.elem()), + e_value(r.unreduced()) == e_mul(self.elem(), rhs.elem()), + { + proof { + use_type_invariant(&self); + use_type_invariant(&rhs); + lemma_clmul_sel(); + } + let ((a01, a22), (b01, b22)) = ((self.c01, self.c22), (rhs.c01, rhs.c22)); + // SAFETY: the module's cfg enables pclmulqdq; the rest is SSE2. + unsafe { + // `[c0 + c2, c1 + c2]`, and `c0 + c1` in both qwords. + let (a_s, b_s) = (_mm_xor_si128(a01, a22), _mm_xor_si128(b01, b22)); + let a_x = _mm_xor_si128(a01, _mm_shuffle_epi32::<0x4E>(a01)); + let b_x = _mm_xor_si128(b01, _mm_shuffle_epi32::<0x4E>(b01)); + let p0 = _mm_clmulepi64_si128::<0x00>(a01, b01); + let p1 = _mm_clmulepi64_si128::<0x11>(a01, b01); + let p2 = _mm_clmulepi64_si128::<0x00>(a22, b22); + let p01 = _mm_clmulepi64_si128::<0x00>(a_x, b_x); + let p02 = _mm_clmulepi64_si128::<0x00>(a_s, b_s); + let p12 = _mm_clmulepi64_si128::<0x11>(a_s, b_s); + let xor = |x: __m128i, y: __m128i| -> (z: __m128i) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] m128(z)[i] == m128(x)[i] ^ m128(y)[i], + { _mm_xor_si128(x, y) }; + let d = fold(p0, p1, p2, p01, p02, p12, xor); + proof { + let (a, b) = (self.elem(), rhs.elem()); + lemma_swap_words(m128(a01)@, 0); + lemma_swap_words(m128(b01)@, 0); + assert(m128(a_x)[0] == a.c0 ^ a.c1 && m128(b_x)[0] == b.c0 ^ b.c1); + lemma_fold(xor, |v: __m128i| m128(v)@, 2, p0, p1, p2, p01, p02, p12, d); + lemma_karatsuba_words(a, b); + lemma_clmul_words(a.c0, b.c0); + lemma_clmul_words(a.c1, b.c1); + lemma_clmul_words(a.c2, b.c2); + lemma_clmul_words(a.c0 ^ a.c1, b.c0 ^ b.c1); + lemma_clmul_words(a.c0 ^ a.c2, b.c0 ^ b.c2); + lemma_clmul_words(a.c1 ^ a.c2, b.c1 ^ b.c2); + assert forall|k: int, w: int| 0 <= k < 3 && 0 <= w < 2 implies #[trigger] m128(d[k])[w] == split( + prod_coeffs(a, b)[k], + )[w] by { + assert(m128(p0)[w] == cw(a.c0, b.c0, w)); + assert(m128(p1)[w] == cw(a.c1, b.c1, w)); + assert(m128(p2)[w] == cw(a.c2, b.c2, w)); + assert(m128(p01)[w] == cw(a.c0 ^ a.c1, b.c0 ^ b.c1, w)); + assert(m128(p02)[w] == cw(a.c0 ^ a.c2, b.c0 ^ b.c2, w)); + assert(m128(p12)[w] == cw(a.c1 ^ a.c2, b.c1 ^ b.c2, w)); + assert(m128(d[k])[w] == kara_word(a, b, k, w)); + } + lemma_regs128(d, prod_coeffs(a, b)); + lemma_prod_unreduced(a, b); + } + F192x1Unreduced(d) + } + } + + /// The mixed product by a base-field scalar, without the reduction. + #[inline(always)] + pub fn mul_base_unreduced(self, k: F64) -> (r: F192x1Unreduced) + ensures + r.unreduced() == base_unreduced(self.elem(), k.0), + e_value(r.unreduced()) == e_mul(self.elem(), e_from_k(k.0)), + { + let ghost k0 = k.0; + proof { + use_type_invariant(&self); + lemma_clmul_sel(); + lemma_i64_round_trip(k.0); + } + // SAFETY: the module's cfg enables pclmulqdq; the rest is SSE2. + unsafe { + let k = _mm_cvtsi64_si128(k.0 as i64); + let r = F192x1Unreduced([ + _mm_clmulepi64_si128::<0x00>(self.c01, k), + _mm_clmulepi64_si128::<0x01>(self.c01, k), + _mm_clmulepi64_si128::<0x00>(self.c22, k), + ]); + proof { + let a = self.elem(); + lemma_clmul_words(a.c0, k0); + lemma_clmul_words(a.c1, k0); + lemma_clmul_words(a.c2, k0); + let c = [clmul(a.c0, k0 as u128), clmul(a.c1, k0 as u128), clmul(a.c2, k0 as u128)]; + assert forall|j: int, w: int| 0 <= j < 3 && 0 <= w < 2 implies #[trigger] m128(r.0[j])[w] == split( + c[j], + )[w] by {} + lemma_regs128(r.0, c); + lemma_base_unreduced(a, k0); + } + r + } + } +} + +/// The element at `e`'s first two words in one register. +/// +/// Trusted (`external_body`): the body is production's load; `tests/equivalence/gf2_64x3.rs` checks it. +#[verifier::external_body] +#[inline(always)] +pub fn load_c01(e: &F192) -> (r: __m128i) + ensures + m128(r)[0] == e.c0, + m128(r)[1] == e.c1, +{ + let words = (e as *const F192).cast::(); + // SAFETY: `e` is three words, `c0` and `c1` adjacent under `repr(C)`. + unsafe { _mm_loadu_si128(words.cast()) } +} + +/// The element at `e`'s third word. +/// +/// Trusted (`external_body`): the body is production's read; `tests/equivalence/gf2_64x3.rs` checks it. +#[verifier::external_body] +#[inline(always)] +pub fn load_c2(e: &F192) -> (r: u64) + ensures + r == e.c2, +{ + let words = (e as *const F192).cast::(); + // SAFETY: `e` is three words. + unsafe { *words.add(2) } +} + +/// Store `[c0, c1]` and the low word of `c22` as the three words of `out`. +/// +/// Trusted (`external_body`): the body is production's two stores; `tests/equivalence/gf2_64x3.rs` checks it. +#[verifier::external_body] +#[inline(always)] +pub fn store_c01_c2(c01: __m128i, c22: __m128i, out: &mut MaybeUninit) + ensures + final(out).mem_contents() == MemContents::Init(F192 { c0: m128(c01)[0], c1: m128(c01)[1], c2: m128(c22)[0] }), +{ + let words = out.as_mut_ptr().cast::(); + // SAFETY: `out` is three words, `c0` and `c1` adjacent under `repr(C)`. + unsafe { + _mm_storeu_si128(words.cast(), c01); + _mm_storel_epi64(words.add(2).cast(), c22); + } +} + +// The operator traits' Verus specifications: no closed form for the registers' bits, so each impl's own `ensures` +// states its result (for the conversions, the value held). +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::AddSpecImpl for F192x1 { + open spec fn obeys_add_spec() -> bool { + false + } + + open spec fn add_req(self, rhs: F192x1) -> bool { + true + } + + open spec fn add_spec(self, rhs: F192x1) -> F192x1 { + self + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::MulSpecImpl for F192x1 { + open spec fn obeys_mul_spec() -> bool { + false + } + + open spec fn mul_req(self, rhs: F192x1) -> bool { + true + } + + open spec fn mul_spec(self, rhs: F192x1) -> F192x1 { + self + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::BitXorSpecImpl for F192x1Unreduced { + open spec fn obeys_bitxor_spec() -> bool { + false + } + + open spec fn bitxor_req(self, rhs: F192x1Unreduced) -> bool { + true + } + + open spec fn bitxor_spec(self, rhs: F192x1Unreduced) -> F192x1Unreduced { + self + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::BitXorAssignSpecImpl for F192x1Unreduced { + open spec fn obeys_bitxor_assign_spec() -> bool { + false + } + + open spec fn bitxor_assign_req(&self, rhs: F192x1Unreduced) -> bool { + true + } + + open spec fn bitxor_assign_spec(&self, rhs: F192x1Unreduced) -> &F192x1Unreduced { + self + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::convert::FromSpecImpl for F192Unreduced { + open spec fn obeys_from_spec() -> bool { + true + } + + open spec fn from_spec(u: F192x1Unreduced) -> F192Unreduced { + u.unreduced() + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::convert::FromSpecImpl for F192 { + open spec fn obeys_from_spec() -> bool { + true + } + + open spec fn from_spec(e: F192x1) -> F192 { + e.elem() + } +} + +impl Add for F192x1 { + type Output = Self; + + #[inline(always)] + fn add(self, rhs: Self) -> (r: Self) + ensures + r.elem() == e_add(self.elem(), rhs.elem()), + { + proof { + use_type_invariant(&self); + use_type_invariant(&rhs); + } + // SAFETY: SSE2 is part of the x86-64 baseline. + unsafe { Self { c01: _mm_xor_si128(self.c01, rhs.c01), c22: _mm_xor_si128(self.c22, rhs.c22) } } + } +} + +impl Mul for F192x1 { + type Output = Self; + + #[inline(always)] + fn mul(self, rhs: Self) -> (r: Self) + ensures + r.elem() == e_mul(self.elem(), rhs.elem()), + { + self.mul_unreduced(rhs).reduce() + } +} + +impl F192x1Unreduced { + /// The unreduced value held. + pub closed spec fn unreduced(self) -> F192Unreduced { + regs128(self.0) + } + + #[inline(always)] + pub fn zero() -> (r: Self) + ensures + r.unreduced() == F192Unreduced::ZERO, + { + // SAFETY: SSE2 is part of the x86-64 baseline. + let r = Self([unsafe { _mm_setzero_si128() }; 3]); + proof { + assert(r.unreduced().coeffs[0] =~= F192Unreduced::ZERO.coeffs[0]); + assert(r.unreduced().coeffs[1] =~= F192Unreduced::ZERO.coeffs[1]); + assert(r.unreduced().coeffs[2] =~= F192Unreduced::ZERO.coeffs[2]); + assert(r.unreduced().coeffs =~= F192Unreduced::ZERO.coeffs); + } + r + } + + /// Reduce each coefficient, into the element's two registers. + /// + /// Rewritten for Verus: the array pattern `let [d0, d1, d2]` becomes indexing. + #[inline(always)] + pub fn reduce(self) -> (r: F192x1) + ensures + r.elem() == e_value(self.unreduced()), + { + let (d0, d1, d2) = (self.0[0], self.0[1], self.0[2]); + // SAFETY: SSE2 is part of the x86-64 baseline. + unsafe { + F192x1 { + c01: reduce_lanes128(_mm_unpacklo_epi64(d0, d1), _mm_unpackhi_epi64(d0, d1)), + c22: reduce_lanes128(_mm_unpacklo_epi64(d2, d2), _mm_unpackhi_epi64(d2, d2)), + } + } + } +} + +impl BitXor for F192x1Unreduced { + type Output = Self; + + /// Rewritten for Verus: the array patterns become indexing. + #[inline(always)] + fn bitxor(self, rhs: Self) -> (r: Self) + ensures + r.unreduced() == u_xor(self.unreduced(), rhs.unreduced()), + { + let ((a0, a1, a2), (b0, b1, b2)) = ((self.0[0], self.0[1], self.0[2]), (rhs.0[0], rhs.0[1], rhs.0[2])); + // SAFETY: SSE2 is part of the x86-64 baseline. + let r = unsafe { Self([_mm_xor_si128(a0, b0), _mm_xor_si128(a1, b1), _mm_xor_si128(a2, b2)]) }; + proof { + let (u, v) = (self.unreduced(), rhs.unreduced()); + assert(r.unreduced().coeffs[0] =~= u_xor(u, v).coeffs[0]); + assert(r.unreduced().coeffs[1] =~= u_xor(u, v).coeffs[1]); + assert(r.unreduced().coeffs[2] =~= u_xor(u, v).coeffs[2]); + assert(r.unreduced().coeffs =~= u_xor(u, v).coeffs); + } + r + } +} + +impl BitXorAssign for F192x1Unreduced { + #[inline(always)] + fn bitxor_assign(&mut self, rhs: Self) + ensures + final(self).unreduced() == u_xor(old(self).unreduced(), rhs.unreduced()), + { + *self = *self ^ rhs; + } +} + +impl From for F192Unreduced { + /// Rewritten for Verus: production maps the reinterpret over the three registers. + #[inline(always)] + fn from(u: F192x1Unreduced) -> (r: Self) + ensures + r == u.unreduced(), + { + Self { + // SAFETY: the reinterprets are between 128-bit values. + coeffs: unsafe { + [ + transmute::<__m128i, [u64; 2]>(u.0[0]), + transmute::<__m128i, [u64; 2]>(u.0[1]), + transmute::<__m128i, [u64; 2]>(u.0[2]), + ] + }, + } + } +} + +impl From for F192 { + /// Rewritten for Verus: the array patterns become indexing. + #[inline(always)] + fn from(e: F192x1) -> (r: Self) + ensures + r == e.elem(), + { + // SAFETY: the reinterprets are between 128-bit values. + let (w01, w22) = unsafe { + (transmute::<__m128i, [u64; 2]>(e.c01), transmute::<__m128i, [u64; 2]>(e.c22)) + }; + let (c0, c1, c2) = (w01[0], w01[1], w22[0]); + Self::new(c0, c1, c2) + } +} + +// --------------------------------------------------------------------------------------------- +// Batched products: one product per 128-bit lane +// --------------------------------------------------------------------------------------------- +/// The extract immediate 1 selects the upper half. +pub proof fn lemma_imm_one() + ensures + (1i32 as u8 & 1u8) == 1, + (0i32 as u8 & 1u8) == 0, +{ + assert((1u8 & 1u8) == 1 && (0u8 & 1u8) == 0) by (bit_vector); +} + +/// Every word of a PCLMULQDQ lane is a word of [`cw`]. +pub proof fn lemma_cw_all() + ensures + forall|a: u64, b: u64, w: int| 0 <= w < 2 ==> #[trigger] clmul_words(a, b)[w] == cw(a, b, w), +{ +} + +/// The reduced product's coefficients are the reductions of the schoolbook words. +pub proof fn lemma_coeffs_reduce(a: F192, b: F192) + ensures + ({ + let pc = prod_coeffs(a, b); + e_mul(a, b) == F192 { + c0: reduce_word(split(pc[0])[0], split(pc[0])[1]), + c1: reduce_word(split(pc[1])[0], split(pc[1])[1]), + c2: reduce_word(split(pc[2])[0], split(pc[2])[1]), + } + }), +{ + lemma_prod_unreduced(a, b); + let pc = prod_coeffs(a, b); + assert([split(pc[0])[0], split(pc[0])[1]] =~= prod_unreduced(a, b).coeffs[0]); + assert([split(pc[1])[0], split(pc[1])[1]] =~= prod_unreduced(a, b).coeffs[1]); + assert([split(pc[2])[0], split(pc[2])[1]] =~= prod_unreduced(a, b).coeffs[2]); +} + +/// Lane-wise base reduction: qword `i` of the result is the reduction of `hi[i] * x^64 + lo[i]`. +/// +/// The same shift network as the scalar reduction, applied to every qword at once. +/// +/// # Safety +/// +/// Requires the `avx2` target feature. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx2"))] +#[inline] +#[target_feature(enable = "avx2")] +unsafe fn reduce_lanes256(lo: __m256i, hi: __m256i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == reduce_word(m256(lo)[i], m256(hi)[i]), +{ + // The bits of `hi * 0x1B` shifted past x^63. + let spill = _mm256_xor_si256( + _mm256_xor_si256(_mm256_srli_epi64::<63>(hi), _mm256_srli_epi64::<61>(hi)), + _mm256_srli_epi64::<60>(hi), + ); + // lo ^ f(hi ^ spill), with f(v) = v ^ v<<1 ^ v<<3 ^ v<<4. + let v = _mm256_xor_si256(hi, spill); + let f = _mm256_xor_si256( + _mm256_xor_si256(v, _mm256_slli_epi64::<1>(v)), + _mm256_xor_si256(_mm256_slli_epi64::<3>(v), _mm256_slli_epi64::<4>(v)), + ); + let r = _mm256_xor_si256(lo, f); + proof { + assert forall|i: int| 0 <= i < 4 implies #[trigger] m256(r)[i] == reduce_word(m256(lo)[i], m256(hi)[i]) by { + lemma_reduce_net(m256(lo)[i], m256(hi)[i]); + } + } + r +} + +/// Lane-wise base reduction on eight qwords; see the four-qword version. +/// +/// # Safety +/// +/// Requires the `avx512f` target feature. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "avx512f")] +unsafe fn reduce_lanes512(lo: __m512i, hi: __m512i) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == reduce_word(m512(lo)[i], m512(hi)[i]), +{ + // The bits of `hi * 0x1B` shifted past x^63. + let spill = _mm512_xor_si512( + _mm512_xor_si512(_mm512_srli_epi64::<63>(hi), _mm512_srli_epi64::<61>(hi)), + _mm512_srli_epi64::<60>(hi), + ); + // lo ^ f(hi ^ spill), with f(v) = v ^ v<<1 ^ v<<3 ^ v<<4. + let v = _mm512_xor_si512(hi, spill); + let f = _mm512_xor_si512( + _mm512_xor_si512(v, _mm512_slli_epi64::<1>(v)), + _mm512_xor_si512(_mm512_slli_epi64::<3>(v), _mm512_slli_epi64::<4>(v)), + ); + let r = _mm512_xor_si512(lo, f); + proof { + assert forall|i: int| 0 <= i < 8 implies #[trigger] m512(r)[i] == reduce_word(m512(lo)[i], m512(hi)[i]) by { + lemma_reduce_net(m512(lo)[i], m512(hi)[i]); + } + } + r +} + +/// The y-folded Karatsuba products of two pairs, one pair per 128-bit lane. +/// +/// Rewritten for Verus: `a.map(|e| e.c0)` and the like become array literals; the closures carry their +/// specifications; the six products are bound to names before `fold`. +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx2` target features. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx2"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx2")] +unsafe fn karatsuba_vec2(a: [F192; 2], b: [F192; 2]) -> (r: [__m256i; 3]) + ensures + forall|k: int, i: int| + 0 <= k < 3 && 0 <= i < 4 ==> #[trigger] m256(r[k])[i] == split(prod_coeffs(a[i / 2], b[i / 2])[k])[i % 2], +{ + // One coefficient of both elements, in the low qword of each lane. + let pack = |c: [u64; 2]| -> (r: __m256i) + ensures + m256(r)[0] == c[0] && m256(r)[1] == 0 && m256(r)[2] == c[1] && m256(r)[3] == 0, + { + proof { + lemma_i64_round_trip(c[0]); + lemma_i64_round_trip(c[1]); + } + _mm256_set_epi64x(0, c[1] as i64, 0, c[0] as i64) + }; + let (a0, a1, a2) = (pack([a[0].c0, a[1].c0]), pack([a[0].c1, a[1].c1]), pack([a[0].c2, a[1].c2])); + let (b0, b1, b2) = (pack([b[0].c0, b[1].c0]), pack([b[0].c1, b[1].c1]), pack([b[0].c2, b[1].c2])); + // One instruction per base product covers both lanes. + let mul = |x: __m256i, y: __m256i| -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == cw(m256(x)[2 * (i / 2)], m256(y)[2 * (i / 2)], i % 2), + { + proof { + lemma_clmul_sel(); + lemma_cw_all(); + } + _mm256_clmulepi64_epi128::<0x00>(x, y) + }; + let xor = |x: __m256i, y: __m256i| -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == m256(x)[i] ^ m256(y)[i], + { _mm256_xor_si256(x, y) }; + let p0 = mul(a0, b0); + let p1 = mul(a1, b1); + let p2 = mul(a2, b2); + let p01 = mul(xor(a0, a1), xor(b0, b1)); + let p02 = mul(xor(a0, a2), xor(b0, b2)); + let p12 = mul(xor(a1, a2), xor(b1, b2)); + let r = fold(p0, p1, p2, p01, p02, p12, xor); + proof { + lemma_fold(xor, |v: __m256i| m256(v)@, 4, p0, p1, p2, p01, p02, p12, r); + assert forall|k: int, i: int| 0 <= k < 3 && 0 <= i < 4 implies #[trigger] m256(r[k])[i] == split( + prod_coeffs(a[i / 2], b[i / 2])[k], + )[i % 2] by { + let (x, y, l, w) = (a[i / 2], b[i / 2], 2 * (i / 2), i % 2); + lemma_karatsuba_words(x, y); + assert(m256(p0)[i] == cw(x.c0, y.c0, w)); + assert(m256(p1)[i] == cw(x.c1, y.c1, w)); + assert(m256(p2)[i] == cw(x.c2, y.c2, w)); + assert(m256(p01)[i] == cw(x.c0 ^ x.c1, y.c0 ^ y.c1, w)); + assert(m256(p02)[i] == cw(x.c0 ^ x.c2, y.c0 ^ y.c2, w)); + assert(m256(p12)[i] == cw(x.c1 ^ x.c2, y.c1 ^ y.c2, w)); + assert(m256(r[k])[i] == kara_word(x, y, k, w)); + } + } + r +} + +/// Two independent products in the two 128-bit lanes of a YMM register. +/// +/// Rewritten for Verus: the array pattern becomes indexing; `std::array::from_fn` becomes an array literal. +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx2` target features. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx2"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx2")] +pub unsafe fn mul_vec2(a: [F192; 2], b: [F192; 2]) -> (r: [F192; 2]) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] r[i] == e_mul(a[i], b[i]), +{ + // SAFETY: the function carries both features. + unsafe { + let d = karatsuba_vec2(a, b); + let (d0, d1, d2) = (d[0], d[1], d[2]); + // Lane i of c01 is [c0_i, c1_i]; the low qword of lane i of c2 is c2_i. + let c01 = reduce_lanes256(_mm256_unpacklo_epi64(d0, d1), _mm256_unpackhi_epi64(d0, d1)); + let c2 = reduce_lanes256(d2, _mm256_unpackhi_epi64(d2, d2)); + let (w01, w2) = (transmute::<__m256i, [u64; 4]>(c01), transmute::<__m256i, [u64; 4]>(c2)); + let r = [F192::new(w01[0], w01[1], w2[0]), F192::new(w01[2], w01[3], w2[2])]; + proof { + assert forall|i: int| 0 <= i < 2 implies #[trigger] r[i] == e_mul(a[i], b[i]) by { + lemma_coeffs_reduce(a[i], b[i]); + assert(m256(d0)[2 * i] == split(prod_coeffs(a[i], b[i])[0])[0]); + assert(m256(d0)[2 * i + 1] == split(prod_coeffs(a[i], b[i])[0])[1]); + assert(m256(d1)[2 * i] == split(prod_coeffs(a[i], b[i])[1])[0]); + assert(m256(d1)[2 * i + 1] == split(prod_coeffs(a[i], b[i])[1])[1]); + assert(m256(d2)[2 * i] == split(prod_coeffs(a[i], b[i])[2])[0]); + assert(m256(d2)[2 * i + 1] == split(prod_coeffs(a[i], b[i])[2])[1]); + assert(m256(c01)[2 * i] == reduce_word(m256(d0)[2 * i], m256(d0)[2 * i + 1])); + assert(m256(c01)[2 * i + 1] == reduce_word(m256(d1)[2 * i], m256(d1)[2 * i + 1])); + assert(m256(c2)[2 * i] == reduce_word(m256(d2)[2 * i], m256(d2)[2 * i + 1])); + } + } + r + } +} + +/// Two independent unreduced products, packed as for the reduced version. +/// +/// Rewritten for Verus: the reinterpret map and `std::array::from_fn` become array literals. +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx2` target features. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx2"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx2")] +pub unsafe fn mul_unreduced_vec2(a: [F192; 2], b: [F192; 2]) -> (r: [F192Unreduced; 2]) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] r[i] == prod_unreduced(a[i], b[i]), + forall|i: int| 0 <= i < 2 ==> #[trigger] e_value(r[i]) == e_mul(a[i], b[i]), +{ + // SAFETY: the function carries both features; lane i of each register is one coefficient. + unsafe { + let k = karatsuba_vec2(a, b); + let d = [ + transmute::<__m256i, [[u64; 2]; 2]>(k[0]), + transmute::<__m256i, [[u64; 2]; 2]>(k[1]), + transmute::<__m256i, [[u64; 2]; 2]>(k[2]), + ]; + let r = [ + F192Unreduced { coeffs: [d[0][0], d[1][0], d[2][0]] }, + F192Unreduced { coeffs: [d[0][1], d[1][1], d[2][1]] }, + ]; + proof { + axiom_m256_as_pairs(k[0]); + axiom_m256_as_pairs(k[1]); + axiom_m256_as_pairs(k[2]); + assert forall|i: int| 0 <= i < 2 implies #[trigger] r[i] == prod_unreduced(a[i], b[i]) && e_value(r[i]) + == e_mul(a[i], b[i]) by { + let pu = prod_unreduced(a[i], b[i]); + assert forall|j: int| 0 <= j < 3 implies #[trigger] r[i].coeffs[j] == pu.coeffs[j] by { + assert(m256(k[j])[2 * i] == split(prod_coeffs(a[i], b[i])[j])[0]); + assert(m256(k[j])[2 * i + 1] == split(prod_coeffs(a[i], b[i])[j])[1]); + assert(r[i].coeffs[j] =~= pu.coeffs[j]); + } + assert(r[i].coeffs =~= pu.coeffs); + lemma_prod_unreduced(a[i], b[i]); + } + } + r + } +} + +/// The y-folded Karatsuba products of four pairs, one pair per 128-bit lane. +/// +/// Rewritten for Verus: `a.map(|e| e.c0)` and the like become array literals; the closures carry their +/// specifications; the six products are bound to names before `fold`. +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx512f` target features. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +unsafe fn karatsuba_vec4(a: [F192; 4], b: [F192; 4]) -> (r: [__m512i; 3]) + ensures + forall|k: int, i: int| + 0 <= k < 3 && 0 <= i < 8 ==> #[trigger] m512(r[k])[i] == split(prod_coeffs(a[i / 2], b[i / 2])[k])[i % 2], +{ + // One coefficient of all four elements, in the low qword of each lane. + let pack = |c: [u64; 4]| -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == if i % 2 == 0 { + c[i / 2] + } else { + 0 + }, + { + proof { + lemma_i64_round_trip(c[0]); + lemma_i64_round_trip(c[1]); + lemma_i64_round_trip(c[2]); + lemma_i64_round_trip(c[3]); + } + _mm512_set_epi64(0, c[3] as i64, 0, c[2] as i64, 0, c[1] as i64, 0, c[0] as i64) + }; + let (a0, a1, a2) = ( + pack([a[0].c0, a[1].c0, a[2].c0, a[3].c0]), + pack([a[0].c1, a[1].c1, a[2].c1, a[3].c1]), + pack([a[0].c2, a[1].c2, a[2].c2, a[3].c2]), + ); + let (b0, b1, b2) = ( + pack([b[0].c0, b[1].c0, b[2].c0, b[3].c0]), + pack([b[0].c1, b[1].c1, b[2].c1, b[3].c1]), + pack([b[0].c2, b[1].c2, b[2].c2, b[3].c2]), + ); + // One instruction per base product covers all four lanes. + let mul = |x: __m512i, y: __m512i| -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == cw(m512(x)[2 * (i / 2)], m512(y)[2 * (i / 2)], i % 2), + { + proof { + lemma_clmul_sel(); + lemma_cw_all(); + } + _mm512_clmulepi64_epi128::<0x00>(x, y) + }; + let xor = |x: __m512i, y: __m512i| -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == m512(x)[i] ^ m512(y)[i], + { _mm512_xor_si512(x, y) }; + let p0 = mul(a0, b0); + let p1 = mul(a1, b1); + let p2 = mul(a2, b2); + let p01 = mul(xor(a0, a1), xor(b0, b1)); + let p02 = mul(xor(a0, a2), xor(b0, b2)); + let p12 = mul(xor(a1, a2), xor(b1, b2)); + let r = fold(p0, p1, p2, p01, p02, p12, xor); + proof { + lemma_fold(xor, |v: __m512i| m512(v)@, 8, p0, p1, p2, p01, p02, p12, r); + assert forall|k: int, i: int| 0 <= k < 3 && 0 <= i < 8 implies #[trigger] m512(r[k])[i] == split( + prod_coeffs(a[i / 2], b[i / 2])[k], + )[i % 2] by { + let (x, y, l, w) = (a[i / 2], b[i / 2], 2 * (i / 2), i % 2); + lemma_karatsuba_words(x, y); + assert(m512(a0)[l] == x.c0 && m512(a1)[l] == x.c1 && m512(a2)[l] == x.c2); + assert(m512(b0)[l] == y.c0 && m512(b1)[l] == y.c1 && m512(b2)[l] == y.c2); + assert(m512(p0)[i] == cw(x.c0, y.c0, w)); + assert(m512(p1)[i] == cw(x.c1, y.c1, w)); + assert(m512(p2)[i] == cw(x.c2, y.c2, w)); + assert(m512(p01)[i] == cw(x.c0 ^ x.c1, y.c0 ^ y.c1, w)); + assert(m512(p02)[i] == cw(x.c0 ^ x.c2, y.c0 ^ y.c2, w)); + assert(m512(p12)[i] == cw(x.c1 ^ x.c2, y.c1 ^ y.c2, w)); + assert(m512(r[k])[i] == kara_word(x, y, k, w)); + } + } + r +} + +/// Four independent products in the four 128-bit lanes of a ZMM register. +/// +/// Rewritten for Verus: the array pattern becomes indexing; `std::array::from_fn` becomes an array literal. +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx512f` target features. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +pub unsafe fn mul_vec4(a: [F192; 4], b: [F192; 4]) -> (r: [F192; 4]) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] r[i] == e_mul(a[i], b[i]), +{ + // SAFETY: the function carries both features. + unsafe { + let d = karatsuba_vec4(a, b); + let (d0, d1, d2) = (d[0], d[1], d[2]); + // Lane i of c01 is [c0_i, c1_i]; the low qword of lane i of c2 is c2_i. + let c01 = reduce_lanes512(_mm512_unpacklo_epi64(d0, d1), _mm512_unpackhi_epi64(d0, d1)); + let c2 = reduce_lanes512(d2, _mm512_unpackhi_epi64(d2, d2)); + let (w01, w2) = (transmute::<__m512i, [u64; 8]>(c01), transmute::<__m512i, [u64; 8]>(c2)); + let r = [ + F192::new(w01[0], w01[1], w2[0]), + F192::new(w01[2], w01[3], w2[2]), + F192::new(w01[4], w01[5], w2[4]), + F192::new(w01[6], w01[7], w2[6]), + ]; + proof { + assert forall|i: int| 0 <= i < 4 implies #[trigger] r[i] == e_mul(a[i], b[i]) by { + lemma_coeffs_reduce(a[i], b[i]); + assert(m512(d0)[2 * i] == split(prod_coeffs(a[i], b[i])[0])[0]); + assert(m512(d0)[2 * i + 1] == split(prod_coeffs(a[i], b[i])[0])[1]); + assert(m512(d1)[2 * i] == split(prod_coeffs(a[i], b[i])[1])[0]); + assert(m512(d1)[2 * i + 1] == split(prod_coeffs(a[i], b[i])[1])[1]); + assert(m512(d2)[2 * i] == split(prod_coeffs(a[i], b[i])[2])[0]); + assert(m512(d2)[2 * i + 1] == split(prod_coeffs(a[i], b[i])[2])[1]); + assert(m512(c01)[2 * i] == reduce_word(m512(d0)[2 * i], m512(d0)[2 * i + 1])); + assert(m512(c01)[2 * i + 1] == reduce_word(m512(d1)[2 * i], m512(d1)[2 * i + 1])); + assert(m512(c2)[2 * i] == reduce_word(m512(d2)[2 * i], m512(d2)[2 * i + 1])); + } + } + r + } +} + +/// Four independent unreduced products, packed as for the reduced version. +/// +/// Rewritten for Verus: the reinterpret map and `std::array::from_fn` become array literals. +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx512f` target features. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +pub unsafe fn mul_unreduced_vec4(a: [F192; 4], b: [F192; 4]) -> (r: [F192Unreduced; 4]) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] r[i] == prod_unreduced(a[i], b[i]), + forall|i: int| 0 <= i < 4 ==> #[trigger] e_value(r[i]) == e_mul(a[i], b[i]), +{ + // SAFETY: the function carries both features; lane i of each register is one coefficient. + unsafe { + let k = karatsuba_vec4(a, b); + let d = [ + transmute::<__m512i, [[u64; 2]; 4]>(k[0]), + transmute::<__m512i, [[u64; 2]; 4]>(k[1]), + transmute::<__m512i, [[u64; 2]; 4]>(k[2]), + ]; + let r = [ + F192Unreduced { coeffs: [d[0][0], d[1][0], d[2][0]] }, + F192Unreduced { coeffs: [d[0][1], d[1][1], d[2][1]] }, + F192Unreduced { coeffs: [d[0][2], d[1][2], d[2][2]] }, + F192Unreduced { coeffs: [d[0][3], d[1][3], d[2][3]] }, + ]; + proof { + axiom_m512_as_pairs(k[0]); + axiom_m512_as_pairs(k[1]); + axiom_m512_as_pairs(k[2]); + assert forall|i: int| 0 <= i < 4 implies #[trigger] r[i] == prod_unreduced(a[i], b[i]) && e_value(r[i]) + == e_mul(a[i], b[i]) by { + let pu = prod_unreduced(a[i], b[i]); + assert forall|j: int| 0 <= j < 3 implies #[trigger] r[i].coeffs[j] == pu.coeffs[j] by { + assert(m512(k[j])[2 * i] == split(prod_coeffs(a[i], b[i])[j])[0]); + assert(m512(k[j])[2 * i + 1] == split(prod_coeffs(a[i], b[i])[j])[1]); + assert(r[i].coeffs[j] =~= pu.coeffs[j]); + } + assert(r[i].coeffs =~= pu.coeffs); + lemma_prod_unreduced(a[i], b[i]); + } + } + r + } +} + +// --------------------------------------------------------------------------------------------- +// Mixed products: eight base scalars against shared operand registers +// --------------------------------------------------------------------------------------------- +/// Coefficient `c` of an element. +pub open spec fn coef(e: F192, c: int) -> u64 { + if c == 0 { + e.c0 + } else if c == 1 { + e.c1 + } else { + e.c2 + } +} + +/// Unreduced values with the same words are equal. +pub proof fn lemma_unreduced_ext(u: F192Unreduced, v: F192Unreduced) + requires + forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 ==> #[trigger] u.coeffs[c][x] == v.coeffs[c][x], + ensures + u == v, +{ + assert(u.coeffs[0][0] == v.coeffs[0][0] && u.coeffs[0][1] == v.coeffs[0][1]); + assert(u.coeffs[1][0] == v.coeffs[1][0] && u.coeffs[1][1] == v.coeffs[1][1]); + assert(u.coeffs[2][0] == v.coeffs[2][0] && u.coeffs[2][1] == v.coeffs[2][1]); + assert(u.coeffs[0] =~= v.coeffs[0]); + assert(u.coeffs[1] =~= v.coeffs[1]); + assert(u.coeffs[2] =~= v.coeffs[2]); + assert(u.coeffs =~= v.coeffs); +} + +/// The words of the mixed product's coefficients. +pub proof fn lemma_base_words(a: F192, k: u64) + ensures + forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 ==> #[trigger] base_unreduced(a, k).coeffs[c][x] == cw(coef(a, c), k, x), +{ +} + +/// `0 + e = e` in `E`. +pub proof fn lemma_e_add_zero(e: F192) + ensures + e_add(F192::ZERO, e) == e, +{ + let (x, y, z) = (e.c0, e.c1, e.c2); + assert(0u64 ^ x == x && 0u64 ^ y == y && 0u64 ^ z == z) by (bit_vector); +} + +/// Adding a product to zero gives the product. +pub proof fn lemma_zero_xor_base(u: F192Unreduced, a: F192, k: u64) + requires + forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 ==> #[trigger] u.coeffs[c][x] == 0, + ensures + e_value(u_xor(u, base_unreduced(a, k))) == e_mul(a, e_from_k(k)), +{ + lemma_unreduced_ext(u, F192Unreduced::ZERO); + lemma_u_zero(); + lemma_e_value_xor(u, base_unreduced(a, k)); + lemma_base_unreduced(a, k); + lemma_e_add_zero(e_mul(a, e_from_k(k))); +} + +/// Which of [`mul_by_pairs`]'s six registers holds coefficient `c` of row `i`: `[e0, e1, e2]` for the even rows, +/// `[o0, o1, o2]` for the odd ones, at positions `[0, 1, 4]` and `[2, 3, 5]`. +pub open spec fn acc_reg(i: int, c: int) -> int { + if i % 2 == 0 { + if c == 2 { + 4 + } else { + c + } + } else { + if c == 2 { + 5 + } else { + 2 + c + } + } +} + +/// Row `i` of six product registers: lane `i / 2` of the registers [`acc_reg`] names. +pub open spec fn row512(acc: [__m512i; 6], i: int) -> F192Unreduced { + let j = i / 2; + F192Unreduced { + coeffs: [ + [m512(acc[acc_reg(i, 0)])[2 * j], m512(acc[acc_reg(i, 0)])[2 * j + 1]], + [m512(acc[acc_reg(i, 1)])[2 * j], m512(acc[acc_reg(i, 1)])[2 * j + 1]], + [m512(acc[acc_reg(i, 2)])[2 * j], m512(acc[acc_reg(i, 2)])[2 * j + 1]], + ], + } +} + +/// The six products of a shared pair of operand registers against eight packed scalars. +/// +/// `a` lane `j` meets `k_2j` through `lo` and `k_2j+1` through `hi`. +/// Returns the products of `(a_lo, a_hi)` by `k_2j`, then by `k_2j+1`. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx512f` target features. +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +unsafe fn mul_by_pairs(lo: __m512i, hi: __m512i, c2: __m512i, k: __m512i) -> (r: [__m512i; 6]) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r[0])[i] == cw(m512(lo)[2 * (i / 2)], m512(k)[2 * (i / 2)], i % 2), + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r[1])[i] == cw(m512(lo)[2 * (i / 2) + 1], m512(k)[2 * (i / 2)], i % 2), + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r[2])[i] == cw(m512(hi)[2 * (i / 2)], m512(k)[2 * (i / 2) + 1], i % 2), + forall|i: int| + 0 <= i < 8 ==> #[trigger] m512(r[3])[i] == cw(m512(hi)[2 * (i / 2) + 1], m512(k)[2 * (i / 2) + 1], i % 2), + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r[4])[i] == cw(m512(c2)[2 * (i / 2)], m512(k)[2 * (i / 2)], i % 2), + forall|i: int| + 0 <= i < 8 ==> #[trigger] m512(r[5])[i] == cw(m512(c2)[2 * (i / 2) + 1], m512(k)[2 * (i / 2) + 1], i % 2), +{ + proof { + lemma_clmul_sel(); + lemma_cw_all(); + } + // Immediate bit 0 picks the qword of the first operand, bit 4 that of the second. + [ + _mm512_clmulepi64_epi128::<0x00>(lo, k), + _mm512_clmulepi64_epi128::<0x01>(lo, k), + _mm512_clmulepi64_epi128::<0x10>(hi, k), + _mm512_clmulepi64_epi128::<0x11>(hi, k), + _mm512_clmulepi64_epi128::<0x00>(c2, k), + _mm512_clmulepi64_epi128::<0x11>(c2, k), + ] +} + +/// Eight mixed products by one scalar. +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx512f` target features. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +pub unsafe fn mul_base8(t: F192, k: [F64; 8]) -> (r: [F192; 8]) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] r[i] == e_mul(t, e_from_k(k[i].0)), +{ + // SAFETY: the function carries both features. + unsafe { + let mut acc = [_mm512_setzero_si512(); 6]; + let ghost zero = acc; + mul_base8_add(&mut acc, t, k); + proof { + assert forall|i: int| 0 <= i < 8 implies #[trigger] e_value(row512(acc, i)) == e_mul(t, e_from_k(k[i].0)) by { + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] row512(zero, i).coeffs[c][x] + == 0 by { + assert(m512(zero[acc_reg(i, c)])[2 * (i / 2) + x] == 0); + } + lemma_zero_xor_base(row512(zero, i), t, k[i].0); + } + } + mul_base8_reduce(acc) + } +} + +/// The eight scalars in one register. +/// +/// Trusted (`external_body`): the body is production's load; `tests/equivalence/gf2_64x3.rs` checks it. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[verifier::external_body] +#[inline(always)] +pub fn load_k8(k: &[F64; 8]) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == k[i].0, +{ + // SAFETY: `k` is eight qwords. + unsafe { _mm512_loadu_si512(k.as_ptr().cast()) } +} + +/// Add the eight mixed products `t * k[i]` to the six product registers of [`mul_by_pairs`]. +/// +/// Rewritten for Verus: the load of `k` is the helper [`load_k8`]; the `iter_mut().zip(..)` loop is an index loop. +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx512f` target features. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +pub unsafe fn mul_base8_add(acc: &mut [__m512i; 6], t: F192, k: [F64; 8]) + ensures + forall|i: int| + 0 <= i < 8 ==> #[trigger] row512(*final(acc), i) == u_xor(row512(*old(acc), i), base_unreduced(t, k[i].0)), +{ + proof { + lemma_i64_round_trip(t.c2); + } + // SAFETY: the function carries both features; `k` is eight qwords. + unsafe { + // `t` in every lane: [c0, c1] for both pair registers, [c2, c2] for the last. + let t01 = _mm512_broadcast_i32x4(pair(t.c0, t.c1)); + let t2 = _mm512_set1_epi64(t.c2 as i64); + let kv = load_k8(&k); + let p = mul_by_pairs(t01, t01, t2, kv); + for c in 0..6usize + invariant + forall|d: int, x: int| + 0 <= d < c && 0 <= x < 8 ==> #[trigger] m512(acc[d])[x] == m512(old(acc)[d])[x] ^ m512(p[d])[x], + forall|d: int| c <= d < 6 ==> #[trigger] acc[d] == old(acc)[d], + { + acc[c] = _mm512_xor_si512(acc[c], p[c]); + } + proof { + assert forall|i: int| 0 <= i < 8 implies #[trigger] row512(*acc, i) == u_xor( + row512(*old(acc), i), + base_unreduced(t, k[i].0), + ) by { + let j = i / 2; + lemma_base_words(t, k[i].0); + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] row512(*acc, i).coeffs[c][x] + == u_xor(row512(*old(acc), i), base_unreduced(t, k[i].0)).coeffs[c][x] by { + let r = acc_reg(i, c); + assert(m512(acc[r])[2 * j + x] == m512(old(acc)[r])[2 * j + x] ^ m512(p[r])[2 * j + x]); + assert(m512(p[r])[2 * j + x] == cw(coef(t, c), k[i].0, x)); + } + lemma_unreduced_ext(row512(*acc, i), u_xor(row512(*old(acc), i), base_unreduced(t, k[i].0))); + } + } + } +} + +/// Reduce the six product registers of [`mul_base8_add`] into the eight sums. +/// +/// Rewritten for Verus: the parameter pattern `[e0, e1, o0, o1, e2, o2]` is the parameter `acc`, indexed; +/// `std::array::from_fn` is its closure called on `0..8`; the closures carry their specifications. +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx512f` target features. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +pub unsafe fn mul_base8_reduce(acc: [__m512i; 6]) -> (r: [F192; 8]) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] r[i] == e_value(row512(acc, i)), +{ + let (e0, e1, o0, o1, e2, o2) = (acc[0], acc[1], acc[2], acc[3], acc[4], acc[5]); + // SAFETY: the function carries both features. + unsafe { + // Gather each product's low and high halves into qword-wise vectors, then reduce. + let red = |x: __m512i, y: __m512i| -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m512(r)[2 * i] == reduce_word(m512(x)[2 * i], m512(x)[2 * i + 1]), + forall|i: int| 0 <= i < 4 ==> #[trigger] m512(r)[2 * i + 1] == reduce_word(m512(y)[2 * i], m512(y)[2 * i + 1]), + { reduce_lanes512(_mm512_unpacklo_epi64(x, y), _mm512_unpackhi_epi64(x, y)) }; + let (ev, od, cv) = (red(e0, e1), red(o0, o1), red(e2, o2)); + let even = transmute::<__m512i, [u64; 8]>(ev); + let odd = transmute::<__m512i, [u64; 8]>(od); + let c2 = transmute::<__m512i, [u64; 8]>(cv); + proof { + assert(even == m512(ev) && odd == m512(od) && c2 == m512(cv)); + } + let elem = |i: usize| -> (r: F192) + requires + i < 8, + ensures + r == (F192 { + c0: (if i % 2 == 0 { even } else { odd })[2 * (i / 2) as int], + c1: (if i % 2 == 0 { even } else { odd })[2 * (i / 2) + 1], + c2: c2[i as int], + }), + { + let (j, w) = (i / 2, if i % 2 == 0 { &even } else { &odd }); + F192::new(w[2 * j], w[2 * j + 1], c2[i]) + }; + let r = [elem(0), elem(1), elem(2), elem(3), elem(4), elem(5), elem(6), elem(7)]; + proof { + assert forall|i: int| 0 <= i < 8 implies #[trigger] r[i] == e_value(row512(acc, i)) by { + let j = i / 2; + assert(i == 0 || i == 1 || i == 2 || i == 3 || i == 4 || i == 5 || i == 6 || i == 7); + if i % 2 == 0 { + assert(i == 2 * j); + assert(even[2 * j] == reduce_word(m512(e0)[2 * j], m512(e0)[2 * j + 1])); + assert(even[2 * j + 1] == reduce_word(m512(e1)[2 * j], m512(e1)[2 * j + 1])); + assert(c2[2 * j] == reduce_word(m512(e2)[2 * j], m512(e2)[2 * j + 1])); + } else { + assert(i == 2 * j + 1); + assert(odd[2 * j] == reduce_word(m512(o0)[2 * j], m512(o0)[2 * j + 1])); + assert(odd[2 * j + 1] == reduce_word(m512(o1)[2 * j], m512(o1)[2 * j + 1])); + assert(c2[2 * j + 1] == reduce_word(m512(o2)[2 * j], m512(o2)[2 * j + 1])); + } + } + } + r + } +} + +/// Words `x` of the four 128-bit lanes of a register, XORed as [`sum_lanes4`] folds them: halves, then quarters. +pub open spec fn sum4(a: __m512i, x: int) -> u64 { + (m512(a)[x] ^ m512(a)[4 + x]) ^ (m512(a)[2 + x] ^ m512(a)[6 + x]) +} + +/// The unreduced value of three coefficient registers whose four lanes each hold part of the coefficient. +pub open spec fn sum_lanes512(d: [__m512i; 3]) -> F192Unreduced { + F192Unreduced { + coeffs: [[sum4(d[0], 0), sum4(d[0], 1)], [sum4(d[1], 0), sum4(d[1], 1)], [sum4(d[2], 0), sum4(d[2], 1)]], + } +} + +/// Word `x` of coefficient `c` of the mixed product `w_m * k_m`. +pub open spec fn tw(w: Seq, k: Seq, m: int, c: int, x: int) -> u64 { + cw(coef(w8_get(w[m / 8], m % 8), c), k[m].0, x) +} + +/// Word `a`, then the words of the eight mixed products of block `b` XORed on in order. +pub open spec fn block_words(a: u64, w: Seq, k: Seq, b: int, c: int, x: int) -> u64 { + a ^ tw(w, k, 8 * b, c, x) ^ tw(w, k, 8 * b + 1, c, x) ^ tw(w, k, 8 * b + 2, c, x) ^ tw(w, k, 8 * b + 3, c, x) + ^ tw(w, k, 8 * b + 4, c, x) ^ tw(w, k, 8 * b + 5, c, x) ^ tw(w, k, 8 * b + 6, c, x) ^ tw(w, k, 8 * b + 7, c, x) +} + +/// The mixed product `w_m * k_m`, unreduced. +pub open spec fn dot_term(w: Seq, k: Seq, m: int) -> F192Unreduced { + base_unreduced(w8_get(w[m / 8], m % 8), k[m].0) +} + +/// One step of the inner product: XORing on the unreduced product `w_m * k_m` adds it to the sum. +pub proof fn lemma_dot_term(w: Seq, k: Seq, m: int, u: F192Unreduced) + requires + 0 <= m, + e_value(u) == dot_spec(w, k, m as nat), + ensures + e_value(u_xor(u, dot_term(w, k, m))) == dot_spec(w, k, (m + 1) as nat), +{ + lemma_e_value_xor(u, dot_term(w, k, m)); + lemma_base_unreduced(w8_get(w[m / 8], m % 8), k[m].0); +} + +/// A block of eight terms XORed on in order adds the block to the sum. +pub proof fn lemma_dot_block(w: Seq, k: Seq, b: int, u_old: F192Unreduced, u_new: F192Unreduced) + requires + 0 <= b, + e_value(u_old) == dot_spec(w, k, (8 * b) as nat), + forall|c: int, x: int| + 0 <= c < 3 && 0 <= x < 2 ==> #[trigger] u_new.coeffs[c][x] == block_words(u_old.coeffs[c][x], w, k, b, c, x), + ensures + e_value(u_new) == dot_spec(w, k, (8 * b + 8) as nat), +{ + let t = |m: int| dot_term(w, k, m); + let u1 = u_xor(u_old, t(8 * b)); + let u2 = u_xor(u1, t(8 * b + 1)); + let u3 = u_xor(u2, t(8 * b + 2)); + let u4 = u_xor(u3, t(8 * b + 3)); + let u5 = u_xor(u4, t(8 * b + 4)); + let u6 = u_xor(u5, t(8 * b + 5)); + let u7 = u_xor(u6, t(8 * b + 6)); + let u8 = u_xor(u7, t(8 * b + 7)); + lemma_dot_term(w, k, 8 * b, u_old); + lemma_dot_term(w, k, 8 * b + 1, u1); + lemma_dot_term(w, k, 8 * b + 2, u2); + lemma_dot_term(w, k, 8 * b + 3, u3); + lemma_dot_term(w, k, 8 * b + 4, u4); + lemma_dot_term(w, k, 8 * b + 5, u5); + lemma_dot_term(w, k, 8 * b + 6, u6); + lemma_dot_term(w, k, 8 * b + 7, u7); + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] u_new.coeffs[c][x] == u8.coeffs[c][x] by { + assert forall|m: int| #![auto] 0 <= m ==> t(m).coeffs[c][x] == tw(w, k, m, c, x) by { + lemma_base_words(w8_get(w[m / 8], m % 8), k[m].0); + } + } + lemma_unreduced_ext(u_new, u8); +} + +/// The mixed inner product over packed weights. +/// +/// Rewritten for Verus: the `enumerate` loop is an index loop; the loads are the helpers [`load_k_at`] and +/// [`load_weights`]; the array pattern of the products becomes indexing; the lane fold mapped over `acc` is its +/// closure called on each register; the closures carry their specifications. The `Safety` condition on the lengths +/// is a `requires`. +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx512f` target features, and `k.len() == 8 * w.len()`. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +pub unsafe fn dot_base(w: &[Weights8], k: &[F64]) -> (r: F192Unreduced) + requires + k.len() == 8 * w.len(), + ensures + e_value(r) == dot_spec(w@, k@, k.len() as nat), +{ + // SAFETY: the function carries both features; `Weights8` is 64-byte aligned, `k` holds 8 qwords per block. + unsafe { + let xor = |x: __m512i, y: __m512i| -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == m512(x)[i] ^ m512(y)[i], + { _mm512_xor_si512(x, y) }; + let mut acc = [_mm512_setzero_si512(); 3]; + proof { + lemma_u_zero(); + let z = sum_lanes512(acc); + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] z.coeffs[c][x] + == F192Unreduced::ZERO.coeffs[c][x] by { + assert(m512(acc[c])[x] == 0 && m512(acc[c])[4 + x] == 0 && m512(acc[c])[2 + x] == 0 && m512(acc[c])[6 + + x] == 0); + assert((0u64 ^ 0u64) ^ (0u64 ^ 0u64) == 0u64) by (bit_vector); + } + lemma_unreduced_ext(z, F192Unreduced::ZERO); + } + for b in 0..w.len() + invariant + k.len() == 8 * w.len(), + e_value(sum_lanes512(acc)) == dot_spec(w@, k@, (8 * b) as nat), + forall|x: __m512i, y: __m512i| #[trigger] xor.requires((x, y)), + forall|x: __m512i, y: __m512i, z: __m512i, i: int| + #![trigger xor.ensures((x, y), z), m512(z)[i]] + xor.ensures((x, y), z) && 0 <= i < 8 ==> m512(z)[i] == m512(x)[i] ^ m512(y)[i], + { + let kv = load_k_at(k, 8 * b); + let (lo, hi, c2) = load_weights(&w[b]); + let p = mul_by_pairs(lo, hi, c2, kv); + let (e0, e1, o0, o1, e2, o2) = (p[0], p[1], p[2], p[3], p[4], p[5]); + let ghost old_acc = acc; + acc = [xor(acc[0], xor(e0, o0)), xor(acc[1], xor(e1, o1)), xor(acc[2], xor(e2, o2))]; + proof { + let (u_old, u_new) = (sum_lanes512(old_acc), sum_lanes512(acc)); + let ws = w[b as int]; + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] u_new.coeffs[c][x] + == block_words(u_old.coeffs[c][x], w@, k@, b as int, c, x) by { + // Lane `j` of the even and odd products: terms `8b + 2j` and `8b + 2j + 1`. + assert forall|j: int| 0 <= j < 4 implies { + &&& #[trigger] m512(p[acc_reg(0, c)])[2 * j + x] == tw(w@, k@, 8 * b + 2 * j, c, x) + &&& m512(p[acc_reg(1, c)])[2 * j + x] == tw(w@, k@, 8 * b + 2 * j + 1, c, x) + } by { + assert((8 * b + 2 * j) / 8 == b && (8 * b + 2 * j) % 8 == 2 * j); + assert((8 * b + 2 * j + 1) / 8 == b && (8 * b + 2 * j + 1) % 8 == 2 * j + 1); + assert((2 * j) / 2 == j && (2 * j + 1) / 2 == j && (2 * j) % 2 == 0 && (2 * j + 1) % 2 == 1); + } + let a = |l: int| m512(old_acc[c])[2 * l + x]; + let t = |m: int| tw(w@, k@, 8 * b + m, c, x); + assert(m512(p[acc_reg(0, c)])[x] == t(0) && m512(p[acc_reg(1, c)])[x] == t(1)); + assert(m512(p[acc_reg(0, c)])[2 + x] == t(2) && m512(p[acc_reg(1, c)])[2 + x] == t(3)); + assert(m512(p[acc_reg(0, c)])[4 + x] == t(4) && m512(p[acc_reg(1, c)])[4 + x] == t(5)); + assert(m512(p[acc_reg(0, c)])[6 + x] == t(6) && m512(p[acc_reg(1, c)])[6 + x] == t(7)); + let (a0, a1, a2, a3) = (a(0), a(1), a(2), a(3)); + let (t0, t1, t2, t3, t4, t5, t6, t7) = (t(0), t(1), t(2), t(3), t(4), t(5), t(6), t(7)); + assert(((a0 ^ (t0 ^ t1)) ^ (a2 ^ (t4 ^ t5))) ^ ((a1 ^ (t2 ^ t3)) ^ (a3 ^ (t6 ^ t7))) == ((a0 ^ a2) ^ ( + a1 ^ a3)) ^ t0 ^ t1 ^ t2 ^ t3 ^ t4 ^ t5 ^ t6 ^ t7) by (bit_vector); + } + lemma_dot_block(w@, k@, b as int, u_old, u_new); + } + } + // Fold the four 128-bit lanes of each sum into one. + let lane = |a: __m512i| -> (q: [u64; 2]) + ensures + q[0] == sum4(a, 0), + q[1] == sum4(a, 1), + { + let half = _mm256_xor_si256(_mm512_castsi512_si256(a), _mm512_extracti64x4_epi64::<1>(a)); + let q = _mm_xor_si128(_mm256_castsi256_si128(half), _mm256_extracti128_si256::<1>(half)); + proof { + lemma_imm_one(); + assert(m128(q)[0] == sum4(a, 0) && m128(q)[1] == sum4(a, 1)); + } + transmute::<__m128i, [u64; 2]>(q) + }; + let lanes = [lane(acc[0]), lane(acc[1]), lane(acc[2])]; + let r = F192Unreduced { coeffs: lanes }; + proof { + lemma_unreduced_ext(r, sum_lanes512(acc)); + } + r + } +} + +/// Eight scalars from `k[at..]` in one register. +/// +/// Trusted (`external_body`): the body is production's load; `tests/equivalence/gf2_64x3.rs` checks it. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[verifier::external_body] +#[inline(always)] +pub fn load_k_at(k: &[F64], at: usize) -> (r: __m512i) + requires + at + 8 <= k.len(), + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == k[at + i].0, +{ + // SAFETY: `k` holds eight qwords from `at`. + unsafe { _mm512_loadu_si512(k.as_ptr().add(at).cast()) } +} + +/// The three weight registers of a block. +/// +/// Trusted (`external_body`): the body is production's three aligned loads (`Weights8` is 64-byte aligned); +/// `tests/equivalence/gf2_64x3.rs` checks it. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[verifier::external_body] +#[inline(always)] +pub fn load_weights(w: &Weights8) -> (r: (__m512i, __m512i, __m512i)) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r.0)[i] == w.lo[i], + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r.1)[i] == w.hi[i], + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r.2)[i] == w.c2[i], +{ + // SAFETY: the fields of `Weights8` are 64-byte aligned. + unsafe { + ( + _mm512_load_si512(w.lo.as_ptr().cast()), + _mm512_load_si512(w.hi.as_ptr().cast()), + _mm512_load_si512(w.c2.as_ptr().cast()), + ) + } +} + +// --------------------------------------------------------------------------------------------- +// Lane-major kernels (`F192x4`, `MixedSums8`) on AVX-512 +// --------------------------------------------------------------------------------------------- +/// Every `u64` survives the round trip through `i64`. +pub proof fn lemma_i64_round_trip_all() + ensures + forall|a: u64| #[trigger] ((a as i64) as u64) == a, +{ + assert forall|a: u64| #[trigger] ((a as i64) as u64) == a by { + lemma_i64_round_trip(a); + } +} + +/// Word `n` of four consecutive elements, as they lie in memory (`repr(C)`: `c0, c1, c2` of each). +pub open spec fn f192_word(v: [F192; 4], n: int) -> u64 { + coef(v[n / 3], n % 3) +} + +/// The element whose words are `3i .. 3i + 3` of `head` (words 0 to 7) then `tail` (words 8 to 11). +pub open spec fn words12_elem(head: Seq, tail: Seq, i: int) -> F192 { + let w = |n: int| + if n < 8 { + head[n] + } else { + tail[n - 8] + }; + F192 { c0: w(3 * i), c1: w(3 * i + 1), c2: w(3 * i + 2) } +} + +/// The index bits `_mm512_permutex2var_epi64` reads, for the indices below 16. +pub proof fn lemma_idx(x: u64) + requires + x < 16, + ensures + x & 7 == x % 8, + (x & 8 == 0) == (x < 8), +{ + assert(x < 16 ==> (x & 7) == x % 8 && ((x & 8) == 0) == (x < 8)) by (bit_vector); +} + +/// [`super::MixedSums8`]'s registers on AVX-512: [`mul_by_pairs`]'s six products. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub type MixedAcc8 = [__m512i; 6]; + +/// [`super::F192x4`]'s registers on AVX-512: [`lanes4`]'s form. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub type Lanes4 = [__m512i; 2]; + +/// [`super::F192x4Unreduced`]'s registers on AVX-512: [`mul_lanes4`]'s three coefficients. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub type Wide4 = [__m512i; 3]; + +/// Row `i` of [`MixedAcc8`]. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub open spec fn acc8_row(acc: MixedAcc8, i: int) -> F192Unreduced { + row512(acc, i) +} + +/// Element `i` of [`Lanes4`]: `[c0, c1]` in lane `i` of the first register, `c2` in the low word of lane `i` of +/// the second. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub open spec fn lanes4_elem(l: Lanes4, i: int) -> F192 { + F192 { c0: m512(l[0])[2 * i], c1: m512(l[0])[2 * i + 1], c2: m512(l[1])[2 * i] } +} + +/// The form [`lanes4`] documents: lane `i` of the second register is `[c2, c2]`. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub open spec fn lanes4_ok(l: Lanes4) -> bool { + forall|i: int| 0 <= i < 4 ==> #[trigger] m512(l[1])[2 * i + 1] == m512(l[1])[2 * i] +} + +/// The unreduced value in lane `i` of [`Wide4`]. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub open spec fn wide4_value(d: Wide4, i: int) -> F192Unreduced { + F192Unreduced { + coeffs: [ + [m512(d[0])[2 * i], m512(d[0])[2 * i + 1]], + [m512(d[1])[2 * i], m512(d[1])[2 * i + 1]], + [m512(d[2])[2 * i], m512(d[2])[2 * i + 1]], + ], + } +} + +/// The sum of the four lanes' values of [`Wide4`]. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub open spec fn wide4_sum(d: Wide4) -> F192 { + e_add( + e_add(e_add(e_value(wide4_value(d, 0)), e_value(wide4_value(d, 1))), e_value(wide4_value(d, 2))), + e_value(wide4_value(d, 3)), + ) +} + +/// Four elements in lanes: `[c0, c1]` and `[c2, c2]` in lane `i` for element `i`. +/// +/// # Safety +/// +/// Requires the `avx512f` target feature. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn lanes4(v: [F192; 4]) -> (r: [__m512i; 2]) + ensures + lanes4_ok(r), + forall|i: int| 0 <= i < 4 ==> #[trigger] lanes4_elem(r, i) == v[i], +{ + proof { + lemma_i64_round_trip_all(); + } + let w = |i: usize| -> (e: F192) + requires + i < 4, + ensures + e == v[i as int], + { v[i] }; + [ + _mm512_set_epi64( + w(3).c1 as i64, + w(3).c0 as i64, + w(2).c1 as i64, + w(2).c0 as i64, + w(1).c1 as i64, + w(1).c0 as i64, + w(0).c1 as i64, + w(0).c0 as i64, + ), + _mm512_set_epi64( + w(3).c2 as i64, + w(3).c2 as i64, + w(2).c2 as i64, + w(2).c2 as i64, + w(1).c2 as i64, + w(1).c2 as i64, + w(0).c2 as i64, + w(0).c2 as i64, + ), + ] +} + +/// Words 0 to 7 of four consecutive elements. +/// +/// Trusted (`external_body`): the body is production's load; `tests/equivalence/gf2_64x3.rs` checks it. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[verifier::external_body] +#[inline(always)] +pub fn load_head8(v: &[F192; 4]) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == f192_word(*v, i), +{ + let words = v.as_ptr().cast::(); + // SAFETY: `v` is twelve words. + unsafe { _mm512_loadu_si512(words.cast()) } +} + +/// Words 8 to 11 of four consecutive elements. +/// +/// Trusted (`external_body`): the body is production's load; `tests/equivalence/gf2_64x3.rs` checks it. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx2"))] +#[verifier::external_body] +#[inline(always)] +pub fn load_tail4(v: &[F192; 4]) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == f192_word(*v, 8 + i), +{ + let words = v.as_ptr().cast::(); + // SAFETY: `v` is twelve words. + unsafe { _mm256_loadu_si256(words.add(8).cast()) } +} + +/// Four consecutive elements, twelve words, into [`lanes4`]'s form. +/// +/// Rewritten for Verus: the two loads through the cast pointer are the helpers [`load_head8`] and [`load_tail4`]. +/// +/// # Safety +/// +/// Requires the `avx512f` target feature. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn load_lanes4(v: &[F192; 4]) -> (r: [__m512i; 2]) + ensures + lanes4_ok(r), + forall|i: int| 0 <= i < 4 ==> #[trigger] lanes4_elem(r, i) == v[i], +{ + // `v` is twelve words, words 0..8 and 8..12 (the helpers hold production's `unsafe` loads). + let (head, tail) = (load_head8(v), _mm512_castsi256_si512(load_tail4(v))); + // An index of 8 or more takes word `index - 8` of `tail`. + let c01 = _mm512_set_epi64(10, 9, 7, 6, 4, 3, 1, 0); + let c22 = _mm512_set_epi64(11, 11, 8, 8, 5, 5, 2, 2); + let r = [_mm512_permutex2var_epi64(head, c01, tail), _mm512_permutex2var_epi64(head, c22, tail)]; + proof { + lemma_idx(0); + lemma_idx(1); + lemma_idx(2); + lemma_idx(3); + lemma_idx(4); + lemma_idx(5); + lemma_idx(6); + lemma_idx(7); + lemma_idx(8); + lemma_idx(9); + lemma_idx(10); + lemma_idx(11); + let (a, b) = (r[0], r[1]); + assert(m512(a)[0] == v[0].c0 && m512(a)[1] == v[0].c1 && m512(b)[0] == v[0].c2 && m512(b)[1] == v[0].c2); + assert(m512(a)[2] == v[1].c0 && m512(a)[3] == v[1].c1 && m512(b)[2] == v[1].c2 && m512(b)[3] == v[1].c2); + assert(m512(a)[4] == v[2].c0 && m512(a)[5] == v[2].c1 && m512(b)[4] == v[2].c2 && m512(b)[5] == v[2].c2); + assert(m512(a)[6] == v[3].c0 && m512(a)[7] == v[3].c1 && m512(b)[6] == v[3].c2 && m512(b)[7] == v[3].c2); + } + r +} + +/// Store the twelve words `head[0..8]`, `tail[0..4]` as four consecutive elements. +/// +/// Trusted (`external_body`): the body is production's two stores; `tests/equivalence/gf2_64x3.rs` checks it. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[verifier::external_body] +#[inline(always)] +pub fn store_head_tail(out: &mut [MaybeUninit; 4], head: __m512i, tail: __m256i) + ensures + forall|i: int| + 0 <= i < 4 ==> #[trigger] final(out)[i].mem_contents() == MemContents::Init( + words12_elem(m512(head)@, m256(tail)@, i), + ), +{ + let words = out.as_mut_ptr().cast::(); + // SAFETY: `out` is twelve words, words 0..8 and 8..12. + unsafe { + _mm512_storeu_si512(words.cast(), head); + _mm256_storeu_si256(words.add(8).cast(), tail); + } +} + +/// [`lanes4`]'s form back to twelve consecutive words, the inverse of [`load_lanes4`]. +/// +/// Rewritten for Verus: the parameter pattern `[c01, c22]` is the parameter `lanes`, indexed; the two stores +/// through the cast pointer are the helper [`store_head_tail`]. +/// +/// # Safety +/// +/// Requires the `avx512f` target feature. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn store_lanes4(lanes: [__m512i; 2], out: &mut [MaybeUninit; 4]) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] final(out)[i].mem_contents() == MemContents::Init(lanes4_elem(lanes, i)), +{ + let (c01, c22) = (lanes[0], lanes[1]); + // An index of 8 or more takes word `index - 8` of `c22`. + let head = _mm512_permutex2var_epi64(c01, _mm512_set_epi64(5, 4, 10, 3, 2, 8, 1, 0), c22); + let tail = _mm512_permutex2var_epi64(c01, _mm512_set_epi64(0, 0, 0, 0, 14, 7, 6, 12), c22); + proof { + lemma_idx(0); + lemma_idx(1); + lemma_idx(2); + lemma_idx(3); + lemma_idx(4); + lemma_idx(5); + lemma_idx(6); + lemma_idx(7); + lemma_idx(8); + lemma_idx(10); + lemma_idx(12); + lemma_idx(14); + } + // `out` is twelve words, words 0..8 and 8..12 (the helper holds production's `unsafe` stores). + let t = _mm512_castsi512_si256(tail); + store_head_tail(out, head, t); + proof { + assert forall|i: int| 0 <= i < 4 implies #[trigger] words12_elem(m512(head)@, m256(t)@, i) == lanes4_elem( + lanes, + i, + ) by { + assert(m512(head)[0] == m512(c01)[0] && m512(head)[1] == m512(c01)[1] && m512(head)[2] == m512(c22)[0]); + assert(m512(head)[3] == m512(c01)[2] && m512(head)[4] == m512(c01)[3] && m512(head)[5] == m512(c22)[2]); + assert(m512(head)[6] == m512(c01)[4] && m512(head)[7] == m512(c01)[5]); + assert(m256(t)[0] == m512(c22)[4] && m256(t)[1] == m512(c01)[6] && m256(t)[2] == m512(c01)[7]); + assert(m256(t)[3] == m512(c22)[6]); + } + } +} + +/// The 4x4 transpose of [`lanes4`] values: lane `j` of `out[i]` is lane `i` of `rows[j]`. +/// +/// Rewritten for Verus: `rows.map(|r| r[p])` and its array pattern become indexing; `std::array::from_fn` becomes an +/// array literal; the closure carries its specification. +/// +/// # Safety +/// +/// Requires the `avx512f` target feature. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn transpose_lanes4(rows: [[__m512i; 2]; 4]) -> (r: [[__m512i; 2]; 4]) + ensures + forall|i: int, j: int, p: int, x: int| + 0 <= i < 4 && 0 <= j < 4 && 0 <= p < 2 && 0 <= x < 2 ==> #[trigger] m512(r[i][p])[2 * j + x] == m512( + rows[j][p], + )[2 * i + x], + forall|i: int, j: int| 0 <= i < 4 && 0 <= j < 4 ==> #[trigger] lanes4_elem(r[i], j) == lanes4_elem(rows[j], i), + (forall|j: int| 0 <= j < 4 ==> #[trigger] lanes4_ok(rows[j])) ==> forall|i: int| 0 <= i < 4 ==> #[trigger] lanes4_ok(r[i]), +{ + let part = |p: usize| -> (o: [__m512i; 4]) + requires + p < 2, + ensures + forall|i: int, j: int, x: int| + 0 <= i < 4 && 0 <= j < 4 && 0 <= x < 2 ==> #[trigger] m512(o[i])[2 * j + x] == m512( + rows[j][p as int], + )[2 * i + x], + { + let (a, b, c, d) = (rows[0][p], rows[1][p], rows[2][p], rows[3][p]); + // `[a0, a1, b0, b1]`, `[a2, a3, b2, b3]`, and the same of `c` and `d`. + let (ab01, ab23) = (_mm512_shuffle_i64x2::<0x44>(a, b), _mm512_shuffle_i64x2::<0xEE>(a, b)); + let (cd01, cd23) = (_mm512_shuffle_i64x2::<0x44>(c, d), _mm512_shuffle_i64x2::<0xEE>(c, d)); + let o = [ + _mm512_shuffle_i64x2::<0x88>(ab01, cd01), + _mm512_shuffle_i64x2::<0xDD>(ab01, cd01), + _mm512_shuffle_i64x2::<0x88>(ab23, cd23), + _mm512_shuffle_i64x2::<0xDD>(ab23, cd23), + ]; + proof { + lemma_shuffle_sel(0); + lemma_shuffle_sel(1); + lemma_shuffle_sel(2); + lemma_shuffle_sel(3); + assert forall|i: int, j: int, x: int| 0 <= i < 4 && 0 <= j < 4 && 0 <= x < 2 implies #[trigger] m512( + o[i], + )[2 * j + x] == m512(rows[j][p as int])[2 * i + x] by { + assert((2 * j + x) / 2 == j && (2 * j + x) % 2 == x); + assert((2 * i + x) / 2 == i); + let (s, h) = (i / 2, i % 2); + assert(2 * i + x == 2 * (2 * s + h) + x); + } + } + o + }; + let (c01, c22) = (part(0), part(1)); + let r = [[c01[0], c22[0]], [c01[1], c22[1]], [c01[2], c22[2]], [c01[3], c22[3]]]; + proof { + assert forall|i: int, j: int, p: int, x: int| + 0 <= i < 4 && 0 <= j < 4 && 0 <= p < 2 && 0 <= x < 2 implies #[trigger] m512(r[i][p])[2 * j + x] == m512( + rows[j][p], + )[2 * i + x] by { + if p == 0 { + assert(r[i][p] == c01[i]); + } else { + assert(r[i][p] == c22[i]); + } + } + assert forall|i: int, j: int| 0 <= i < 4 && 0 <= j < 4 implies #[trigger] lanes4_elem(r[i], j) == lanes4_elem( + rows[j], + i, + ) by { + assert(m512(r[i][0])[2 * j + 0] == m512(rows[j][0])[2 * i + 0]); + assert(m512(r[i][0])[2 * j + 1] == m512(rows[j][0])[2 * i + 1]); + assert(m512(r[i][1])[2 * j + 0] == m512(rows[j][1])[2 * i + 0]); + } + if forall|j: int| 0 <= j < 4 ==> #[trigger] lanes4_ok(rows[j]) { + assert forall|i: int| 0 <= i < 4 implies #[trigger] lanes4_ok(r[i]) by { + assert forall|j: int| 0 <= j < 4 implies #[trigger] m512(r[i][1])[2 * j + 1] == m512(r[i][1])[2 * j] by { + assert(lanes4_ok(rows[j])); + assert(m512(r[i][1])[2 * j + 1] == m512(rows[j][1])[2 * i + 1]); + assert(m512(r[i][1])[2 * j + 0] == m512(rows[j][1])[2 * i + 0]); + } + } + } + } + r +} + +/// The lane selections of the four `_mm512_shuffle_i64x2` immediates [`transpose_lanes4`] uses. +pub proof fn lemma_shuffle_sel(q: int) + requires + 0 <= q < 4, + ensures + ((0x44i32 as u8) >> ((2 * q) as u8)) & 3 == q % 2, + ((0xEEi32 as u8) >> ((2 * q) as u8)) & 3 == 2 + q % 2, + ((0x88i32 as u8) >> ((2 * q) as u8)) & 3 == 2 * (q % 2), + ((0xDDi32 as u8) >> ((2 * q) as u8)) & 3 == 2 * (q % 2) + 1, +{ + if q == 0 { + assert((0x44u8 >> 0u8) & 3 == 0 && (0xEEu8 >> 0u8) & 3 == 2 && (0x88u8 >> 0u8) & 3 == 0 && (0xDDu8 >> 0u8) & 3 + == 1) by (bit_vector); + } else if q == 1 { + assert((0x44u8 >> 2u8) & 3 == 1 && (0xEEu8 >> 2u8) & 3 == 3 && (0x88u8 >> 2u8) & 3 == 2 && (0xDDu8 >> 2u8) & 3 + == 3) by (bit_vector); + } else if q == 2 { + assert((0x44u8 >> 4u8) & 3 == 0 && (0xEEu8 >> 4u8) & 3 == 2 && (0x88u8 >> 4u8) & 3 == 0 && (0xDDu8 >> 4u8) & 3 + == 1) by (bit_vector); + } else { + assert((0x44u8 >> 6u8) & 3 == 1 && (0xEEu8 >> 6u8) & 3 == 3 && (0x88u8 >> 6u8) & 3 == 2 && (0xDDu8 >> 6u8) & 3 + == 3) by (bit_vector); + } +} + +/// Lane-wise XOR. +/// +/// Rewritten for Verus: `std::array::from_fn` becomes a loop over a copy of `a`. +/// +/// # Safety +/// +/// Requires the `avx512f` target feature. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn xor_lanes(a: [__m512i; N], b: [__m512i; N]) -> (r: [__m512i; N]) + ensures + lanes_xor(a, b, r), +{ + let mut r = a; + for i in 0..N + invariant + forall|j: int, x: int| 0 <= j < i && 0 <= x < 8 ==> #[trigger] m512(r[j])[x] == m512(a[j])[x] ^ m512(b[j])[x], + forall|j: int| i <= j < N ==> #[trigger] r[j] == a[j], + { + r[i] = _mm512_xor_si512(a[i], b[i]); + } + r +} + +/// [`xor_lanes`]'s result: every word of `r` is the XOR of those of `a` and `b`. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub open spec fn lanes_xor(a: [__m512i; N], b: [__m512i; N], r: [__m512i; N]) -> bool { + forall|i: int, x: int| 0 <= i < N && 0 <= x < 8 ==> #[trigger] m512(r[i])[x] == m512(a[i])[x] ^ m512(b[i])[x] +} + +/// [`xor_lanes`] on [`Lanes4`] adds the elements and keeps the form. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub proof fn lemma_lanes4_xor(a: Lanes4, b: Lanes4, r: Lanes4) + requires + lanes_xor(a, b, r), + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] lanes4_elem(r, i) == e_add(lanes4_elem(a, i), lanes4_elem(b, i)), + lanes4_ok(a) && lanes4_ok(b) ==> lanes4_ok(r), +{ + assert forall|i: int| 0 <= i < 4 implies #[trigger] lanes4_elem(r, i) == e_add(lanes4_elem(a, i), lanes4_elem(b, i)) by { + assert(m512(r[0])[2 * i] == m512(a[0])[2 * i] ^ m512(b[0])[2 * i]); + assert(m512(r[0])[2 * i + 1] == m512(a[0])[2 * i + 1] ^ m512(b[0])[2 * i + 1]); + assert(m512(r[1])[2 * i] == m512(a[1])[2 * i] ^ m512(b[1])[2 * i]); + } + if lanes4_ok(a) && lanes4_ok(b) { + assert forall|i: int| 0 <= i < 4 implies #[trigger] m512(r[1])[2 * i + 1] == m512(r[1])[2 * i] by { + assert(m512(r[1])[2 * i] == m512(a[1])[2 * i] ^ m512(b[1])[2 * i]); + assert(m512(r[1])[2 * i + 1] == m512(a[1])[2 * i + 1] ^ m512(b[1])[2 * i + 1]); + } + } +} + +/// [`xor_lanes`] on [`Wide4`] XORs the unreduced values. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub proof fn lemma_wide4_xor(a: Wide4, b: Wide4, r: Wide4) + requires + lanes_xor(a, b, r), + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] wide4_value(r, i) == u_xor(wide4_value(a, i), wide4_value(b, i)), +{ + assert forall|i: int| 0 <= i < 4 implies #[trigger] wide4_value(r, i) == u_xor(wide4_value(a, i), wide4_value(b, i)) by { + lemma_u_xor_words(wide4_value(a, i), wide4_value(b, i)); + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] wide4_value(r, i).coeffs[c][x] == u_xor( + wide4_value(a, i), + wide4_value(b, i), + ).coeffs[c][x] by { + assert(m512(r[c])[2 * i + x] == m512(a[c])[2 * i + x] ^ m512(b[c])[2 * i + x]); + assert(wide4_value(r, i).coeffs[c][x] == m512(r[c])[2 * i + x]); + assert(wide4_value(a, i).coeffs[c][x] == m512(a[c])[2 * i + x]); + assert(wide4_value(b, i).coeffs[c][x] == m512(b[c])[2 * i + x]); + } + lemma_unreduced_ext(wide4_value(r, i), u_xor(wide4_value(a, i), wide4_value(b, i))); + } +} + +/// All-zero [`Wide4`] registers hold zero products. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub proof fn lemma_zeroed_wide4() + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] wide4_value(zeroed_value::(), i) == F192Unreduced::ZERO, +{ + axiom_zeroed_m512::<3>(); + let z = zeroed_value::(); + assert forall|i: int| 0 <= i < 4 implies #[trigger] wide4_value(z, i) == F192Unreduced::ZERO by { + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] wide4_value(z, i).coeffs[c][x] + == F192Unreduced::ZERO.coeffs[c][x] by { + assert(m512(z[c])[2 * i + x] == 0); + assert(wide4_value(z, i).coeffs[c][x] == m512(z[c])[2 * i + x]); + } + lemma_unreduced_ext(wide4_value(z, i), F192Unreduced::ZERO); + } +} + +/// All-zero [`MixedAcc8`] registers hold zero sums. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub proof fn lemma_zeroed_acc8() + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] acc8_row(zeroed_value::(), i) == F192Unreduced::ZERO, +{ + axiom_zeroed_m512::<6>(); + let z = zeroed_value::(); + assert forall|i: int| 0 <= i < 8 implies #[trigger] acc8_row(z, i) == F192Unreduced::ZERO by { + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] acc8_row(z, i).coeffs[c][x] + == F192Unreduced::ZERO.coeffs[c][x] by { + assert(m512(z[acc_reg(i, c)])[2 * (i / 2) + x] == 0); + if c == 0 { + } else if c == 1 { + } else { + } + } + lemma_unreduced_ext(acc8_row(z, i), F192Unreduced::ZERO); + } +} + +/// The y-folded Karatsuba products of two [`lanes4`] operands, lane by lane. +/// +/// Rewritten for Verus: the parameter patterns `[a01, a22]`, `[b01, b22]` are the parameters `a`, `b`, indexed; +/// the six products and the XOR closure are bound to names before `fold`. +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx512f` target features. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +pub unsafe fn mul_lanes4(a: [__m512i; 2], b: [__m512i; 2]) -> (r: [__m512i; 3]) + requires + lanes4_ok(a), + lanes4_ok(b), + ensures + forall|i: int| + 0 <= i < 4 ==> #[trigger] wide4_value(r, i) == prod_unreduced(lanes4_elem(a, i), lanes4_elem(b, i)), +{ + proof { + lemma_clmul_sel(); + lemma_cw_all(); + } + let (a01, a22, b01, b22) = (a[0], a[1], b[0], b[1]); + // `[c0 + c2, c1 + c2]`, and `c0 + c1` in both qwords (the swap stays in its lane). + let (a_s, b_s) = (_mm512_xor_si512(a01, a22), _mm512_xor_si512(b01, b22)); + let a_x = _mm512_xor_si512(a01, _mm512_shuffle_epi32::<0x4E>(a01)); + let b_x = _mm512_xor_si512(b01, _mm512_shuffle_epi32::<0x4E>(b01)); + let p0 = _mm512_clmulepi64_epi128::<0x00>(a01, b01); + let p1 = _mm512_clmulepi64_epi128::<0x11>(a01, b01); + let p2 = _mm512_clmulepi64_epi128::<0x00>(a22, b22); + let p01 = _mm512_clmulepi64_epi128::<0x00>(a_x, b_x); + let p02 = _mm512_clmulepi64_epi128::<0x00>(a_s, b_s); + let p12 = _mm512_clmulepi64_epi128::<0x11>(a_s, b_s); + let xor = |x: __m512i, y: __m512i| -> (z: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(z)[i] == m512(x)[i] ^ m512(y)[i], + { _mm512_xor_si512(x, y) }; + let r = fold(p0, p1, p2, p01, p02, p12, xor); + proof { + lemma_fold(xor, |v: __m512i| m512(v)@, 8, p0, p1, p2, p01, p02, p12, r); + assert forall|i: int| 0 <= i < 4 implies #[trigger] wide4_value(r, i) == prod_unreduced( + lanes4_elem(a, i), + lanes4_elem(b, i), + ) by { + let (x, y) = (lanes4_elem(a, i), lanes4_elem(b, i)); + lemma_karatsuba_words(x, y); + lemma_swap_words(m512(a01)@, 2 * i); + lemma_swap_words(m512(b01)@, 2 * i); + assert(m512(a22)[2 * i + 1] == x.c2 && m512(b22)[2 * i + 1] == y.c2); + assert(m512(a_x)[2 * i] == x.c0 ^ x.c1 && m512(b_x)[2 * i] == y.c0 ^ y.c1); + assert(m512(a_s)[2 * i + 1] == x.c1 ^ x.c2 && m512(b_s)[2 * i + 1] == y.c1 ^ y.c2); + assert forall|k: int, w: int| 0 <= k < 3 && 0 <= w < 2 implies #[trigger] wide4_value(r, i).coeffs[k][w] + == prod_unreduced(x, y).coeffs[k][w] by { + assert((2 * i + w) / 2 == i && (2 * i + w) % 2 == w); + assert(m512(p0)[2 * i + w] == cw(x.c0, y.c0, w)); + assert(m512(p1)[2 * i + w] == cw(x.c1, y.c1, w)); + assert(m512(p2)[2 * i + w] == cw(x.c2, y.c2, w)); + assert(m512(p01)[2 * i + w] == cw(x.c0 ^ x.c1, y.c0 ^ y.c1, w)); + assert(m512(p02)[2 * i + w] == cw(x.c0 ^ x.c2, y.c0 ^ y.c2, w)); + assert(m512(p12)[2 * i + w] == cw(x.c1 ^ x.c2, y.c1 ^ y.c2, w)); + assert(m512(r[k])[2 * i + w] == kara_word(x, y, k, w)); + assert(prod_unreduced(x, y).coeffs[k][w] == split(prod_coeffs(x, y)[k])[w]); + assert(wide4_value(r, i).coeffs[k][w] == m512(r[k])[2 * i + w]); + } + lemma_unreduced_ext(wide4_value(r, i), prod_unreduced(x, y)); + } + } + r +} + +/// Reduce [`mul_lanes4`]'s products into [`lanes4`]'s form. +/// +/// Rewritten for Verus: the parameter pattern `[d0, d1, d2]` is the parameter `d`, indexed. +/// +/// # Safety +/// +/// Requires the `vpclmulqdq` and `avx512f` target features. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f")] +pub unsafe fn reduce_lanes4(d: [__m512i; 3]) -> (r: [__m512i; 2]) + ensures + lanes4_ok(r), + forall|i: int| 0 <= i < 4 ==> #[trigger] lanes4_elem(r, i) == e_value(wide4_value(d, i)), +{ + let (d0, d1, d2) = (d[0], d[1], d[2]); + // SAFETY: the function carries both features. + unsafe { + let r = [ + reduce_lanes512(_mm512_unpacklo_epi64(d0, d1), _mm512_unpackhi_epi64(d0, d1)), + reduce_lanes512(_mm512_unpacklo_epi64(d2, d2), _mm512_unpackhi_epi64(d2, d2)), + ]; + proof { + assert forall|i: int| 0 <= i < 4 implies #[trigger] m512(r[1])[2 * i + 1] == m512(r[1])[2 * i] by { + assert(m512(r[1])[2 * i] == reduce_word(m512(d2)[2 * i], m512(d2)[2 * i + 1])); + } + assert forall|i: int| 0 <= i < 4 implies #[trigger] lanes4_elem(r, i) == e_value(wide4_value(d, i)) by { + assert(m512(r[0])[2 * i] == reduce_word(m512(d0)[2 * i], m512(d0)[2 * i + 1])); + assert(m512(r[0])[2 * i + 1] == reduce_word(m512(d1)[2 * i], m512(d1)[2 * i + 1])); + assert(m512(r[1])[2 * i] == reduce_word(m512(d2)[2 * i], m512(d2)[2 * i + 1])); + } + } + r + } +} + +/// The four lanes' sum: XOR of the lanes, then the lazy-reduction identity. +pub proof fn lemma_sum_lanes512(d: [__m512i; 3]) + ensures + e_value(sum_lanes512(d)) == e_add( + e_add(e_add(e_value(wide512_lane(d, 0)), e_value(wide512_lane(d, 1))), e_value(wide512_lane(d, 2))), + e_value(wide512_lane(d, 3)), + ), +{ + let w = |i: int| wide512_lane(d, i); + let u = u_xor(u_xor(u_xor(w(0), w(1)), w(2)), w(3)); + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] sum_lanes512(d).coeffs[c][x] + == u.coeffs[c][x] by { + let (l0, l1, l2, l3) = (m512(d[c])[x], m512(d[c])[2 + x], m512(d[c])[4 + x], m512(d[c])[6 + x]); + assert((l0 ^ l2) ^ (l1 ^ l3) == ((l0 ^ l1) ^ l2) ^ l3) by (bit_vector); + } + lemma_unreduced_ext(sum_lanes512(d), u); + lemma_e_value_xor(w(0), w(1)); + lemma_e_value_xor(u_xor(w(0), w(1)), w(2)); + lemma_e_value_xor(u_xor(u_xor(w(0), w(1)), w(2)), w(3)); +} + +/// The unreduced value in lane `i` of three 512-bit coefficient registers. +pub open spec fn wide512_lane(d: [__m512i; 3], i: int) -> F192Unreduced { + F192Unreduced { + coeffs: [ + [m512(d[0])[2 * i], m512(d[0])[2 * i + 1]], + [m512(d[1])[2 * i], m512(d[1])[2 * i + 1]], + [m512(d[2])[2 * i], m512(d[2])[2 * i + 1]], + ], + } +} + +/// The sum of the four 128-bit lanes of each of three unreduced coefficient registers: of [`mul_lanes4`]'s lanes, or of an [`F192x8Sum`]. +/// +/// Rewritten for Verus: the lane fold mapped over `d` is its closure called on each register; the closure carries +/// its specification. +/// +/// # Safety +/// +/// Requires the `avx512f` target feature. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn sum_lanes4(d: [__m512i; 3]) -> (r: F192Unreduced) + ensures + r == sum_lanes512(d), + e_value(r) == wide4_sum(d), +{ + let lane = |a: __m512i| -> (q: [u64; 2]) + ensures + q[0] == sum4(a, 0), + q[1] == sum4(a, 1), + { + let half = _mm256_xor_si256(_mm512_castsi512_si256(a), _mm512_extracti64x4_epi64::<1>(a)); + let q = _mm_xor_si128(_mm256_castsi256_si128(half), _mm256_extracti128_si256::<1>(half)); + proof { + lemma_imm_one(); + assert(m128(q)[0] == sum4(a, 0) && m128(q)[1] == sum4(a, 1)); + } + // SAFETY: the reinterpret is between 128-bit values. + unsafe { transmute::<__m128i, [u64; 2]>(q) } + }; + let coeffs = [lane(d[0]), lane(d[1]), lane(d[2])]; + let r = F192Unreduced { coeffs }; + proof { + lemma_unreduced_ext(r, sum_lanes512(d)); + lemma_sum_lanes512(d); + assert(wide512_lane(d, 0) == wide4_value(d, 0) && wide512_lane(d, 1) == wide4_value(d, 1)); + assert(wide512_lane(d, 2) == wide4_value(d, 2) && wide512_lane(d, 3) == wide4_value(d, 3)); + } + r +} + +// --------------------------------------------------------------------------------------------- +// Planar kernels: eight elements in coefficient planes +// --------------------------------------------------------------------------------------------- +/// The word [`F192x8::karatsuba`]'s immediate selects in each lane: 0 for `0x00`, 1 for `0x11`. +pub open spec fn imm_sel(imm: i32) -> int { + if imm == 0 { + 0 + } else { + 1 + } +} + +/// `vpternlogq` with immediate `0x96` is the three-way XOR. +pub proof fn lemma_ternlog_96(a: u64, b: u64, c: u64) + ensures + ternlog(0x96i32 as u8, a, b, c) == a ^ b ^ c, +{ + assert((0x96u8 >> 0u8) & 1 == 0 && (0x96u8 >> 1u8) & 1 == 1 && (0x96u8 >> 2u8) & 1 == 1 && (0x96u8 >> 3u8) & 1 + == 0) by (bit_vector); + assert((0x96u8 >> 4u8) & 1 == 1 && (0x96u8 >> 5u8) & 1 == 0 && (0x96u8 >> 6u8) & 1 == 0 && (0x96u8 >> 7u8) & 1 + == 1) by (bit_vector); + let (z, o) = (0u64, !0u64); + assert((z & !a & !b & !c) | (o & !a & !b & c) | (o & !a & b & !c) | (z & !a & b & c) | (o & a & !b & !c) | (z & a + & !b & c) | (z & a & b & !c) | (o & a & b & c) == a ^ b ^ c) by (bit_vector) + requires + z == 0u64, + o == !0u64, + ; +} + +/// Eight elements in coefficient planes: qword `l` of plane `k` is coefficient `k` of element `l`. +/// +/// A product needs no packing: CLMUL immediate 0x00 multiplies the even elements, 0x11 the odd ones. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[derive(Clone, Copy, Debug)] +pub struct F192x8(pub [__m512i; 3]); + +/// A sum of [`F192x8`] products, unreduced: every 128-bit lane of plane `k` holds part of coefficient `k`. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[derive(Clone, Copy, Debug)] +pub struct F192x8Sum([__m512i; 3]); + +/// Element `l` of an [`F192x8`]. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub open spec fn x8_elem(v: F192x8, l: int) -> F192 { + F192 { c0: m512(v.0[0])[l], c1: m512(v.0[1])[l], c2: m512(v.0[2])[l] } +} + +/// `s` plus the first `n` lane-wise products of `a` and `b`, added in lane order. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub open spec fn add_prods(s: F192, a: F192x8, b: F192x8, n: nat) -> F192 + decreases n, +{ + if n == 0 { + s + } else { + e_add(add_prods(s, a, b, (n - 1) as nat), e_mul(x8_elem(a, n - 1), x8_elem(b, n - 1))) + } +} + +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +impl F192x8 { + /// The lane-wise sum. + /// + /// Rewritten for Verus: `[0, 1, 2].map(..)` is its closure called on each plane. + /// + /// # Safety + /// + /// Requires the features this impl is compiled with. + #[inline] + #[target_feature(enable = "avx512f")] + pub fn add(self, rhs: Self) -> (r: Self) + ensures + forall|l: int| 0 <= l < 8 ==> #[trigger] x8_elem(r, l) == e_add(x8_elem(self, l), x8_elem(rhs, l)), + { + let f = |k: usize| -> (v: __m512i) + requires + k < 3, + ensures + forall|l: int| 0 <= l < 8 ==> #[trigger] m512(v)[l] == m512(self.0[k as int])[l] ^ m512(rhs.0[k as int])[l], + { _mm512_xor_si512(self.0[k], rhs.0[k]) }; + Self([f(0), f(1), f(2)]) + } + + /// The y-folded Karatsuba products of the even (`IMM = 0x00`) or odd (`IMM = 0x11`) elements, one per lane. + /// + /// Rewritten for Verus: the array patterns become indexing; the closures carry their specifications; the six + /// products are bound to names before `fold`. + #[inline] + #[target_feature(enable = "vpclmulqdq", enable = "avx512f")] + fn karatsuba(self, rhs: Self) -> (r: [__m512i; 3]) + requires + IMM == 0x00 || IMM == 0x11, + ensures + forall|k: int, i: int| + 0 <= k < 3 && 0 <= i < 8 ==> #[trigger] m512(r[k])[i] == split( + prod_coeffs(x8_elem(self, 2 * (i / 2) + imm_sel(IMM)), x8_elem(rhs, 2 * (i / 2) + imm_sel(IMM)))[k], + )[i % 2], + { + let ((a0, a1, a2), (b0, b1, b2)) = ((self.0[0], self.0[1], self.0[2]), (rhs.0[0], rhs.0[1], rhs.0[2])); + let mul = |x: __m512i, y: __m512i| -> (z: __m512i) + requires + IMM == 0x00 || IMM == 0x11, + ensures + forall|i: int| + 0 <= i < 8 ==> #[trigger] m512(z)[i] == cw( + m512(x)[2 * (i / 2) + imm_sel(IMM)], + m512(y)[2 * (i / 2) + imm_sel(IMM)], + i % 2, + ), + { + proof { + lemma_clmul_sel(); + lemma_cw_all(); + } + _mm512_clmulepi64_epi128::(x, y) + }; + let xor = |x: __m512i, y: __m512i| -> (z: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(z)[i] == m512(x)[i] ^ m512(y)[i], + { _mm512_xor_si512(x, y) }; + let p0 = mul(a0, b0); + let p1 = mul(a1, b1); + let p2 = mul(a2, b2); + let p01 = mul(xor(a0, a1), xor(b0, b1)); + let p02 = mul(xor(a0, a2), xor(b0, b2)); + let p12 = mul(xor(a1, a2), xor(b1, b2)); + let r = fold(p0, p1, p2, p01, p02, p12, xor); + proof { + lemma_fold(xor, |v: __m512i| m512(v)@, 8, p0, p1, p2, p01, p02, p12, r); + assert forall|k: int, i: int| 0 <= k < 3 && 0 <= i < 8 implies #[trigger] m512(r[k])[i] == split( + prod_coeffs(x8_elem(self, 2 * (i / 2) + imm_sel(IMM)), x8_elem(rhs, 2 * (i / 2) + imm_sel(IMM)))[k], + )[i % 2] by { + let l = 2 * (i / 2) + imm_sel(IMM); + let (x, y) = (x8_elem(self, l), x8_elem(rhs, l)); + let w = i % 2; + lemma_karatsuba_words(x, y); + assert(m512(p0)[i] == cw(x.c0, y.c0, w)); + assert(m512(p1)[i] == cw(x.c1, y.c1, w)); + assert(m512(p2)[i] == cw(x.c2, y.c2, w)); + assert(m512(p01)[i] == cw(x.c0 ^ x.c1, y.c0 ^ y.c1, w)); + assert(m512(p02)[i] == cw(x.c0 ^ x.c2, y.c0 ^ y.c2, w)); + assert(m512(p12)[i] == cw(x.c1 ^ x.c2, y.c1 ^ y.c2, w)); + assert(m512(r[k])[i] == kara_word(x, y, k, w)); + } + } + r + } + + /// The eight lane-wise products. + /// + /// Rewritten for Verus: `[0, 1, 2].map(..)` is its closure called on each plane. + /// + /// # Safety + /// + /// Requires the features this impl is compiled with. + #[inline] + #[target_feature(enable = "vpclmulqdq", enable = "avx512f")] + pub fn mul(self, rhs: Self) -> (r: Self) + ensures + forall|l: int| 0 <= l < 8 ==> #[trigger] x8_elem(r, l) == e_mul(x8_elem(self, l), x8_elem(rhs, l)), + { + let (even, odd) = (self.karatsuba::<0x00>(rhs), self.karatsuba::<0x11>(rhs)); + // Lane j of `even` is element 2j's product and of `odd` element 2j + 1's: unpacking restores qword order. + // SAFETY: the function carries both features. + let f = |k: usize| -> (v: __m512i) + requires + k < 3, + ensures + forall|i: int| + 0 <= i < 8 ==> #[trigger] m512(v)[i] == if i % 2 == 0 { + reduce_word(m512(even[k as int])[i], m512(even[k as int])[i + 1]) + } else { + reduce_word(m512(odd[k as int])[i - 1], m512(odd[k as int])[i]) + }, + { + unsafe { + reduce_lanes512(_mm512_unpacklo_epi64(even[k], odd[k]), _mm512_unpackhi_epi64(even[k], odd[k])) + } + }; + let r = Self([f(0), f(1), f(2)]); + proof { + assert forall|l: int| 0 <= l < 8 implies #[trigger] x8_elem(r, l) == e_mul(x8_elem(self, l), x8_elem(rhs, l)) by { + let (j, h) = (l / 2, l % 2); + let (x, y) = (x8_elem(self, l), x8_elem(rhs, l)); + lemma_coeffs_reduce(x, y); + let pc = prod_coeffs(x, y); + if h == 0 { + assert(m512(even[0])[l] == split(pc[0])[0] && m512(even[0])[l + 1] == split(pc[0])[1]); + assert(m512(even[1])[l] == split(pc[1])[0] && m512(even[1])[l + 1] == split(pc[1])[1]); + assert(m512(even[2])[l] == split(pc[2])[0] && m512(even[2])[l + 1] == split(pc[2])[1]); + } else { + assert(m512(odd[0])[l - 1] == split(pc[0])[0] && m512(odd[0])[l] == split(pc[0])[1]); + assert(m512(odd[1])[l - 1] == split(pc[1])[0] && m512(odd[1])[l] == split(pc[1])[1]); + assert(m512(odd[2])[l - 1] == split(pc[2])[0] && m512(odd[2])[l] == split(pc[2])[1]); + } + assert(m512(r.0[0])[l] == reduce_word(split(pc[0])[0], split(pc[0])[1])); + assert(m512(r.0[1])[l] == reduce_word(split(pc[1])[0], split(pc[1])[1])); + assert(m512(r.0[2])[l] == reduce_word(split(pc[2])[0], split(pc[2])[1])); + } + } + r + } +} + +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +impl F192x8Sum { + /// The unreduced value held: each plane's four lanes summed. + pub closed spec fn value(self) -> F192Unreduced { + sum_lanes512(self.0) + } + + /// The empty sum. + /// + /// # Safety + /// + /// Requires the features this impl is compiled with. + #[inline] + #[target_feature(enable = "avx512f")] + pub fn zero() -> (r: Self) + ensures + r.value() == F192Unreduced::ZERO, + e_value(r.value()) == F192::ZERO, + { + let r = Self([_mm512_setzero_si512(); 3]); + proof { + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] r.value().coeffs[c][x] + == F192Unreduced::ZERO.coeffs[c][x] by { + assert(m512(r.0[c])[x] == 0 && m512(r.0[c])[4 + x] == 0 && m512(r.0[c])[2 + x] == 0 && m512(r.0[c])[6 + + x] == 0); + assert((0u64 ^ 0u64) ^ (0u64 ^ 0u64) == 0u64) by (bit_vector); + } + lemma_unreduced_ext(r.value(), F192Unreduced::ZERO); + lemma_u_zero(); + } + r + } + + /// Add the eight lane-wise products of `a` and `b`. + /// + /// Rewritten for Verus: the plane loop writes through a copy of the registers. + /// + /// # Safety + /// + /// Requires the features this impl is compiled with. + #[inline] + #[target_feature(enable = "vpclmulqdq", enable = "avx512f")] + pub fn mul_add(&mut self, a: F192x8, b: F192x8) + ensures + e_value(final(self).value()) == add_prods(e_value(old(self).value()), a, b, 8), + { + let (even, odd) = (a.karatsuba::<0x00>(b), a.karatsuba::<0x11>(b)); + let ghost before = *self; + let mut planes = self.0; + for k in 0..3usize + invariant + forall|c: int, x: int| + 0 <= c < k && 0 <= x < 8 ==> #[trigger] m512(planes[c])[x] == m512(before.0[c])[x] ^ m512(even[c])[x] + ^ m512(odd[c])[x], + forall|c: int| k <= c < 3 ==> #[trigger] planes[c] == before.0[c], + { + let v = _mm512_ternarylogic_epi64::<0x96>(planes[k], even[k], odd[k]); + proof { + assert forall|x: int| 0 <= x < 8 implies #[trigger] m512(v)[x] == m512(planes[k as int])[x] ^ m512( + even[k as int], + )[x] ^ m512(odd[k as int])[x] by { + lemma_ternlog_96(m512(planes[k as int])[x], m512(even[k as int])[x], m512(odd[k as int])[x]); + } + } + planes[k] = v; + } + self.0 = planes; + proof { + lemma_x8_mul_add(before, *self, a, b, even, odd); + } + } + + /// The whole sum, its lanes folded together. + /// + /// # Safety + /// + /// Requires the features this impl is compiled with. + #[inline] + #[target_feature(enable = "avx512f")] + pub fn total(self) -> (r: F192Unreduced) + ensures + r == self.value(), + { + // SAFETY: the function carries `avx512f`. + unsafe { sum_lanes4(self.0) } + } +} + +/// `u` with the first `n` lane-wise unreduced products of `a` and `b` XORed on, in lane order. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub open spec fn u_prods(u: F192Unreduced, a: F192x8, b: F192x8, n: nat) -> F192Unreduced + decreases n, +{ + if n == 0 { + u + } else { + u_xor(u_prods(u, a, b, (n - 1) as nat), prod_unreduced(x8_elem(a, n - 1), x8_elem(b, n - 1))) + } +} + +/// Word `x` of coefficient `c` of [`u_prods`], from word `s` of `u`. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +pub open spec fn w_prods(s: u64, a: F192x8, b: F192x8, n: nat, c: int, x: int) -> u64 + decreases n, +{ + if n == 0 { + s + } else { + w_prods(s, a, b, (n - 1) as nat, c, x) ^ split(prod_coeffs(x8_elem(a, n - 1), x8_elem(b, n - 1))[c])[x] + } +} + +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +proof fn lemma_u_prods_words(u: F192Unreduced, a: F192x8, b: F192x8, n: nat) + ensures + forall|c: int, x: int| + 0 <= c < 3 && 0 <= x < 2 ==> #[trigger] u_prods(u, a, b, n).coeffs[c][x] == w_prods(u.coeffs[c][x], a, b, n, c, x), + decreases n, +{ + if n > 0 { + lemma_u_prods_words(u, a, b, (n - 1) as nat); + lemma_u_xor_words(u_prods(u, a, b, (n - 1) as nat), prod_unreduced(x8_elem(a, n - 1), x8_elem(b, n - 1))); + lemma_prod_unreduced_words(x8_elem(a, n - 1), x8_elem(b, n - 1)); + } +} + +/// The lazy-reduction identity for the products: reducing once equals adding the reduced products. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +proof fn lemma_u_prods_value(u: F192Unreduced, a: F192x8, b: F192x8, n: nat) + ensures + e_value(u_prods(u, a, b, n)) == add_prods(e_value(u), a, b, n), + decreases n, +{ + if n > 0 { + lemma_u_prods_value(u, a, b, (n - 1) as nat); + lemma_e_value_xor(u_prods(u, a, b, (n - 1) as nat), prod_unreduced(x8_elem(a, n - 1), x8_elem(b, n - 1))); + lemma_prod_unreduced(x8_elem(a, n - 1), x8_elem(b, n - 1)); + } +} + +/// [`F192x8Sum::mul_add`]'s registers hold the old sum plus the eight products. +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +proof fn lemma_x8_mul_add(before: F192x8Sum, after: F192x8Sum, a: F192x8, b: F192x8, even: [__m512i; 3], odd: [__m512i; 3]) + requires + forall|c: int, x: int| + 0 <= c < 3 && 0 <= x < 8 ==> #[trigger] m512(after.0[c])[x] == m512(before.0[c])[x] ^ m512(even[c])[x] + ^ m512(odd[c])[x], + forall|k: int, i: int| + 0 <= k < 3 && 0 <= i < 8 ==> #[trigger] m512(even[k])[i] == split( + prod_coeffs(x8_elem(a, 2 * (i / 2)), x8_elem(b, 2 * (i / 2)))[k], + )[i % 2], + forall|k: int, i: int| + 0 <= k < 3 && 0 <= i < 8 ==> #[trigger] m512(odd[k])[i] == split( + prod_coeffs(x8_elem(a, 2 * (i / 2) + 1), x8_elem(b, 2 * (i / 2) + 1))[k], + )[i % 2], + ensures + e_value(after.value()) == add_prods(e_value(before.value()), a, b, 8), +{ + let u = before.value(); + lemma_u_prods_words(u, a, b, 8); + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] after.value().coeffs[c][x] == u_prods( + u, + a, + b, + 8, + ).coeffs[c][x] by { + lemma_x8_word(before, after, a, b, even, odd, c, x); + } + lemma_unreduced_ext(after.value(), u_prods(u, a, b, 8)); + lemma_u_prods_value(u, a, b, 8); +} + +/// The words of a XOR of unreduced values. +pub proof fn lemma_u_xor_words(u: F192Unreduced, v: F192Unreduced) + ensures + forall|c: int, x: int| + 0 <= c < 3 && 0 <= x < 2 ==> #[trigger] u_xor(u, v).coeffs[c][x] == u.coeffs[c][x] ^ v.coeffs[c][x], +{ + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] u_xor(u, v).coeffs[c][x] == u.coeffs[c][x] + ^ v.coeffs[c][x] by { + if c == 0 { + if x == 0 { + } else { + } + } else if c == 1 { + if x == 0 { + } else { + } + } else { + if x == 0 { + } else { + } + } + } +} + +/// The words of the schoolbook unreduced product. +pub proof fn lemma_prod_unreduced_words(a: F192, b: F192) + ensures + forall|c: int, x: int| + 0 <= c < 3 && 0 <= x < 2 ==> #[trigger] prod_unreduced(a, b).coeffs[c][x] == split(prod_coeffs(a, b)[c])[x], +{ + assert forall|c: int, x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] prod_unreduced(a, b).coeffs[c][x] == split( + prod_coeffs(a, b)[c], + )[x] by { + if c == 0 { + if x == 0 { + } else { + } + } else if c == 1 { + if x == 0 { + } else { + } + } else { + if x == 0 { + } else { + } + } + } +} + +#[cfg(all(target_feature = "vpclmulqdq", target_feature = "avx512f"))] +proof fn lemma_x8_word( + before: F192x8Sum, + after: F192x8Sum, + a: F192x8, + b: F192x8, + even: [__m512i; 3], + odd: [__m512i; 3], + c: int, + x: int, +) + requires + 0 <= c < 3, + 0 <= x < 2, + forall|c: int, x: int| + 0 <= c < 3 && 0 <= x < 8 ==> #[trigger] m512(after.0[c])[x] == m512(before.0[c])[x] ^ m512(even[c])[x] + ^ m512(odd[c])[x], + forall|k: int, i: int| + 0 <= k < 3 && 0 <= i < 8 ==> #[trigger] m512(even[k])[i] == split( + prod_coeffs(x8_elem(a, 2 * (i / 2)), x8_elem(b, 2 * (i / 2)))[k], + )[i % 2], + forall|k: int, i: int| + 0 <= k < 3 && 0 <= i < 8 ==> #[trigger] m512(odd[k])[i] == split( + prod_coeffs(x8_elem(a, 2 * (i / 2) + 1), x8_elem(b, 2 * (i / 2) + 1))[k], + )[i % 2], + ensures + after.value().coeffs[c][x] == w_prods(before.value().coeffs[c][x], a, b, 8, c, x), +{ + reveal_with_fuel(w_prods, 9); + let t = |l: int| split(prod_coeffs(x8_elem(a, l), x8_elem(b, l))[c])[x]; + assert(m512(even[c])[x] == t(0) && m512(odd[c])[x] == t(1)); + assert(m512(even[c])[2 + x] == t(2) && m512(odd[c])[2 + x] == t(3)); + assert(m512(even[c])[4 + x] == t(4) && m512(odd[c])[4 + x] == t(5)); + assert(m512(even[c])[6 + x] == t(6) && m512(odd[c])[6 + x] == t(7)); + let s = |l: int| m512(before.0[c])[2 * l + x]; + let (s0, s1, s2, s3) = (s(0), s(1), s(2), s(3)); + let (t0, t1, t2, t3, t4, t5, t6, t7) = (t(0), t(1), t(2), t(3), t(4), t(5), t(6), t(7)); + assert(((s0 ^ t0 ^ t1) ^ (s2 ^ t4 ^ t5)) ^ ((s1 ^ t2 ^ t3) ^ (s3 ^ t6 ^ t7)) == ((s0 ^ s2) ^ (s1 ^ s3)) ^ t0 ^ t1 ^ t2 + ^ t3 ^ t4 ^ t5 ^ t6 ^ t7) by (bit_vector); + assert(after.value().coeffs[c][x] == sum4(after.0[c], x)); + assert(before.value().coeffs[c][x] == sum4(before.0[c], x)); +} + +} // verus! + +#[cfg(all( + target_feature = "vpclmulqdq", + target_feature = "avx2", + not(target_feature = "avx512f") +))] +#[path = "gf2_64x3_x86_avx2.rs"] +mod avx2; +#[cfg(all( + target_feature = "vpclmulqdq", + target_feature = "avx2", + not(target_feature = "avx512f") +))] +pub use avx2::*; diff --git a/verification/verus/src/gf2_64x3_x86_avx2.rs b/verification/verus/src/gf2_64x3_x86_avx2.rs new file mode 100644 index 0000000000..28af7473ee --- /dev/null +++ b/verification/verus/src/gf2_64x3_x86_avx2.rs @@ -0,0 +1,483 @@ +//! AVX2/VPCLMULQDQ lane-major copies. Half h contains elements 2h and 2h+1. +//! Iterator/array closures are spelled as bounded loops; pointer helpers are trusted explicitly. +use super::*; + +verus! { +pub type Lanes4 = [[__m256i; 2]; 2]; +pub type Wide4 = [[__m256i; 3]; 2]; +pub type MixedAcc8 = [[__m256i; 6]; 2]; + +pub open spec fn half_elem(l: [__m256i; 2], i: int) -> F192 { + F192 { c0: m256(l[0])[2*i], c1: m256(l[0])[2*i+1], c2: m256(l[1])[2*i] } +} +pub open spec fn half_ok(l: [__m256i; 2]) -> bool { + forall|i: int| 0 <= i < 2 ==> #[trigger] m256(l[1])[2*i+1] == m256(l[1])[2*i] +} +pub open spec fn half_wide(d: [__m256i; 3], i: int) -> F192Unreduced { + F192Unreduced { coeffs: [[m256(d[0])[2*i], m256(d[0])[2*i+1]], + [m256(d[1])[2*i], m256(d[1])[2*i+1]], [m256(d[2])[2*i], m256(d[2])[2*i+1]]] } +} +pub open spec fn lanes4_elem(l: Lanes4, i: int) -> F192 { half_elem(l[i/2], i%2) } +pub open spec fn lanes4_ok(l: Lanes4) -> bool { half_ok(l[0]) && half_ok(l[1]) } +pub open spec fn wide4_value(d: Wide4, i: int) -> F192Unreduced { half_wide(d[i/2], i%2) } +pub open spec fn wide4_sum(d: Wide4) -> F192 { + e_add(e_add(e_add(e_value(wide4_value(d,0)), e_value(wide4_value(d,1))), + e_value(wide4_value(d,2))), e_value(wide4_value(d,3))) +} +pub open spec fn acc8_row(a: MixedAcc8, i: int) -> F192Unreduced { + let h = i/4; + let j = (i%4)/2; + F192Unreduced { coeffs: [ + [m256(a[h][acc_reg(i,0)])[2*j], m256(a[h][acc_reg(i,0)])[2*j+1]], + [m256(a[h][acc_reg(i,1)])[2*j], m256(a[h][acc_reg(i,1)])[2*j+1]], + [m256(a[h][acc_reg(i,2)])[2*j], m256(a[h][acc_reg(i,2)])[2*j+1]]] } +} + +#[inline] +pub unsafe fn lanes4(v: [F192; 4]) -> (r: Lanes4) + ensures lanes4_ok(r), forall|i: int| 0 <= i < 4 ==> #[trigger] lanes4_elem(r,i) == v[i], +{ + proof { lemma_i64_round_trip_all(); } + let half = |a: F192, b: F192| -> (r: [__m256i; 2]) + ensures half_ok(r), half_elem(r,0) == a, half_elem(r,1) == b, + { + [_mm256_set_epi64x(b.c1 as i64,b.c0 as i64,a.c1 as i64,a.c0 as i64), + _mm256_set_epi64x(b.c2 as i64,b.c2 as i64,a.c2 as i64,a.c2 as i64)] + }; + [half(v[0],v[1]),half(v[2],v[3])] +} + +/// Trusted pointer loads only, before the production permutation. +#[verifier::external_body] +pub fn load_half(v: &[F192;4], h: usize) -> (r: (__m256i, __m256d)) + requires h < 2, + ensures + m256(r.0) == [v[2*h].c0,v[2*h].c1,v[2*h+1].c0,v[2*h+1].c1], + m256d(r.1) == [v[2*h].c2,v[2*h+1].c0,v[2*h+1].c1,v[2*h+1].c2], +{ + unsafe { + let w = v.as_ptr().cast::().add(6*h); + (_mm256_loadu2_m128i(w.add(3).cast(),w.cast()),_mm256_loadu_pd(w.add(2).cast())) + } +} +#[inline] +pub unsafe fn load_lanes4(v: &[F192;4]) -> (r: Lanes4) + ensures lanes4_ok(r), forall|i: int| 0 <= i < 4 ==> #[trigger] lanes4_elem(r,i) == v[i], +{ + let half = |h: usize| -> (r: [__m256i;2]) + requires h < 2, + ensures half_ok(r), forall|i: int| 0 <= i < 2 ==> #[trigger] half_elem(r,i) == v[2*h+i], + { + proof { assert((2u64 >> 1u64) & 1u64 == 1u64 && (0u64 >> 1u64) & 1u64 == 0u64) by (bit_vector); } + let (c01, tail) = load_half(v,h); + let c22 = _mm256_permutevar_pd(tail,_mm256_set_epi64x(2,2,0,0)); + [c01,_mm256_castpd_si256(c22)] + }; + [half(0),half(1)] +} + +/// Trusted two stores for one six-word half. Other slots are preserved. +#[verifier::external_body] +pub fn store_half(out: &mut [MaybeUninit;4], h: usize, head: __m128i, tail: __m256i) + requires h < 2, + ensures + final(out)[2*h].mem_contents() == MemContents::Init(F192 { c0:m128(head)[0],c1:m128(head)[1],c2:m256(tail)[0] }), + final(out)[2*h+1].mem_contents() == MemContents::Init(F192 { c0:m256(tail)[1],c1:m256(tail)[2],c2:m256(tail)[3] }), + forall|i: int| 0 <= i < 4 && !(2*h <= i < 2*h+2) ==> #[trigger] final(out)[i].mem_contents() == old(out)[i].mem_contents(), +{ + unsafe { + let w = out.as_mut_ptr().cast::().add(6*h); + _mm_storeu_si128(w.cast(),head); + _mm256_storeu_si256(w.add(2).cast(),tail); + } +} +pub proof fn lemma_move_selectors() + ensures + ((0xF8u8 >> 2u8) & 3u8) == 2u8, ((0xF8u8 >> 4u8) & 3u8) == 3u8, + (0xC3u8 & 1u8) == 1u8, ((0xC3u8 >> 1u8) & 1u8) == 1u8, + ((0xC3u8 >> 2u8) & 1u8) == 0u8, ((0xC3u8 >> 3u8) & 1u8) == 0u8, + ((0xC3u8 >> 4u8) & 1u8) == 0u8, ((0xC3u8 >> 5u8) & 1u8) == 0u8, + ((0xC3u8 >> 6u8) & 1u8) == 1u8, ((0xC3u8 >> 7u8) & 1u8) == 1u8, + (0x20u8 & 15u8) == 0u8, ((0x20u8 >> 4u8) & 15u8) == 2u8, + (0x31u8 & 15u8) == 1u8, ((0x31u8 >> 4u8) & 15u8) == 3u8, + (0u8 & 8u8) == 0u8, (1u8 & 8u8) == 0u8, (2u8 & 8u8) == 0u8, (3u8 & 8u8) == 0u8, + (0u8 & 3u8) == 0u8, (1u8 & 3u8) == 1u8, (2u8 & 3u8) == 2u8, (3u8 & 3u8) == 3u8, + ((0xC3u8 >> 0u8) & 1u8) == 1u8, + ((0x20u8 >> 0u8) & 15u8) == 0u8, ((0x31u8 >> 0u8) & 15u8) == 1u8, +{ + assert(((0xC3u8 >> 0u8) & 1u8) == 1u8 && ((0x20u8 >> 0u8) & 15u8) == 0u8 + && ((0x31u8 >> 0u8) & 15u8) == 1u8) by (bit_vector); + assert(((0xF8u8 >> 2u8) & 3u8) == 2u8 && ((0xF8u8 >> 4u8) & 3u8) == 3u8) by (bit_vector); + assert((0xC3u8 & 1u8) == 1u8 && ((0xC3u8 >> 1u8) & 1u8) == 1u8 + && ((0xC3u8 >> 2u8) & 1u8) == 0u8 && ((0xC3u8 >> 3u8) & 1u8) == 0u8 + && ((0xC3u8 >> 4u8) & 1u8) == 0u8 && ((0xC3u8 >> 5u8) & 1u8) == 0u8 + && ((0xC3u8 >> 6u8) & 1u8) == 1u8 && ((0xC3u8 >> 7u8) & 1u8) == 1u8) by (bit_vector); + assert((0x20u8 & 15u8) == 0u8 && ((0x20u8 >> 4u8) & 15u8) == 2u8 + && (0x31u8 & 15u8) == 1u8 && ((0x31u8 >> 4u8) & 15u8) == 3u8) by (bit_vector); + assert((0u8 & 8u8) == 0u8 && (1u8 & 8u8) == 0u8 && (2u8 & 8u8) == 0u8 && (3u8 & 8u8) == 0u8 + && (0u8 & 3u8) == 0u8 && (1u8 & 3u8) == 1u8 && (2u8 & 3u8) == 2u8 && (3u8 & 3u8) == 3u8) by (bit_vector); +} + +#[inline] +pub unsafe fn store_lanes4(lanes: Lanes4, out: &mut [MaybeUninit;4]) + requires lanes4_ok(lanes), + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] final(out)[i].mem_contents() == MemContents::Init(lanes4_elem(lanes,i)), +{ + for h in 0..2usize + invariant lanes4_ok(lanes), + forall|i: int| 0 <= i < 2*h ==> #[trigger] out[i].mem_contents() == MemContents::Init(lanes4_elem(lanes,i)), + { + let (c01,c22) = (lanes[h][0],lanes[h][1]); + let perm = _mm256_permute4x64_epi64::<0xF8>(c01); + let tail = _mm256_blend_epi32::<0xC3>(perm,c22); + proof { + lemma_move_selectors(); + assert(half_ok(lanes[h as int])); + assert(m256(perm)[1] == m256(c01)[2] && m256(perm)[2] == m256(c01)[3]); + let (a,b,c,d) = (m256(c22)[0],m256(perm)[1],m256(perm)[2],m256(c22)[3]); + assert((a & 0xFFFF_FFFFu64) | (a & 0xFFFF_FFFF_0000_0000u64) == a + && (b & 0xFFFF_FFFFu64) | (b & 0xFFFF_FFFF_0000_0000u64) == b + && (c & 0xFFFF_FFFFu64) | (c & 0xFFFF_FFFF_0000_0000u64) == c + && (d & 0xFFFF_FFFFu64) | (d & 0xFFFF_FFFF_0000_0000u64) == d) by (bit_vector); + assert(blend_epi32_lane(m256(perm)@,m256(c22)@,0xC3,0) == a); + assert(blend_epi32_lane(m256(perm)@,m256(c22)@,0xC3,1) == b); + assert(blend_epi32_lane(m256(perm)@,m256(c22)@,0xC3,2) == c); + assert(blend_epi32_lane(m256(perm)@,m256(c22)@,0xC3,3) == d); + assert(m256(tail)[0] == m256(c22)[0] && m256(tail)[1] == m256(c01)[2] + && m256(tail)[2] == m256(c01)[3] && m256(tail)[3] == m256(c22)[3]); + } + store_half(out,h,_mm256_castsi256_si128(c01),tail); + } +} + +#[inline] +pub unsafe fn transpose_lanes4(rows: [Lanes4;4]) -> (r: [Lanes4;4]) + ensures + forall|i: int,h: int,p: int,x: int| 0 <= i < 4 && 0 <= h < 2 && 0 <= p < 2 && 0 <= x < 4 ==> + #[trigger] m256(r[i][h][p])[x] == m256(rows[2*h+x/2][i/2][p])[2*(i%2)+x%2], + forall|i: int,j: int| 0 <= i < 4 && 0 <= j < 4 ==> #[trigger] lanes4_elem(r[i],j) == lanes4_elem(rows[j],i), + (forall|i: int| 0 <= i < 4 ==> #[trigger] lanes4_ok(rows[i])) ==> forall|i: int| 0 <= i < 4 ==> #[trigger] lanes4_ok(r[i]), +{ + let pick = |i: usize,h: usize,p: usize| -> (r: __m256i) + requires i < 4,h < 2,p < 2, + ensures forall|x: int| 0 <= x < 4 ==> #[trigger] m256(r)[x] == m256(rows[2*h+x/2][(i/2) as int][p as int])[2*(i%2)+x%2], + { + let (a,b) = (rows[2*h][i/2][p],rows[2*h+1][i/2][p]); + proof { lemma_move_selectors(); } + if i%2 == 0 { + let r = _mm256_permute2x128_si256::<0x20>(a,b); + proof { + assert(m256(r)[0] == m256(a)[0]); assert(m256(r)[1] == m256(a)[1]); + assert(m256(r)[2] == m256(b)[0]); assert(m256(r)[3] == m256(b)[1]); + } + r + } else { + let r = _mm256_permute2x128_si256::<0x31>(a,b); + proof { + assert(m256(r)[0] == m256(a)[2]); assert(m256(r)[1] == m256(a)[3]); + assert(m256(r)[2] == m256(b)[2]); assert(m256(r)[3] == m256(b)[3]); + } + r + } + }; + let mut r = rows; + for i in 0..4usize + invariant + forall|a: usize,h: usize,p: usize| a < 4 && h < 2 && p < 2 ==> #[trigger] pick.requires((a,h,p)), + forall|a: usize,h: usize,p: usize,v: __m256i| a < 4 && h < 2 && p < 2 && #[trigger] pick.ensures((a,h,p),v) ==> + forall|x: int| 0 <= x < 4 ==> #[trigger] m256(v)[x] == m256(rows[2*h+x/2][(a/2) as int][p as int])[2*(a%2)+x%2], + forall|a: int,h: int,p: int,x: int| 0 <= a < i && 0 <= h < 2 && 0 <= p < 2 && 0 <= x < 4 ==> + #[trigger] m256(r[a][h][p])[x] == m256(rows[2*h+x/2][a/2][p])[2*(a%2)+x%2], + { + r[i] = [[pick(i,0,0),pick(i,0,1)],[pick(i,1,0),pick(i,1,1)]]; + } + proof { + assert forall|i: int,j: int| 0 <= i < 4 && 0 <= j < 4 implies #[trigger] lanes4_elem(r[i],j) == lanes4_elem(rows[j],i) by { + assert(m256(r[i][j/2][0])[2*(j%2)] == m256(rows[j][i/2][0])[2*(i%2)]); + assert(m256(r[i][j/2][0])[2*(j%2)+1] == m256(rows[j][i/2][0])[2*(i%2)+1]); + assert(m256(r[i][j/2][1])[2*(j%2)] == m256(rows[j][i/2][1])[2*(i%2)]); + } + if forall|i: int| 0 <= i < 4 ==> #[trigger] lanes4_ok(rows[i]) { + assert forall|i: int| 0 <= i < 4 implies #[trigger] lanes4_ok(r[i]) by { + assert forall|h: int,j: int| 0 <= h < 2 && 0 <= j < 2 implies + #[trigger] m256(r[i][h][1])[2*j+1] == m256(r[i][h][1])[2*j] by { + assert(lanes4_ok(rows[2*h+j])); + assert(half_ok(rows[2*h+j][i/2])); + assert(m256(rows[2*h+j][i/2][1])[2*(i%2)+1] == m256(rows[2*h+j][i/2][1])[2*(i%2)]); + assert(m256(r[i][h][1])[2*j+1] == m256(rows[2*h+j][i/2][1])[2*(i%2)+1]); + assert(m256(r[i][h][1])[2*j] == m256(rows[2*h+j][i/2][1])[2*(i%2)]); + } + } + } + } + r +} + +pub open spec fn lanes_xor(a: [[__m256i;N];2],b: [[__m256i;N];2],r: [[__m256i;N];2]) -> bool { + forall|h: int,i: int,x: int| 0 <= h < 2 && 0 <= i < N && 0 <= x < 4 ==> #[trigger] m256(r[h][i])[x] == m256(a[h][i])[x] ^ m256(b[h][i])[x] +} +#[inline] +pub unsafe fn xor_lanes(a: [[__m256i;N];2],b: [[__m256i;N];2]) -> (r: [[__m256i;N];2]) + ensures lanes_xor(a,b,r), +{ + let mut r = a; + for h in 0..2usize + invariant forall|g: int,i: int,x: int| 0 <= g < h && 0 <= i < N && 0 <= x < 4 ==> #[trigger] m256(r[g][i])[x] == m256(a[g][i])[x] ^ m256(b[g][i])[x], + { + let mut half = a[h]; + for i in 0..N + invariant h < 2, + forall|j: int,x: int| 0 <= j < i && 0 <= x < 4 ==> #[trigger] m256(half[j])[x] == m256(a[h as int][j])[x] ^ m256(b[h as int][j])[x], + { half[i] = _mm256_xor_si256(a[h][i],b[h][i]); } + r[h] = half; + } + r +} +pub proof fn lemma_lanes4_xor(a: Lanes4,b: Lanes4,r: Lanes4) + requires lanes_xor(a,b,r), + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] lanes4_elem(r,i) == e_add(lanes4_elem(a,i),lanes4_elem(b,i)), + lanes4_ok(a) && lanes4_ok(b) ==> lanes4_ok(r), +{} +pub proof fn lemma_wide4_xor(a: Wide4,b: Wide4,r: Wide4) + requires lanes_xor(a,b,r), + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] wide4_value(r,i) == u_xor(wide4_value(a,i),wide4_value(b,i)), +{} +pub proof fn lemma_zeroed_wide4() + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] wide4_value(zeroed_value::(),i) == F192Unreduced::ZERO, +{ axiom_zeroed_m256_pairs::<3>(); } +pub proof fn lemma_zeroed_acc8() + ensures forall|i: int| 0 <= i < 8 ==> #[trigger] acc8_row(zeroed_value::(),i) == F192Unreduced::ZERO, +{ axiom_zeroed_m256_pairs::<6>(); } + +#[inline] +unsafe fn mul_half(a: [__m256i;2],b: [__m256i;2]) -> (r: [__m256i;3]) + requires half_ok(a),half_ok(b), + ensures forall|i: int| 0 <= i < 2 ==> #[trigger] half_wide(r,i) == prod_unreduced(half_elem(a,i),half_elem(b,i)), +{ + proof { lemma_clmul_sel(); lemma_cw_all(); } + let (a01,a22,b01,b22) = (a[0],a[1],b[0],b[1]); + let (a_s,b_s) = (_mm256_xor_si256(a01,a22),_mm256_xor_si256(b01,b22)); + let a_x = _mm256_xor_si256(a01,_mm256_shuffle_epi32::<0x4E>(a01)); + let b_x = _mm256_xor_si256(b01,_mm256_shuffle_epi32::<0x4E>(b01)); + let p0 = _mm256_clmulepi64_epi128::<0x00>(a01,b01); + let p1 = _mm256_clmulepi64_epi128::<0x11>(a01,b01); + let p2 = _mm256_clmulepi64_epi128::<0x00>(a22,b22); + let p01 = _mm256_clmulepi64_epi128::<0x00>(a_x,b_x); + let p02 = _mm256_clmulepi64_epi128::<0x00>(a_s,b_s); + let p12 = _mm256_clmulepi64_epi128::<0x11>(a_s,b_s); + let xor = |x: __m256i,y: __m256i| -> (z: __m256i) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] m256(z)[i] == m256(x)[i] ^ m256(y)[i], + { _mm256_xor_si256(x,y) }; + let r = fold(p0,p1,p2,p01,p02,p12,xor); + proof { + lemma_fold(xor,|v: __m256i| m256(v)@,4,p0,p1,p2,p01,p02,p12,r); + assert forall|i: int| 0 <= i < 2 implies #[trigger] half_wide(r,i) == prod_unreduced(half_elem(a,i),half_elem(b,i)) by { + let (x,y) = (half_elem(a,i),half_elem(b,i)); + lemma_karatsuba_words(x,y); + lemma_swap_words(m256(a01)@,2*i); + lemma_swap_words(m256(b01)@,2*i); + assert(m256(a22)[2*i+1] == x.c2 && m256(b22)[2*i+1] == y.c2); + assert(m256(a_x)[2*i] == x.c0 ^ x.c1 && m256(b_x)[2*i] == y.c0 ^ y.c1); + assert forall|k: int,w: int| 0 <= k < 3 && 0 <= w < 2 implies #[trigger] half_wide(r,i).coeffs[k][w] == prod_unreduced(x,y).coeffs[k][w] by { + assert((2*i+w)/2 == i && (2*i+w)%2 == w); + assert(m256(p0)[2*i+w] == cw(x.c0,y.c0,w)); + assert(m256(p1)[2*i+w] == cw(x.c1,y.c1,w)); + assert(m256(p2)[2*i+w] == cw(x.c2,y.c2,w)); + assert(m256(p01)[2*i+w] == cw(x.c0^x.c1,y.c0^y.c1,w)); + assert(m256(p02)[2*i+w] == cw(x.c0^x.c2,y.c0^y.c2,w)); + assert(m256(p12)[2*i+w] == cw(x.c1^x.c2,y.c1^y.c2,w)); + assert(m256(r[k])[2*i+w] == kara_word(x,y,k,w)); + assert(prod_unreduced(x,y).coeffs[k][w] == split(prod_coeffs(x,y)[k])[w]); + assert(half_wide(r,i).coeffs[k][w] == m256(r[k])[2*i+w]); + } + lemma_unreduced_ext(half_wide(r,i),prod_unreduced(x,y)); + } + } + r +} +pub unsafe fn mul_lanes4(a: Lanes4,b: Lanes4) -> (r: Wide4) + requires lanes4_ok(a),lanes4_ok(b), + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] wide4_value(r,i) == prod_unreduced(lanes4_elem(a,i),lanes4_elem(b,i)), +{ [mul_half(a[0],b[0]),mul_half(a[1],b[1])] } + +unsafe fn reduce_half(d: [__m256i;3]) -> (r: [__m256i;2]) + ensures half_ok(r),forall|i: int| 0 <= i < 2 ==> #[trigger] half_elem(r,i) == e_value(half_wide(d,i)), +{ + let (d0,d1,d2) = (d[0],d[1],d[2]); + let r = [reduce_lanes256(_mm256_unpacklo_epi64(d0,d1),_mm256_unpackhi_epi64(d0,d1)), + reduce_lanes256(_mm256_unpacklo_epi64(d2,d2),_mm256_unpackhi_epi64(d2,d2))]; + proof { + assert forall|i: int| 0 <= i < 2 implies #[trigger] half_elem(r,i) == e_value(half_wide(d,i)) by { + assert(m256(r[0])[2*i] == reduce_word(m256(d0)[2*i],m256(d0)[2*i+1])); + assert(m256(r[0])[2*i+1] == reduce_word(m256(d1)[2*i],m256(d1)[2*i+1])); + assert(m256(r[1])[2*i] == reduce_word(m256(d2)[2*i],m256(d2)[2*i+1])); + } + } + r +} +pub unsafe fn reduce_lanes4(d: Wide4) -> (r: Lanes4) + ensures lanes4_ok(r),forall|i: int| 0 <= i < 4 ==> #[trigger] lanes4_elem(r,i) == e_value(wide4_value(d,i)), +{ [reduce_half(d[0]),reduce_half(d[1])] } + +pub unsafe fn sum_lanes4(d: Wide4) -> (r: F192Unreduced) + ensures e_value(r) == wide4_sum(d), + r == u_xor(u_xor(u_xor(wide4_value(d,0),wide4_value(d,1)),wide4_value(d,2)),wide4_value(d,3)), +{ + proof { lemma_imm_one(); } + let mut coeffs = [[0u64;2];3]; + for k in 0..3usize + invariant forall|c: int,x: int| 0 <= c < k && 0 <= x < 2 ==> #[trigger] coeffs[c][x] == + m256(d[0][c])[x] ^ m256(d[0][c])[2+x] ^ m256(d[1][c])[x] ^ m256(d[1][c])[2+x], + { + proof { lemma_imm_one(); } + let half = _mm256_xor_si256(d[0][k],d[1][k]); + let low = _mm256_castsi256_si128(half); + let high = _mm256_extracti128_si256::<1>(half); + let q = _mm_xor_si128(low,high); + coeffs[k] = transmute::<__m128i,[u64;2]>(q); + proof { + assert forall|x: int| 0 <= x < 2 implies coeffs[k as int][x] == + m256(d[0][k as int])[x] ^ m256(d[0][k as int])[2+x] ^ m256(d[1][k as int])[x] ^ m256(d[1][k as int])[2+x] by { + assert(m128(q)[x] == m256(half)[x] ^ m256(half)[2+x]); + assert(m256(half)[x] == m256(d[0][k as int])[x] ^ m256(d[1][k as int])[x]); + assert(m256(half)[2+x] == m256(d[0][k as int])[2+x] ^ m256(d[1][k as int])[2+x]); + let (a,b,c,e) = (m256(d[0][k as int])[x],m256(d[0][k as int])[2+x],m256(d[1][k as int])[x],m256(d[1][k as int])[2+x]); + assert((a^c)^(b^e) == a^b^c^e) by (bit_vector); + } + } + } + let r = F192Unreduced { coeffs }; + proof { + let (a,b,c,e) = (wide4_value(d,0),wide4_value(d,1),wide4_value(d,2),wide4_value(d,3)); + lemma_u_xor_words(a,b); lemma_u_xor_words(u_xor(a,b),c); lemma_u_xor_words(u_xor(u_xor(a,b),c),e); + let s = u_xor(u_xor(u_xor(a,b),c),e); + assert forall|k: int,x: int| 0 <= k < 3 && 0 <= x < 2 implies #[trigger] r.coeffs[k][x] == s.coeffs[k][x] by { + assert(a.coeffs[k][x] == m256(d[0][k])[x]); + assert(b.coeffs[k][x] == m256(d[0][k])[2+x]); + assert(c.coeffs[k][x] == m256(d[1][k])[x]); + assert(e.coeffs[k][x] == m256(d[1][k])[2+x]); + } + lemma_unreduced_ext(r,s); + lemma_e_value_xor(a,b); lemma_e_value_xor(u_xor(a,b),c); lemma_e_value_xor(u_xor(u_xor(a,b),c),e); + } + r +} + +unsafe fn mul_by_pairs256(lo: __m256i,hi: __m256i,c2: __m256i,k: __m256i) -> (r: [__m256i;6]) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r[0])[i] == cw(m256(lo)[2*(i/2)],m256(k)[2*(i/2)],i%2), + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r[1])[i] == cw(m256(lo)[2*(i/2)+1],m256(k)[2*(i/2)],i%2), + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r[2])[i] == cw(m256(hi)[2*(i/2)],m256(k)[2*(i/2)+1],i%2), + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r[3])[i] == cw(m256(hi)[2*(i/2)+1],m256(k)[2*(i/2)+1],i%2), + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r[4])[i] == cw(m256(c2)[2*(i/2)],m256(k)[2*(i/2)],i%2), + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r[5])[i] == cw(m256(c2)[2*(i/2)+1],m256(k)[2*(i/2)+1],i%2), +{ + proof { lemma_clmul_sel(); lemma_cw_all(); } + [_mm256_clmulepi64_epi128::<0x00>(lo,k),_mm256_clmulepi64_epi128::<0x01>(lo,k), + _mm256_clmulepi64_epi128::<0x10>(hi,k),_mm256_clmulepi64_epi128::<0x11>(hi,k), + _mm256_clmulepi64_epi128::<0x00>(c2,k),_mm256_clmulepi64_epi128::<0x11>(c2,k)] +} +/// Trusted unaligned load of four base scalars. +#[verifier::external_body] +pub fn load_k4(k: &[F64;8],h: usize) -> (r: __m256i) + requires h < 2, + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == k[4*h+i].0, +{ unsafe { _mm256_loadu_si256(k.as_ptr().add(4*h).cast()) } } + +pub unsafe fn mul_base8_add(acc: &mut MixedAcc8,t: F192,k: [F64;8]) + ensures forall|i: int| 0 <= i < 8 ==> #[trigger] acc8_row(*final(acc),i) == u_xor(acc8_row(*old(acc),i),base_unreduced(t,k[i].0)), +{ + proof { lemma_i64_round_trip_all(); } + let t01 = _mm256_broadcastsi128_si256(pair(t.c0,t.c1)); + let t2 = _mm256_set1_epi64x(t.c2 as i64); + for h in 0..2usize + invariant + m256(t01) == [t.c0,t.c1,t.c0,t.c1], m256(t2) == [t.c2,t.c2,t.c2,t.c2], + forall|g: int,c: int,x: int| h <= g < 2 && 0 <= c < 6 && 0 <= x < 4 ==> #[trigger] m256(acc[g][c])[x] == m256(old(acc)[g][c])[x], + forall|i: int| 0 <= i < 4*h ==> #[trigger] acc8_row(*acc,i) == u_xor(acc8_row(*old(acc),i),base_unreduced(t,k[i].0)), + { + let kv = load_k4(&k,h); + let p = mul_by_pairs256(t01,t01,t2,kv); + let ghost before_update = *acc; + let a = acc[h]; + acc[h] = [_mm256_xor_si256(a[0],p[0]),_mm256_xor_si256(a[1],p[1]), + _mm256_xor_si256(a[2],p[2]),_mm256_xor_si256(a[3],p[3]), + _mm256_xor_si256(a[4],p[4]),_mm256_xor_si256(a[5],p[5])]; + proof { + assert forall|i: int| 0 <= i < 4*h implies #[trigger] acc8_row(*acc,i) == acc8_row(before_update,i) by { + assert(i/4 < h); + } + } + proof { + assert forall|i: int| 4*h <= i < 4*h+4 implies #[trigger] acc8_row(*acc,i) == u_xor(acc8_row(*old(acc),i),base_unreduced(t,k[i].0)) by { + lemma_base_words(t,k[i].0); + let (before,after,b) = (acc8_row(*old(acc),i),acc8_row(*acc,i),base_unreduced(t,k[i].0)); + lemma_u_xor_words(before,b); + assert(i/4 == h && 4*h+i%4 == i); + assert forall|c: int,x: int| 0 <= c < 3 && 0 <= x < 2 implies #[trigger] after.coeffs[c][x] == u_xor(before,b).coeffs[c][x] by { + if i%2 == 0 {} else {} + if c == 0 {} else if c == 1 {} else {} + assert(m256(p[acc_reg(i,c)])[2*((i%4)/2)+x] == cw(coef(t,c),k[i].0,x)); + } + lemma_unreduced_ext(after,u_xor(before,b)); + } + } + } +} + +pub unsafe fn mul_base8_reduce(acc: MixedAcc8) -> (r: [F192;8]) + ensures forall|i: int| 0 <= i < 8 ==> #[trigger] r[i] == e_value(acc8_row(acc,i)), +{ + let mut r = [F192::ZERO;8]; + for h in 0..2usize + invariant forall|i: int| 0 <= i < 4*h ==> #[trigger] r[i] == e_value(acc8_row(acc,i)), + { + let a = acc[h]; + let red = |x: __m256i,y: __m256i| -> (r: [u64;4]) + ensures forall|j: int| 0 <= j < 4 ==> #[trigger] r[j] == (if j%2 == 0 { + reduce_word(m256(x)[j],m256(x)[j+1]) + } else { reduce_word(m256(y)[j-1],m256(y)[j]) }), + { + let lo = _mm256_unpacklo_epi64(x,y); + let hi = _mm256_unpackhi_epi64(x,y); + let v = reduce_lanes256(lo,hi); + proof { + assert forall|j: int| 0 <= j < 4 implies #[trigger] m256(v)[j] == (if j%2 == 0 { + reduce_word(m256(x)[j],m256(x)[j+1]) + } else { reduce_word(m256(y)[j-1],m256(y)[j]) }) by { + assert(m256(v)[j] == reduce_word(m256(lo)[j],m256(hi)[j])); + assert(m256(lo)[j] == unpacklo_lane(m256(x)@,m256(y)@,j)); + assert(m256(hi)[j] == unpackhi_lane(m256(x)@,m256(y)@,j)); + } + } + let r = transmute::<__m256i,[u64;4]>(v); + proof { + assert(r == m256(v)); + assert forall|j: int| 0 <= j < 4 implies #[trigger] r[j] == (if j%2 == 0 { + reduce_word(m256(x)[j],m256(x)[j+1]) + } else { reduce_word(m256(y)[j-1],m256(y)[j]) }) by { + assert(m256(v)[j] == reduce_word(m256(lo)[j],m256(hi)[j])); + } + } + r + }; + let even = red(a[0],a[1]); + let odd = red(a[2],a[3]); + let c2 = red(a[4],a[5]); + for i in 0..4usize + invariant h < 2, + forall|j: int| 0 <= j < 4*h+i ==> #[trigger] r[j] == e_value(acc8_row(acc,j)), + forall|j: int| 0 <= j < 4 ==> + #[trigger] even[j] == reduce_word(m256(a[j%2])[2*(j/2)],m256(a[j%2])[2*(j/2)+1]), + forall|j: int| 0 <= j < 4 ==> + #[trigger] odd[j] == reduce_word(m256(a[2+j%2])[2*(j/2)],m256(a[2+j%2])[2*(j/2)+1]), + forall|j: int| 0 <= j < 4 ==> + #[trigger] c2[j] == reduce_word(m256(a[4+j%2])[2*(j/2)],m256(a[4+j%2])[2*(j/2)+1]), + a == acc[h as int], + { + let w = if i%2 == 0 { even } else { odd }; + r[4*h+i] = F192::new(w[2*(i/2)],w[2*(i/2)+1],c2[i]); + } + } + r +} +} // verus! diff --git a/verification/verus/src/gf2_64x3_x86_wrappers.rs b/verification/verus/src/gf2_64x3_x86_wrappers.rs new file mode 100644 index 0000000000..dda48eb90c --- /dev/null +++ b/verification/verus/src/gf2_64x3_x86_wrappers.rs @@ -0,0 +1,178 @@ +//! Copies of the public lane-major wrappers in `gf2_64x3.rs`, using the configured x86 kernels. +use super::x86_64::*; +use super::*; +use core::mem::MaybeUninit; +#[cfg(verus_keep_ghost)] +use vstd::raw_ptr::MemContents; +#[cfg(verus_keep_ghost)] +use vstd::std_specs::maybe_uninit::MaybeUninitAdditionalSpecFns; + +verus! { +#[derive(Clone,Copy)] +pub struct MixedSums8 { acc: MixedAcc8 } +impl MixedSums8 { + pub closed spec fn value(self,i: int) -> F192 { e_value(acc8_row(self.acc,i)) } + pub const fn new() -> (r: Self) + ensures forall|i: int| 0 <= i < 8 ==> #[trigger] r.value(i) == F192::ZERO, + { + proof { lemma_zeroed_acc8(); lemma_u_zero(); } + Self { acc: unsafe { core::mem::zeroed() } } + } + pub fn add(&mut self,t: F192,k: [F64;8]) + ensures forall|i: int| 0 <= i < 8 ==> #[trigger] final(self).value(i) == e_add(old(self).value(i),e_mul(t,e_from_k(k[i].0))), + { + let ghost old_acc = self.acc; + unsafe { mul_base8_add(&mut self.acc,t,k); } + proof { + assert forall|i: int| 0 <= i < 8 implies #[trigger] self.value(i) == e_add(e_value(acc8_row(old_acc,i)),e_mul(t,e_from_k(k[i].0))) by { + lemma_base_unreduced(t,k[i].0); + lemma_e_value_xor(acc8_row(old_acc,i),base_unreduced(t,k[i].0)); + } + } + } + pub fn reduce(self) -> (r: [F192;8]) + ensures forall|i: int| 0 <= i < 8 ==> #[trigger] r[i] == self.value(i), + { unsafe { mul_base8_reduce(self.acc) } } +} +impl Default for MixedSums8 { + fn default() -> (r: Self) + ensures forall|i: int| 0 <= i < 8 ==> #[trigger] r.value(i) == F192::ZERO, + { Self::new() } +} + +#[derive(Clone,Copy)] +pub struct F192x4 { lanes: Lanes4 } +#[derive(Clone,Copy)] +pub struct F192x4Unreduced { lanes: Wide4 } +impl F192x4 { + #[verifier::type_invariant] + spec fn inv(self) -> bool { lanes4_ok(self.lanes) } + pub closed spec fn elem(self,i: int) -> F192 { lanes4_elem(self.lanes,i) } + pub fn new(v: [F192;4]) -> (r: Self) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] r.elem(i) == v[i], + { Self { lanes: unsafe { lanes4(v) } } } + pub fn splat(v: F192) -> (r: Self) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] r.elem(i) == v, + { Self::new([v;4]) } + pub fn load(v: &[F192;4]) -> (r: Self) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] r.elem(i) == v[i], + { Self { lanes: unsafe { load_lanes4(v) } } } + pub fn store(self,out: &mut [MaybeUninit;4]) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] final(out)[i].mem_contents() == MemContents::Init(self.elem(i)), + { + proof { use_type_invariant(&self); } + unsafe { store_lanes4(self.lanes,out); } + } + pub fn to_array(self) -> (r: [F192;4]) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] r[i] == self.elem(i), + { + let mut out = [MaybeUninit::uninit();4]; + self.store(&mut out); + unsafe { [out[0].assume_init(),out[1].assume_init(),out[2].assume_init(),out[3].assume_init()] } + } + pub fn transpose(rows: [Self;4]) -> (r: [Self;4]) + ensures forall|i: int,j: int| 0 <= i < 4 && 0 <= j < 4 ==> #[trigger] r[i].elem(j) == rows[j].elem(i), + { + let (a,b,c,d) = (rows[0],rows[1],rows[2],rows[3]); + proof { + use_type_invariant(&a); use_type_invariant(&b); + use_type_invariant(&c); use_type_invariant(&d); + } + let lanes = unsafe { transpose_lanes4([rows[0].lanes,rows[1].lanes,rows[2].lanes,rows[3].lanes]) }; + [Self { lanes:lanes[0] },Self { lanes:lanes[1] },Self { lanes:lanes[2] },Self { lanes:lanes[3] }] + } + pub fn mul_unreduced(self,rhs: Self) -> (r: F192x4Unreduced) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] r.value(i) == e_mul(self.elem(i),rhs.elem(i)), + { + proof { use_type_invariant(&self); use_type_invariant(&rhs); } + let r = F192x4Unreduced { lanes: unsafe { mul_lanes4(self.lanes,rhs.lanes) } }; + proof { + assert forall|i: int| 0 <= i < 4 implies #[trigger] r.value(i) == e_mul(self.elem(i),rhs.elem(i)) by { + lemma_prod_unreduced(self.elem(i),rhs.elem(i)); + } + } + r + } +} +impl Add for F192x4 { + type Output = Self; + fn add(self,rhs: Self) -> (r: Self) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] r.elem(i) == e_add(self.elem(i),rhs.elem(i)), + { + proof { use_type_invariant(&self); use_type_invariant(&rhs); } + let lanes = unsafe { xor_lanes(self.lanes,rhs.lanes) }; + proof { lemma_lanes4_xor(self.lanes,rhs.lanes,lanes); } + Self { lanes } + } +} +impl Mul for F192x4 { + type Output = Self; + fn mul(self,rhs: Self) -> (r: Self) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] r.elem(i) == e_mul(self.elem(i),rhs.elem(i)), + { + // The production nested kernel calls, factored through the two verified wrappers. + self.mul_unreduced(rhs).reduce() + } +} +impl F192x4Unreduced { + pub closed spec fn value(self,i: int) -> F192 { e_value(wide4_value(self.lanes,i)) } + pub const fn zero() -> (r: Self) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] r.value(i) == F192::ZERO, + { + proof { lemma_zeroed_wide4(); lemma_u_zero(); } + Self { lanes: unsafe { core::mem::zeroed() } } + } + pub fn sum(self) -> (r: F192Unreduced) + ensures e_value(r) == e_add(e_add(e_add(self.value(0),self.value(1)),self.value(2)),self.value(3)), + { unsafe { sum_lanes4(self.lanes) } } + pub fn reduce(self) -> (r: F192x4) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] r.elem(i) == self.value(i), + { F192x4 { lanes: unsafe { reduce_lanes4(self.lanes) } } } +} +impl BitXorAssign for F192x4Unreduced { + fn bitxor_assign(&mut self,rhs: Self) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] final(self).value(i) == e_add(old(self).value(i),rhs.value(i)), + { + let ghost before = self.lanes; + self.lanes = unsafe { xor_lanes(self.lanes,rhs.lanes) }; + proof { + lemma_wide4_xor(before,rhs.lanes,self.lanes); + assert forall|i: int| 0 <= i < 4 implies #[trigger] self.value(i) == e_add(e_value(wide4_value(before,i)),rhs.value(i)) by { + lemma_e_value_xor(wide4_value(before,i),wide4_value(rhs.lanes,i)); + } + } + } +} +impl BitXor for F192x4Unreduced { + type Output = Self; + fn bitxor(self,rhs: Self) -> (r: Self) + ensures forall|i: int| 0 <= i < 4 ==> #[trigger] r.value(i) == e_add(self.value(i),rhs.value(i)), + { let mut s = self; s ^= rhs; s } +} +// Registers have no canonical bit-level representation in the trait spec. The concrete +// operator contracts above specify lane values, as for F192x1. +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::AddSpecImpl for F192x4 { + open spec fn obeys_add_spec() -> bool { false } + open spec fn add_req(self,rhs: Self) -> bool { true } + open spec fn add_spec(self,rhs: Self) -> Self { self } +} +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::MulSpecImpl for F192x4 { + open spec fn obeys_mul_spec() -> bool { false } + open spec fn mul_req(self,rhs: Self) -> bool { true } + open spec fn mul_spec(self,rhs: Self) -> Self { self } +} +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::BitXorSpecImpl for F192x4Unreduced { + open spec fn obeys_bitxor_spec() -> bool { false } + open spec fn bitxor_req(self,rhs: Self) -> bool { true } + open spec fn bitxor_spec(self,rhs: Self) -> Self { self } +} +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::BitXorAssignSpecImpl for F192x4Unreduced { + open spec fn obeys_bitxor_assign_spec() -> bool { false } + open spec fn bitxor_assign_req(&self,rhs: Self) -> bool { true } + open spec fn bitxor_assign_spec(&self,rhs: Self) -> &Self { self } +} +} // verus! diff --git a/verification/verus/src/gf2_8.rs b/verification/verus/src/gf2_8.rs new file mode 100644 index 0000000000..b8c6fd57ba --- /dev/null +++ b/verification/verus/src/gf2_8.rs @@ -0,0 +1,1240 @@ +//! The field `GF(2^8) = GF(2)[x] / (x^8 + x^4 + x^3 + x + 1)`. +//! +//! The executable functions are the portable paths of `crates/primitives/src/field/gf2_8.rs`, copied +//! with the same bodies, and its SIMD arms (`clmul8_neon`, the NEON `neon` and the AVX2 `avx2` helpers); +//! `tests/equivalence/gf2_8.rs` checks the two agree. +//! +//! Specification: an element is a polynomial over GF(2) of degree below 8, bit `i` its coefficient of +//! `x^i`. The product [`f8_mul`] is the carry-less product ([`clmul`], the one of `crate::clmul`) reduced +//! modulo `M = x^8 + x^4 + x^3 + x + 1`, where "reduced" is the definition of a remainder +//! ([`is_remainder8`]). +use crate::clmul::*; +#[cfg(all(verus_keep_ghost, target_arch = "aarch64"))] +use crate::intrinsics::aarch64_gfneon::*; +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +use core::arch::aarch64::*; +use core::ops::{Add, AddAssign, Mul, MulAssign}; +use vstd::arithmetic::power2::*; +use vstd::prelude::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// The modulus `M = x^8 + x^4 + x^3 + x + 1`. +pub open spec fn modulus8() -> u128 { + 0x11Bu128 +} + +/// `r` is the remainder of `p` modulo `M`: `p = q * M + r` for some polynomial `q`, and `deg r < 8`. +/// +/// A `p` of degree below 16 has a quotient of degree below 8, so `q < 2^8` loses nothing. +pub open spec fn is_remainder8(p: u16, r: u8) -> bool { + exists|q: u64| q < 256 && p as u128 == clmul(q, modulus8()) ^ (r as u128) +} + +/// `p mod M`, the unique remainder (unique by [`lemma_remainder8_unique`]). +pub open spec fn f8_mod(p: u16) -> u8 { + choose|r: u8| is_remainder8(p, r) +} + +/// The product in `GF(2^8)`: `a * b mod M`. The carry-less product of two bytes has degree below 15 +/// ([`lemma_clmul8_closed`]), so the cast to `u16` loses nothing. +pub open spec fn f8_mul(a: u8, b: u8) -> u8 { + f8_mod(clmul(a as u64, b as u128) as u16) +} + +/// `a^n` in `GF(2^8)`. +pub open spec fn f8_pow(a: u8, n: nat) -> u8 + decreases n, +{ + if n == 0 { + 1 + } else { + f8_mul(f8_pow(a, (n - 1) as nat), a) + } +} + +// --------------------------------------------------------------------------------------------- +// The carry-less product of two bytes, in closed form +// --------------------------------------------------------------------------------------------- +/// `[a_i] * b * x^i` at 16 bits. +pub open spec fn term8(a: u8, b: u8, i: u16) -> u16 { + if (a >> i) & 1 == 1 { + (b as u16) << i + } else { + 0 + } +} + +/// The eight partial products of [`clmul`] for bytes, XORed: a bit-vector term. +pub open spec fn clmul8_closed(a: u8, b: u8) -> u16 { + term8(a, b, 0) ^ term8(a, b, 1) ^ term8(a, b, 2) ^ term8(a, b, 3) ^ term8(a, b, 4) ^ term8(a, b, 5) + ^ term8(a, b, 6) ^ term8(a, b, 7) +} + +/// The partial products past bit 7 of a byte are zero. +proof fn lemma_clmul_upto_byte(a: u8, b: u128, n: nat) + requires + 8 <= n <= 64, + ensures + clmul_upto(a as u64, b, n) == clmul_upto(a as u64, b, 8), + decreases n, +{ + if n > 8 { + let i = (n - 1) as nat; + lemma_clmul_upto_byte(a, b, i); + let s = i as u64; + assert(s >= 8 ==> ((a as u64) >> s) & 1 != 1) by (bit_vector); + let x = clmul_upto(a as u64, b, i); + assert(x ^ 0 == x) by (bit_vector); + } +} + +/// [`clmul`] of two bytes is [`clmul8_closed`], of degree below 15. +pub proof fn lemma_clmul8_closed(a: u8, b: u8) + ensures + clmul(a as u64, b as u128) == clmul8_closed(a, b) as u128, + clmul8_closed(a, b) >> 15u16 == 0, +{ + lemma_clmul_upto_byte(a, b as u128, 64); + reveal_with_fuel(clmul_upto, 9); + assert(clmul_upto(a as u64, b as u128, 8) == clmul8_closed(a, b) as u128) by { + assert({ + let (a, w) = (a as u64, b as u128); + let t = |i: u64| + if (a >> i) & 1 == 1 { + w << (i as u128) + } else { + 0 + }; + 0 ^ t(0) ^ t(1) ^ t(2) ^ t(3) ^ t(4) ^ t(5) ^ t(6) ^ t(7) == clmul8_closed(a as u8, b) as u128 + }) by (bit_vector); + } + assert(clmul8_closed(a, b) >> 15u16 == 0) by (bit_vector); +} + +// --------------------------------------------------------------------------------------------- +// The reduction +// --------------------------------------------------------------------------------------------- +/// The closed form of the production `gf8_reduce`, as a spec function. +pub open spec fn reduce8_formula(p: u16) -> u8 { + let h: u16 = p >> 8u16; + let t: u16 = (p & 0xff) ^ h ^ (h << 1u16) ^ (h << 3u16) ^ (h << 4u16); + let h2: u16 = t >> 8u16; + ((t & 0xff) ^ h2 ^ (h2 << 1u16) ^ (h2 << 3u16) ^ (h2 << 4u16)) as u8 +} + +/// `q * M = q x^8 + q * (x^4 + x^3 + x + 1)`. +pub proof fn lemma_clmul_modulus8(q: u64) + requires + q < 256, + ensures + clmul(q, modulus8()) == ((q as u128) << 8u128) ^ crate::gf2_64::times_r64(q), +{ + assert(modulus8() == 0x1Bu128 ^ (1u128 << 8u128)) by (bit_vector); + lemma_clmul_xor_right(q, 0x1Bu128, 1u128 << 8u128); + crate::gf2_64::lemma_clmul_r64(q); + lemma_clmul_monomial(q, 8); + let (a, b) = (crate::gf2_64::times_r64(q), (q as u128) << 8u128); + assert(a ^ b == b ^ a) by (bit_vector); +} + +/// The production reduction returns a remainder of `p` modulo `M`, for every `p < 2^16`: its quotient +/// is `h ^ h2`, the two upper bytes it folds. +pub proof fn lemma_reduce8_formula_is_remainder(p: u16) + ensures + is_remainder8(p, reduce8_formula(p)), +{ + let h: u16 = p >> 8u16; + let t: u16 = (p & 0xff) ^ h ^ (h << 1u16) ^ (h << 3u16) ^ (h << 4u16); + let h2: u16 = t >> 8u16; + let q = (h ^ h2) as u8 as u64; + lemma_clmul_modulus8(q); + assert({ + let h: u16 = p >> 8u16; + let t: u16 = (p & 0xff) ^ h ^ (h << 1u16) ^ (h << 3u16) ^ (h << 4u16); + let h2: u16 = t >> 8u16; + let w = ((h ^ h2) as u8) as u128; + p as u128 == ((w << 8u128) ^ (w ^ (w << 1u128) ^ (w << 3u128) ^ (w << 4u128))) ^ (reduce8_formula( + p, + ) as u128) + }) by (bit_vector); + assert(p as u128 == clmul(q, modulus8()) ^ (reduce8_formula(p) as u128)); +} + +/// The remainder modulo `M` is unique. +pub proof fn lemma_remainder8_unique(p: u16, r1: u8, r2: u8) + requires + is_remainder8(p, r1), + is_remainder8(p, r2), + ensures + r1 == r2, +{ + let q1 = choose|q: u64| q < 256 && p as u128 == clmul(q, modulus8()) ^ (r1 as u128); + let q2 = choose|q: u64| q < 256 && p as u128 == clmul(q, modulus8()) ^ (r2 as u128); + lemma_clmul_modulus8(q1); + lemma_clmul_modulus8(q2); + assert({ + let (w1, w2) = (q1 as u128, q2 as u128); + q1 < 256 && q2 < 256 && ((w1 << 8u128) ^ (w1 ^ (w1 << 1u128) ^ (w1 << 3u128) ^ (w1 << 4u128))) ^ ( + r1 as u128) == ((w2 << 8u128) ^ (w2 ^ (w2 << 1u128) ^ (w2 << 3u128) ^ (w2 << 4u128))) ^ (r2 as u128) + ==> r1 == r2 + }) by (bit_vector); +} + +/// `p mod M` is the production reduction's closed form. +pub proof fn lemma_f8_mod(p: u16) + ensures + f8_mod(p) == reduce8_formula(p), + is_remainder8(p, f8_mod(p)), +{ + lemma_reduce8_formula_is_remainder(p); + lemma_remainder8_unique(p, f8_mod(p), reduce8_formula(p)); +} + +/// The product in closed form: a bit-vector term over the two bytes. +pub open spec fn f8_mul_closed(a: u8, b: u8) -> u8 { + reduce8_formula(clmul8_closed(a, b)) +} + +pub proof fn lemma_f8_mul_closed(a: u8, b: u8) + ensures + f8_mul(a, b) == f8_mul_closed(a, b), +{ + lemma_clmul8_closed(a, b); + assert((clmul8_closed(a, b) as u128) as u16 == clmul8_closed(a, b)) by (bit_vector); + lemma_f8_mod(clmul8_closed(a, b)); +} + +/// Reduction is GF(2)-linear. +pub proof fn lemma_f8_mod_xor(p1: u16, p2: u16) + ensures + f8_mod(p1 ^ p2) == f8_mod(p1) ^ f8_mod(p2), +{ + lemma_f8_mod(p1); + lemma_f8_mod(p2); + lemma_f8_mod(p1 ^ p2); + assert(reduce8_formula(p1 ^ p2) == reduce8_formula(p1) ^ reduce8_formula(p2)) by (bit_vector); +} + +/// A polynomial of degree below 8 is its own remainder. +pub proof fn lemma_f8_mod_small(p: u16) + requires + p >> 8u16 == 0, + ensures + f8_mod(p) == p as u8, +{ + lemma_f8_mod(p); + assert(p >> 8u16 == 0 ==> reduce8_formula(p) == p as u8) by (bit_vector); +} + +/// Multiplication by `x` in `GF(2^8)`: shift, and fold `x^8` back as `x^4 + x^3 + x + 1`. +pub open spec fn mulx8(a: u8) -> u8 { + (a << 1u8) ^ (if a >> 7u8 == 1 { + 0x1Bu8 + } else { + 0u8 + }) +} + +/// `a * x^k` in `GF(2^8)`. +pub open spec fn mulx8_pow(a: u8, k: nat) -> u8 + decreases k, +{ + if k == 0 { + a + } else { + mulx8(mulx8_pow(a, (k - 1) as nat)) + } +} + +/// `(p x) mod M = (p mod M) x mod M`. +pub proof fn lemma_f8_mod_shl1(p: u16) + requires + p >> 15u16 == 0, + ensures + f8_mod(p << 1u16) == mulx8(f8_mod(p)), +{ + lemma_f8_mod(p); + lemma_f8_mod(p << 1u16); + assert(p >> 15u16 == 0 ==> reduce8_formula(p << 1u16) == mulx8(reduce8_formula(p))) by (bit_vector); +} + +/// A multiple of `M` reduces to zero. +pub proof fn lemma_f8_mod_multiple(q: u8) + ensures + f8_mod(clmul(q as u64, modulus8()) as u16) == 0, +{ + lemma_clmul_modulus8(q as u64); + lemma_f8_mod(clmul(q as u64, modulus8()) as u16); + assert({ + let w = (q as u64) as u128; + reduce8_formula((((w << 8u128) ^ (w ^ (w << 1u128) ^ (w << 3u128) ^ (w << 4u128)))) as u16) == 0 + }) by (bit_vector); +} + +// --------------------------------------------------------------------------------------------- +// GF(2^8) is a commutative ring +// --------------------------------------------------------------------------------------------- +pub proof fn lemma_f8_mul_comm(a: u8, b: u8) + ensures + f8_mul(a, b) == f8_mul(b, a), +{ + lemma_f8_mul_closed(a, b); + lemma_f8_mul_closed(b, a); + assert(f8_mul_closed(a, b) == f8_mul_closed(b, a)) by (bit_vector); +} + +pub proof fn lemma_f8_mul_one(a: u8) + ensures + f8_mul(a, 1) == a, + f8_mul(1, a) == a, +{ + lemma_f8_mul_closed(a, 1); + lemma_f8_mul_closed(1, a); + assert(f8_mul_closed(a, 1) == a && f8_mul_closed(1, a) == a) by (bit_vector); +} + +pub proof fn lemma_f8_mul_zero(a: u8) + ensures + f8_mul(a, 0) == 0, + f8_mul(0, a) == 0, +{ + lemma_clmul_zero(a as u64); + assert(0u16 >> 8u16 == 0) by (bit_vector); + lemma_f8_mod_small(0); + lemma_f8_mul_comm(a, 0); +} + +pub proof fn lemma_f8_mul_xor_right(a: u8, b: u8, c: u8) + ensures + f8_mul(a, b ^ c) == f8_mul(a, b) ^ f8_mul(a, c), +{ + assert((b ^ c) as u128 == (b as u128) ^ (c as u128)) by (bit_vector); + lemma_clmul_xor_right(a as u64, b as u128, c as u128); + let (x, y) = (clmul(a as u64, b as u128), clmul(a as u64, c as u128)); + assert((x ^ y) as u16 == (x as u16) ^ (y as u16)) by (bit_vector); + lemma_f8_mod_xor(x as u16, y as u16); +} + +/// `a * (x b) = x (a * b)`. +pub proof fn lemma_f8_mul_mulx(a: u8, b: u8) + ensures + f8_mul(a, mulx8(b)) == mulx8(f8_mul(a, b)), +{ + let m: u128 = if b >> 7u8 == 1 { + modulus8() + } else { + 0 + }; + // b x = mulx(b) + [b_7] M, as polynomials. + assert((b as u128) << 1u128 == (mulx8(b) as u128) ^ (if b >> 7u8 == 1 { + 0x11Bu128 + } else { + 0u128 + })) by (bit_vector); + lemma_clmul_shl1(a as u64, b as u128); + lemma_clmul_xor_right(a as u64, mulx8(b) as u128, m); + lemma_clmul8_closed(a, b); + let p = clmul(a as u64, b as u128); + let c16 = clmul8_closed(a, b); + assert(p == (c16 as u128) && c16 >> 15u16 == 0 ==> (p << 1u128) as u16 == (p as u16) << 1u16 && (p as u16) + >> 15u16 == 0) by (bit_vector); + lemma_f8_mod_shl1(p as u16); + let (x, y) = (clmul(a as u64, mulx8(b) as u128), clmul(a as u64, m)); + assert((x ^ y) as u16 == (x as u16) ^ (y as u16)) by (bit_vector); + lemma_f8_mod_xor(x as u16, y as u16); + if b >> 7u8 == 1 { + lemma_f8_mod_multiple(a); + } else { + lemma_clmul_zero(a as u64); + assert(0u16 >> 8u16 == 0) by (bit_vector); + lemma_f8_mod_small(0); + } + let r = f8_mod(x as u16); + assert(r ^ 0 == r) by (bit_vector); +} + +pub proof fn lemma_f8_mul_mulx_pow(a: u8, b: u8, k: nat) + ensures + f8_mul(a, mulx8_pow(b, k)) == mulx8_pow(f8_mul(a, b), k), + decreases k, +{ + if k > 0 { + lemma_f8_mul_mulx_pow(a, b, (k - 1) as nat); + lemma_f8_mul_mulx(a, mulx8_pow(b, (k - 1) as nat)); + } +} + +/// `x^k` is `1 << k` for `k < 8`. +pub proof fn lemma_mulx8_pow_one(k: nat) + requires + k < 8, + ensures + mulx8_pow(1, k) == 1u8 << (k as u8), + decreases k, +{ + if k == 0 { + assert(1u8 << 0u8 == 1) by (bit_vector); + } else { + lemma_mulx8_pow_one((k - 1) as nat); + let s = (k - 1) as u8; + assert(s < 7 ==> mulx8(1u8 << s) == 1u8 << (s + 1)) by (bit_vector); + } +} + +/// `t * x^k = mulx^k(t)`. +pub proof fn lemma_f8_mul_monomial(t: u8, k: nat) + requires + k < 8, + ensures + f8_mul(t, 1u8 << (k as u8)) == mulx8_pow(t, k), +{ + lemma_mulx8_pow_one(k); + lemma_f8_mul_mulx_pow(t, 1, k); + lemma_f8_mul_one(t); +} + +/// The low `n` coefficients of `c`, for `n <= 8`. +pub open spec fn low8(c: u8, n: u16) -> u8 { + ((c as u16) & sub(1u16 << n, 1)) as u8 +} + +/// Associativity, by linearity in `c` and the monomial case `(a b) x^k = a (b x^k)`. +pub proof fn lemma_f8_mul_assoc_upto(a: u8, b: u8, c: u8, n: u16) + requires + n <= 8, + ensures + f8_mul(f8_mul(a, b), low8(c, n)) == f8_mul(a, f8_mul(b, low8(c, n))), + decreases n, +{ + if n == 0 { + assert(((c as u16) & sub(1u16 << 0u16, 1)) as u8 == 0) by (bit_vector); + lemma_f8_mul_zero(f8_mul(a, b)); + lemma_f8_mul_zero(b); + lemma_f8_mul_zero(a); + } else { + let i = sub(n, 1); + lemma_f8_mul_assoc_upto(a, b, c, i); + let e: u8 = if (c >> (i as u8)) & 1 == 1 { + 1u8 << (i as u8) + } else { + 0 + }; + let lo = low8(c, i); + assert(i < 8 ==> ((c as u16) & sub(1u16 << add(i, 1), 1)) as u8 == ((c as u16) & sub(1u16 << i, 1)) as u8 + ^ (if (c >> (i as u8)) & 1 == 1 { + 1u8 << (i as u8) + } else { + 0u8 + })) by (bit_vector); + assert(low8(c, n) == lo ^ e); + lemma_f8_mul_xor_right(f8_mul(a, b), lo, e); + lemma_f8_mul_xor_right(b, lo, e); + lemma_f8_mul_xor_right(a, f8_mul(b, lo), f8_mul(b, e)); + if (c >> (i as u8)) & 1 == 1 { + lemma_f8_mul_monomial(f8_mul(a, b), i as nat); + lemma_f8_mul_monomial(b, i as nat); + lemma_f8_mul_mulx_pow(a, b, i as nat); + } else { + lemma_f8_mul_zero(f8_mul(a, b)); + lemma_f8_mul_zero(b); + lemma_f8_mul_zero(a); + } + } +} + +pub proof fn lemma_f8_mul_assoc(a: u8, b: u8, c: u8) + ensures + f8_mul(f8_mul(a, b), c) == f8_mul(a, f8_mul(b, c)), +{ + lemma_f8_mul_assoc_upto(a, b, c, 8); + assert(((c as u16) & sub(1u16 << 8u16, 1)) as u8 == c) by (bit_vector); +} + +// --------------------------------------------------------------------------------------------- +// Powers +// --------------------------------------------------------------------------------------------- +pub proof fn lemma_f8_pow_add(a: u8, m: nat, n: nat) + ensures + f8_mul(f8_pow(a, m), f8_pow(a, n)) == f8_pow(a, m + n), + decreases n, +{ + if n == 0 { + lemma_f8_mul_one(f8_pow(a, m)); + } else { + lemma_f8_pow_add(a, m, (n - 1) as nat); + lemma_f8_mul_assoc(f8_pow(a, m), f8_pow(a, (n - 1) as nat), a); + assert((m + n - 1) as nat + 1 == m + n); + } +} + +/// `a^255` by an addition chain over the closed-form product (`3 = 2 + 1`, `15 = 3 * 4 + 3`, +/// `255 = 15 * 16 + 15`): ten products, few enough for the interpreter to evaluate at every `a`. +pub open spec fn f8_pow255_chain(a: u8) -> u8 { + let a3 = f8_mul_closed(f8_mul_closed(a, a), a); + let a6 = f8_mul_closed(a3, a3); + let a15 = f8_mul_closed(f8_mul_closed(a6, a6), a3); + let a30 = f8_mul_closed(a15, a15); + let a60 = f8_mul_closed(a30, a30); + let a120 = f8_mul_closed(a60, a60); + let a240 = f8_mul_closed(a120, a120); + f8_mul_closed(a240, a15) +} + +/// `f8_pow(a, x) * f8_pow(a, y) = f8_pow(a, x + y)`, with the product in closed form. +proof fn lemma_pow_closed_add(a: u8, x: nat, y: nat) + ensures + f8_mul_closed(f8_pow(a, x), f8_pow(a, y)) == f8_pow(a, x + y), +{ + lemma_f8_pow_add(a, x, y); + lemma_f8_mul_closed(f8_pow(a, x), f8_pow(a, y)); +} + +pub proof fn lemma_f8_pow255_chain(a: u8) + ensures + f8_pow255_chain(a) == f8_pow(a, 255), +{ + assert(f8_pow(a, 1) == a) by { + assert(f8_pow(a, 0) == 1); + lemma_f8_mul_one(a); + } + lemma_pow_closed_add(a, 1, 1); + lemma_pow_closed_add(a, 2, 1); + lemma_pow_closed_add(a, 3, 3); + lemma_pow_closed_add(a, 6, 6); + lemma_pow_closed_add(a, 12, 3); + lemma_pow_closed_add(a, 15, 15); + lemma_pow_closed_add(a, 30, 30); + lemma_pow_closed_add(a, 60, 60); + lemma_pow_closed_add(a, 120, 120); + lemma_pow_closed_add(a, 240, 15); +} + +/// `a^255 = 1` for every nonzero `a <= n`. +pub open spec fn fermat_upto(n: nat) -> bool + decreases n, +{ + if n == 0 { + true + } else { + f8_pow255_chain(n as u8) == 1 && fermat_upto((n - 1) as nat) + } +} + +proof fn lemma_fermat_upto(n: nat, a: u8) + requires + fermat_upto(n), + 0 < a <= n, + ensures + f8_pow255_chain(a) == 1, + decreases n, +{ + if a < n { + lemma_fermat_upto((n - 1) as nat, a); + } +} + +/// `a^254` is the inverse of every nonzero `a`, by evaluating `a^255` at all 255 of them. +/// +/// Needs no irreducibility argument: the check is exhaustive. +pub proof fn lemma_f8_inv_correct(a: u8) + requires + a != 0, + ensures + f8_mul(a, f8_pow(a, 254)) == 1, +{ + assert(fermat_upto(255)) by (compute_only); + lemma_fermat_upto(255, a); + lemma_f8_pow255_chain(a); + lemma_f8_mul_comm(a, f8_pow(a, 254)); +} + +// --------------------------------------------------------------------------------------------- +// Executable code: the portable paths of `crates/primitives/src/field/gf2_8.rs` +// --------------------------------------------------------------------------------------------- +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[repr(transparent)] +pub struct F8(pub u8); + +impl F8 { + pub const ZERO: Self = Self(0); + + pub const ONE: Self = Self(1); + + /// Multiplicative inverse via Fermat: x^254 = x^{-1} in F_{2^8}. + /// Exponent bit pattern 0xFE = 0b11111110: 7 squarings + 6 multiplies. + pub fn inv(self) -> (r: Self) + ensures + r.0 == f8_pow(self.0, 254), + self.0 != 0 ==> f8_mul(self.0, r.0) == 1, + { + let ghost a = self.0; + proof { + lemma2_to64(); + assert(f8_pow(a, 1) == a) by { + assert(f8_pow(a, 0) == 1); + lemma_f8_mul_one(a); + } + } + let mut result = Self::ONE; + let mut sq = self; + for i in 0..8 + invariant + sq.0 == f8_pow(a, pow2(i as nat)), + result.0 == f8_pow(a, if i == 0 { 0 } else { (pow2(i as nat) - 2) as nat }), + pow2(0) == 1 && pow2(1) == 2 && pow2(8) == 256, + { + proof { + lemma_pow2_unfold((i + 1) as nat); + lemma_pow2_pos(i as nat); + lemma_f8_pow_add(a, pow2(i as nat), pow2(i as nat)); + if i > 0 { + lemma_f8_pow_add(a, (pow2(i as nat) - 2) as nat, pow2(i as nat)); + if i > 1 { + lemma_pow2_strictly_increases(1, i as nat); + } + } + let s = i as u8; + assert(s < 8 ==> (((0xFEu8 >> s) & 1 != 0) == (s != 0))) by (bit_vector); + } + if (0xFEu8 >> i) & 1 != 0 { + result *= sq; + } + sq *= sq; + } + proof { + if a != 0 { + lemma_f8_inv_correct(a); + } + } + result + } +} + +// In GF(2⁸), addition is bitwise XOR by definition. The `^` is correct, not a +// typo for `+` (which is what these Clippy lints guard against). +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::AddSpecImpl for F8 { + open spec fn obeys_add_spec() -> bool { + true + } + + open spec fn add_req(self, rhs: F8) -> bool { + true + } + + open spec fn add_spec(self, rhs: F8) -> F8 { + F8(self.0 ^ rhs.0) + } +} + +#[allow(clippy::suspicious_arithmetic_impl)] +impl Add for F8 { + type Output = Self; + + #[inline] + fn add(self, rhs: Self) -> (r: Self) + ensures + r.0 == self.0 ^ rhs.0, + { + Self(self.0 ^ rhs.0) + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::AddAssignSpecImpl for F8 { + open spec fn obeys_add_assign_spec() -> bool { + true + } + + open spec fn add_assign_req(&self, rhs: F8) -> bool { + true + } + + open spec fn add_assign_spec(&self, rhs: F8) -> &F8 { + &F8(self.0 ^ rhs.0) + } +} + +#[allow(clippy::suspicious_op_assign_impl)] +impl AddAssign for F8 { + #[inline] + fn add_assign(&mut self, rhs: Self) + ensures + final(self).0 == old(self).0 ^ rhs.0, + { + self.0 ^= rhs.0; + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::MulSpecImpl for F8 { + open spec fn obeys_mul_spec() -> bool { + true + } + + open spec fn mul_req(self, rhs: F8) -> bool { + true + } + + open spec fn mul_spec(self, rhs: F8) -> F8 { + F8(f8_mul(self.0, rhs.0)) + } +} + +impl Mul for F8 { + type Output = Self; + + #[inline] + fn mul(self, rhs: Self) -> (r: Self) + ensures + r.0 == f8_mul(self.0, rhs.0), + { + proof { + lemma_clmul8_closed(self.0, rhs.0); + } + Self(gf8_reduce(clmul8(self.0, rhs.0))) + } +} + +#[cfg(verus_keep_ghost)] +impl vstd::std_specs::ops::MulAssignSpecImpl for F8 { + open spec fn obeys_mul_assign_spec() -> bool { + true + } + + open spec fn mul_assign_req(&self, rhs: F8) -> bool { + true + } + + open spec fn mul_assign_spec(&self, rhs: F8) -> &F8 { + &F8(f8_mul(self.0, rhs.0)) + } +} + +impl MulAssign for F8 { + #[inline] + fn mul_assign(&mut self, rhs: Self) + ensures + final(self).0 == f8_mul(old(self).0, rhs.0), + { + *self = *self * rhs; + } +} + +/// Carry-less product of two bytes; result fits in 15 bits. +/// +/// Production's `cfg_attr(.., expect(clippy::missing_const_for_fn))` is dropped: the copy allows `clippy::all`. +#[inline] +pub fn clmul8(a: u8, b: u8) -> (r: u16) + ensures + r as u128 == clmul(a as u64, b as u128), +{ + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + { + // SAFETY: `aes` target feature is enabled at compile time. + unsafe { clmul8_neon(a, b) } + } + #[cfg(not(all(target_arch = "aarch64", target_feature = "aes")))] + { + clmul8_software(a, b) + } +} + +/// PMULL on two broadcast bytes: lane 0 of the eight byte products is `a * b`. +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +#[target_feature(enable = "aes")] +#[inline] +pub unsafe fn clmul8_neon(a: u8, b: u8) -> (r: u16) + ensures + r as u128 == clmul(a as u64, b as u128), +{ + let va = vdup_n_p8(a); + let vb = vdup_n_p8(b); + let prod = vmull_p8(va, vb); + proof { + lemma_clmul8_closed(a, b); + let c = clmul(a as u64, b as u128); + let x = clmul8_closed(a, b); + assert(c == x as u128 ==> (c as u16) as u128 == c) by (bit_vector); + assert(p16x8(prod)[0] == vmull_p8_lane(p8x8(va)[0], p8x8(vb)[0])); + } + vgetq_lane_u16::<0>(vreinterpretq_u16_p16(prod)) +} + +/// Software fallback / test oracle. Used when `aes` is off, and as the +/// cross-check oracle inside the `software_matches_neon` unit test. +#[allow(dead_code)] +#[inline] +pub const fn clmul8_software(a: u8, b: u8) -> (r: u16) + ensures + r as u128 == clmul(a as u64, b as u128), +{ + let b16 = b as u16; + let mut acc: u16 = 0; + let mut i = 0; + while i < 8 + invariant + 0 <= i <= 8, + b16 == b as u16, + acc as u128 == clmul_upto(a as u64, b as u128, i as nat), + decreases 8 - i, + { + proof { + let s = i as u16; + assert(s < 8 ==> ((a >> s) & 1 != 0) == (((a as u64) >> (s as u64)) & 1 == 1)) by (bit_vector); + assert(s < 8 ==> (acc ^ ((b as u16) << s)) as u128 == (acc as u128) ^ ((b as u128) << (s as u128))) + by (bit_vector); + assert((acc as u128) ^ 0 == acc as u128) by (bit_vector); + assert(term(a as u64, b as u128, i as nat) == if ((a as u64) >> (s as u64)) & 1 == 1 { + (b as u128) << (s as u128) + } else { + 0 + }); + } + if (a >> i) & 1 != 0 { + acc ^= b16 << i; + } + i += 1; + } + proof { + lemma_clmul_upto_byte(a, b as u128, 64); + } + acc +} + +/// Reduce a polynomial of degree ≤ 14 modulo x^8 + x^4 + x^3 + x + 1. +/// Two-step fold: first turns 15-bit input into ≤12-bit, second into ≤8-bit. +/// +/// Exposed `pub` so flock's URM shift_reduce inner kernel can reuse it. +/// +/// Proven for every `p: u16`, degree 15 included: the result is the remainder of `p` modulo `M`. +#[inline] +pub const fn gf8_reduce(p: u16) -> (r: u8) + ensures + r == f8_mod(p), + is_remainder8(p, r), +{ + proof { + lemma_f8_mod(p); + } + let h: u16 = p >> 8; + let t: u16 = (p & 0xff) ^ h ^ (h << 1) ^ (h << 3) ^ (h << 4); + let h2: u16 = t >> 8; + ((t & 0xff) ^ h2 ^ (h2 << 1) ^ (h2 << 3) ^ (h2 << 4)) as u8 +} + +// aarch64 NEON helpers: 16-lane GF(2^8) mul and reduce. +// +// These are the building blocks for the round-1 URM shift_reduce inner kernel. +// +// `vmull_p8` is a baseline NEON instruction (no aes feature needed), so the +// only cfg gate is `target_arch = "aarch64"`. +#[cfg(target_arch = "aarch64")] +pub mod neon { + use core::arch::aarch64::*; + use core::mem::transmute; + #[cfg(verus_keep_ghost)] + use super::{clmul8_closed, f8_mod, f8_mul, lemma_clmul8_closed, lemma_f8_mod, reduce8_formula}; + #[cfg(verus_keep_ghost)] + use crate::intrinsics::aarch64_bits::u8x16; + #[cfg(verus_keep_ghost)] + use crate::intrinsics::aarch64_gfneon::*; + #[cfg(verus_keep_ghost)] + use crate::intrinsics::transmuted; + use vstd::prelude::*; + + /// Polynomial lane `i < 16` of the interleaved layout `(c0, c1)`: the low byte at `2 (i % 8)` and the high + /// byte at `2 (i % 8) + 1`, of `c0` for `i < 8` and of `c1` after. + pub open spec fn poly_lane(c0: uint8x16_t, c1: uint8x16_t, i: int) -> u16 { + let c = if i < 8 { + u8x16(c0) + } else { + u8x16(c1) + }; + let j = i % 8; + (c[2 * j] as u16) | ((c[2 * j + 1] as u16) << 8u16) + } + + /// One lane of [`gf8_reduce_vec16`]: the quotient `q` is the high byte of `hi * 0x8d * x` (Barrett, with + /// `x^16 / M = x^8 + x^4 + x^3 + x`), and the remainder is `lo + (q * 0x1b mod x^8)`. + pub open spec fn reduce_vec16_lane(lo: u8, hi: u8) -> u8 { + let q = ((vmull_p8_lane(hi, 0x8d) << 1u16) >> 8u16) as u8; + lo ^ (vmull_p8_lane(q, 0x1b) as u8) + } + + /// The Barrett quotient is exact for every high byte: the lane is the remainder of `lo + hi x^8`. + pub proof fn lemma_reduce_vec16_lane(lo: u8, hi: u8) + ensures + reduce_vec16_lane(lo, hi) == f8_mod((lo as u16) | ((hi as u16) << 8u16)), + { + lemma_clmul8_closed(hi, 0x8d); + let t = clmul8_closed(hi, 0x8d); + assert(crate::clmul::clmul(hi as u64, 0x8du8 as u128) == t as u128 ==> (crate::clmul::clmul( + hi as u64, + 0x8du8 as u128, + ) as u16) == t) by (bit_vector); + let q = ((t << 1u16) >> 8u16) as u8; + lemma_clmul8_closed(q, 0x1b); + let u = clmul8_closed(q, 0x1b); + assert(crate::clmul::clmul(q as u64, 0x1bu8 as u128) == u as u128 ==> (crate::clmul::clmul( + q as u64, + 0x1bu8 as u128, + ) as u16) == u) by (bit_vector); + let p = (lo as u16) | ((hi as u16) << 8u16); + lemma_f8_mod(p); + assert(reduce8_formula((lo as u16) | ((hi as u16) << 8u16)) == lo ^ (clmul8_closed( + ((clmul8_closed(hi, 0x8d) << 1u16) >> 8u16) as u8, + 0x1b, + ) as u8)) by (bit_vector); + } + + /// Byte `k` of 16-bit lanes: the low byte of lane `k / 2` at even `k`, its high byte at odd `k`. + proof fn lemma_u16_bytes() + ensures + forall|w: Seq, k: int| + 0 <= k < 2 * w.len() ==> #[trigger] u16_byte(w, k) == if k % 2 == 0 { + w[k / 2] as u8 + } else { + (w[k / 2] >> 8u16) as u8 + }, + { + assert forall|w: Seq, k: int| 0 <= k < 2 * w.len() implies #[trigger] u16_byte(w, k) == if k % 2 == 0 { + w[k / 2] as u8 + } else { + (w[k / 2] >> 8u16) as u8 + } by { + let x = w[k / 2]; + assert((x >> 0u16) as u8 == x as u8) by (bit_vector); + } + } + + /// The 16 lanes of a broadcast byte constant. + proof fn lemma_splat(x: u64, c: u8) + requires + x == 0x8d8d8d8d8d8d8d8du64 && c == 0x8d || x == 0x1b1b1b1b1b1b1b1bu64 && c == 0x1b, + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] p8x8(transmuted::(x))[i] == c, + { + assert forall|i: int| 0 <= i < 8 implies #[trigger] p8x8(transmuted::(x))[i] == c by { + axiom_u64_as_p8x8(x, i); + let s = (8 * i) as u64; + assert(s == 0 || s == 8 || s == 16 || s == 24 || s == 32 || s == 40 || s == 48 || s == 56); + assert((s == 0 || s == 8 || s == 16 || s == 24 || s == 32 || s == 40 || s == 48 || s == 56) ==> ( + 0x8d8d8d8d8d8d8d8du64 >> s) as u8 == 0x8d && (0x1b1b1b1b1b1b1b1bu64 >> s) as u8 == 0x1b) + by (bit_vector); + } + } + + /// Reduce 16 polynomial products (in interleaved layout `[lo0,hi0, lo1,hi1, ...]`, + /// passed as `(c0, c1)`) modulo `x^8 + x^4 + x^3 + x + 1`, returning 16 reduced + /// GF(2^8) values. + /// + /// Two-stage reduction: + /// Stage 1: ch · QPLUS_RSH1 then ·2 (corrects for /x in QPLUS_RSH1) + /// Stage 2: high bytes of stage-1 · QSTAR; take low bytes only. + /// + /// Constants: + /// QPLUS_RSH1 = (x^8+x^4+x^3+x)/x = 0x8d + /// QSTAR = x^4+x^3+x+1 = 0x1b + /// + /// Proven for every 16-bit lane, not only products of two bytes: lane `i` of the result is the remainder + /// of polynomial lane `i` ([`poly_lane`]) modulo `M`. + /// + /// # Safety + /// Uses `core::arch::aarch64` NEON intrinsics; only call on `aarch64`. + #[inline] + pub unsafe fn gf8_reduce_vec16(c0: uint8x16_t, c1: uint8x16_t) -> (r: uint8x16_t) + ensures + forall|i: int| 0 <= i < 16 ==> #[trigger] u8x16(r)[i] == f8_mod(poly_lane(c0, c1, i)), + { + // SAFETY: NEON, PMULL's `vmull_p8` included, is part of the aarch64 baseline, and nothing here touches + // memory; the transmutes are between same-size vector and integer types. + unsafe { + let q_plus_rsh1: poly8x8_t = transmute::(0x8d8d8d8d8d8d8d8d_u64); + let q_star: poly8x8_t = transmute::(0x1b1b1b1b1b1b1b1b_u64); + + let cl = vuzp1q_u8(c0, c1); // low bytes of all 16 products + let ch = vuzp2q_u8(c0, c1); // high bytes of all 16 products + + // Stage 1. + let t0 = vreinterpretq_u8_u16(vshlq_n_u16::<1>(vreinterpretq_u16_p16(vmull_p8( + transmute::(vget_low_u8(ch)), + q_plus_rsh1, + )))); + let t1 = vreinterpretq_u8_u16(vshlq_n_u16::<1>(vreinterpretq_u16_p16(vmull_p8( + transmute::(vget_high_u8(ch)), + q_plus_rsh1, + )))); + + // Stage 2. + let tmp_hi = vuzp2q_u8(t0, t1); + let r0 = vreinterpretq_u8_u16(vreinterpretq_u16_p16(vmull_p8( + transmute::(vget_low_u8(tmp_hi)), + q_star, + ))); + let r1 = vreinterpretq_u8_u16(vreinterpretq_u16_p16(vmull_p8( + transmute::(vget_high_u8(tmp_hi)), + q_star, + ))); + + proof { + broadcast use axiom_u8x8_as_p8x8; + lemma_splat(0x8d8d8d8d8d8d8d8d_u64, 0x8d); + lemma_splat(0x1b1b1b1b1b1b1b1b_u64, 0x1b); + lemma_u16_bytes(); + // The low and high bytes of each polynomial lane. + assert forall|k: int| 0 <= k < 16 implies { + let c = if k < 8 { + u8x16(c0) + } else { + u8x16(c1) + }; + &&& #[trigger] u8x16(cl)[k] == c[2 * (k % 8)] + &&& u8x16(ch)[k] == c[2 * (k % 8) + 1] + } by {} + // Stage 1: the quotient is the high byte of each shifted product. + assert forall|k: int| 0 <= k < 16 implies #[trigger] u8x16(tmp_hi)[k] == ((vmull_p8_lane( + u8x16(ch)[k], + 0x8d, + ) << 1u16) >> 8u16) as u8 by { + if k < 8 { + assert(u8x16(tmp_hi)[k] == u8x16(t0)[2 * k + 1]); + } else { + assert(u8x16(tmp_hi)[k] == u8x16(t1)[2 * (k - 8) + 1]); + } + } + // Stage 2: the low byte of each quotient times `0x1b`. + assert forall|k: int| 0 <= k < 16 implies #[trigger] uzp_u8_lane(u8x16(r0)@, u8x16(r1)@, 0, k) == ( + vmull_p8_lane(u8x16(tmp_hi)[k], 0x1b) as u8) by { + if k < 8 { + assert(u8x16(r0)[2 * k] == vmull_p8_lane(u8x16(tmp_hi)[k], 0x1b) as u8); + } else { + assert(u8x16(r1)[2 * (k - 8)] == vmull_p8_lane(u8x16(tmp_hi)[k], 0x1b) as u8); + } + } + assert forall|k: int| 0 <= k < 16 implies u8x16(cl)[k] ^ (vmull_p8_lane(u8x16(tmp_hi)[k], 0x1b) as u8) == f8_mod(#[trigger] poly_lane(c0, c1, k)) by { + lemma_reduce_vec16_lane(u8x16(cl)[k], u8x16(ch)[k]); + } + } + veorq_u8(cl, vuzp1q_u8(r0, r1)) + } + } + + /// Element-wise multiply 16 pairs of GF(2^8) values (binius64 13-op NEON kernel). + /// + /// # Safety + /// Uses `core::arch::aarch64` NEON intrinsics (PMULL); only call on `aarch64`. + #[inline] + pub unsafe fn gf8_mul_vec16(a: uint8x16_t, b: uint8x16_t) -> (r: uint8x16_t) + ensures + forall|i: int| 0 <= i < 16 ==> #[trigger] u8x16(r)[i] == f8_mul(u8x16(a)[i], u8x16(b)[i]), + { + // SAFETY: as in the reduction above: baseline NEON on registers only. + unsafe { + let c0 = vreinterpretq_u8_u16(vreinterpretq_u16_p16(vmull_p8( + transmute::(vget_low_u8(a)), + transmute::(vget_low_u8(b)), + ))); + let c1 = vreinterpretq_u8_u16(vreinterpretq_u16_p16(vmull_p8( + transmute::(vget_high_u8(a)), + transmute::(vget_high_u8(b)), + ))); + proof { + broadcast use axiom_u8x8_as_p8x8; + lemma_u16_bytes(); + assert forall|i: int| 0 <= i < 16 implies #[trigger] poly_lane(c0, c1, i) == vmull_p8_lane( + u8x16(a)[i], + u8x16(b)[i], + ) by { + let c = if i < 8 { + c0 + } else { + c1 + }; + let j = i % 8; + let x = vmull_p8_lane(u8x16(a)[i], u8x16(b)[i]); + assert(u8x16(c)[2 * j] == x as u8 && u8x16(c)[2 * j + 1] == (x >> 8u16) as u8); + assert(((x as u8) as u16) | ((((x >> 8u16) as u8) as u16) << 8u16) == x) by (bit_vector); + } + } + gf8_reduce_vec16(c0, c1) + } + } +} + +/// The AVX2 counterpart of the NEON helpers, for x86 without GFNI (whose `gf2p8mulb` is the field product itself). +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +pub mod avx2 { + use core::arch::x86_64::*; + #[cfg(verus_keep_ghost)] + use super::{f8_mul, lemma_f8_mul_mulx, lemma_f8_mul_one, lemma_f8_mul_xor_right, lemma_f8_mul_zero, mulx8}; + #[cfg(verus_keep_ghost)] + use crate::intrinsics::x86::{m256, m256_bytes}; + #[cfg(verus_keep_ghost)] + use crate::intrinsics::x86_gfneon::*; + use vstd::prelude::*; + + /// The top `s` bits of `b`, as a polynomial of degree below `s`. + pub open spec fn top_bits(b: u8, s: u8) -> u8 { + b >> sub(8u8, s) + } + + /// One Horner step on a byte lane: from `r = a * top_bits(b, s)`, doubling `r` and adding `a` where the top + /// bit of `b << s` is set gives `a * top_bits(b, s + 1)`. + proof fn lemma_horner_step(a: u8, b: u8, s: u8, r: u8) + requires + 1 <= s < 8, + r == f8_mul(a, top_bits(b, s)), + ensures + mulx8(r) ^ (if (b << s) >> 7u8 == 1 { + a + } else { + 0u8 + }) == f8_mul(a, top_bits(b, add(s, 1))), + { + let c = top_bits(b, s); + let bit: u8 = if (b << s) >> 7u8 == 1 { + 1 + } else { + 0 + }; + assert(1 <= s < 8 ==> b >> sub(8u8, add(s, 1)) == mulx8(b >> sub(8u8, s)) ^ (if (b << s) >> 7u8 == 1 { + 1u8 + } else { + 0u8 + })) by (bit_vector); + lemma_f8_mul_xor_right(a, mulx8(c), bit); + lemma_f8_mul_mulx(a, c); + lemma_f8_mul_one(a); + lemma_f8_mul_zero(a); + let x = f8_mul(a, c); + assert(x ^ 0 == x) by (bit_vector); + } + + /// The byte operations of one step: `x + x` with `0x1B` where `x`'s top bit is set is [`mulx8`], and `a` + /// masked by `y`'s spread top bit is `a` or zero. + proof fn lemma_byte_ops(x: u8, y: u8, a: u8) + ensures + ((x + x) as u8) ^ (0x1Bu8 & cmpgt_epi8_lane(0, x)) == mulx8(x), + a & cmpgt_epi8_lane(0, y) == (if y >> 7u8 == 1 { + a + } else { + 0u8 + }), + ((y + y) as u8) == y << 1u8, + { + lemma_cmpgt_zero(x); + lemma_cmpgt_zero(y); + assert(((x + x) as u8) == x << 1u8) by (bit_vector); + assert(((x << 1u8) ^ (0x1Bu8 & (if x >= 128 { + 0xFFu8 + } else { + 0u8 + }))) == mulx8(x)) by (bit_vector); + assert(a & (if y >= 128 { + 0xFFu8 + } else { + 0u8 + }) == (if y >> 7u8 == 1 { + a + } else { + 0u8 + })) by (bit_vector); + assert(((y + y) as u8) == y << 1u8) by (bit_vector); + } + + /// Element-wise product of 32 pairs of GF(2^8) values, a bit of `b` at a time from the top (Horner). + /// + /// Each step doubles the running product (`xtime`, folding `x^8` back as `0x1B`) and adds `a` where the bit is set. + /// + /// Rewritten for Verus: the loop counter `_` is named `i`, which the invariant needs, and the nested intrinsic + /// calls are bound to names (`tb`, `rr`, `tr`, `pr`, `ab`), so the proof can read each register bytewise. + /// + /// # Safety + /// Requires the `avx2` target feature. + #[inline] + #[target_feature(enable = "avx2")] + pub unsafe fn gf8_mul_vec32(a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == f8_mul(m256_bytes(a)[k], m256_bytes(b)[k]), + { + let ghost b0 = b; + let zero = _mm256_setzero_si256(); + let poly = _mm256_set1_epi8(0x1B); + proof { + lemma_m256_zero_bytes(zero); + } + // A byte's top bit is its sign, so a signed compare against zero spreads it over the byte. + let top = |x: __m256i| -> (t: __m256i) + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(t)[k] == cmpgt_epi8_lane(m256_bytes(zero)[k], m256_bytes(x)[k]), + { _mm256_cmpgt_epi8(zero, x) }; + let tb = top(b); + let (mut r, mut b) = (_mm256_and_si256(a, tb), _mm256_add_epi8(b, b)); + proof { + lemma_m256_and_bytes(a, tb, r); + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(r)[k] == f8_mul(m256_bytes(a)[k], top_bits(m256_bytes(b0)[k], 1u8)) by { + let (av, bv) = (m256_bytes(a)[k], m256_bytes(b0)[k]); + lemma_byte_ops(av, bv, av); + assert(top_bits(bv, 1u8) == (if bv >> 7u8 == 1 { 1u8 } else { 0u8 })) by (bit_vector); + lemma_f8_mul_one(av); + lemma_f8_mul_zero(av); + } + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(b)[k] == m256_bytes(b0)[k] << 1u8 by { + lemma_byte_ops(0, m256_bytes(b0)[k], 0); + } + } + for i in 1..8 + invariant + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(zero)[k] == 0, + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(poly)[k] == 0x1B, + forall|x: __m256i| #[trigger] top.requires((x,)), + forall|x: __m256i, t: __m256i| #[trigger] top.ensures((x,), t) ==> forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(t)[k] == cmpgt_epi8_lane(m256_bytes(zero)[k], m256_bytes(x)[k]), + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == f8_mul(m256_bytes(a)[k], top_bits(m256_bytes(b0)[k], i as u8)), + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(b)[k] == m256_bytes(b0)[k] << (i as u8), + { + let ghost (r_in, b_in) = (r, b); + let rr = _mm256_add_epi8(r, r); + let tr = top(r); + let pr = _mm256_and_si256(poly, tr); + let doubled = _mm256_xor_si256(rr, pr); + let tb = top(b); + let ab = _mm256_and_si256(a, tb); + r = _mm256_xor_si256(doubled, ab); + b = _mm256_add_epi8(b, b); + proof { + lemma_m256_and_bytes(poly, tr, pr); + lemma_m256_xor_bytes(rr, pr, doubled); + lemma_m256_and_bytes(a, tb, ab); + lemma_m256_xor_bytes(doubled, ab, r); + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(r)[k] == f8_mul(m256_bytes(a)[k], top_bits(m256_bytes(b0)[k], (i + 1) as u8)) by { + let (x, y, av, bv) = (m256_bytes(r_in)[k], m256_bytes(b_in)[k], m256_bytes(a)[k], m256_bytes(b0)[k]); + let s = i as u8; + lemma_byte_ops(x, y, av); + lemma_horner_step(av, bv, s, x); + assert(add(s, 1) == (i + 1) as u8); + } + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(b)[k] == m256_bytes(b0)[k] << ((i + 1) as u8) by { + let (y, bv) = (m256_bytes(b_in)[k], m256_bytes(b0)[k]); + let s = i as u8; + lemma_byte_ops(0, y, 0); + assert(add(s, 1) == (i + 1) as u8); + assert(1 <= s < 8 ==> (bv << s) << 1u8 == bv << add(s, 1)) by (bit_vector); + } + } + } + proof { + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(r)[k] == f8_mul(m256_bytes(a)[k], m256_bytes(b0)[k]) by { + let bv = m256_bytes(b0)[k]; + assert(top_bits(bv, 8u8) == bv) by (bit_vector); + } + } + r + } +} + +} // verus! diff --git a/verification/verus/src/intrinsics/aarch64.rs b/verification/verus/src/intrinsics/aarch64.rs new file mode 100644 index 0000000000..1577036fac --- /dev/null +++ b/verification/verus/src/intrinsics/aarch64.rs @@ -0,0 +1,100 @@ +//! Specifications of the AArch64 intrinsics (NEON, PMULL, SHA3's EOR3). +//! +//! A register is viewed as the array of its lanes, lane 0 first ([`u64x2`], [`p64x2`]). Each intrinsic's +//! result is given lane by lane; where the semantics is more than a lane-wise XOR or a move, an executable +//! twin `model_` computes the same lanes for the differential tests. +#[cfg(verus_keep_ghost)] +use super::transmuted; +#[cfg(verus_keep_ghost)] +use crate::clmul::clmul; +use core::arch::aarch64::*; +use vstd::prelude::*; + +verus! { + +#[verifier::external_type_specification] +#[verifier::external_body] +pub struct ExUint64x2(uint64x2_t); + +#[verifier::external_type_specification] +#[verifier::external_body] +pub struct ExPoly64x2(poly64x2_t); + +// --------------------------------------------------------------------------------------------- +// Views and layout +// --------------------------------------------------------------------------------------------- +/// The two lanes of a `uint64x2_t`, lane 0 first. +pub open spec fn u64x2(v: uint64x2_t) -> [u64; 2] { + transmuted::(v) +} + +/// The two lanes of a `poly64x2_t`, lane 0 first. +pub open spec fn p64x2(v: poly64x2_t) -> [u64; 2] { + transmuted::(v) +} + +/// Layout: a `u128` read as a `uint64x2_t` is its low word in lane 0 and its high word in lane 1. +pub broadcast axiom fn axiom_u128_as_u64x2(x: u128) + ensures + #[trigger] u64x2(transmuted::(x)) == [x as u64, (x >> 64u128) as u64], +; + +/// Layout: a `uint64x2_t` read as a `u128` is its two lanes, little-endian. +pub broadcast axiom fn axiom_u64x2_as_u128(v: uint64x2_t) + ensures + #[trigger] transmuted::(v) == (u64x2(v)[0] as u128) | ((u64x2(v)[1] as u128) << 64u128), +; + +/// Layout: a `uint64x2_t` read as a `poly64x2_t` has the same lanes. +pub broadcast axiom fn axiom_u64x2_as_p64x2(v: uint64x2_t) + ensures + #[trigger] p64x2(transmuted::(v)) == u64x2(v), +; + +// --------------------------------------------------------------------------------------------- +// The specifications +// --------------------------------------------------------------------------------------------- +/// PMULL: the carry-less product of two 64-bit polynomials. +pub assume_specification[ vmull_p64 ](a: u64, b: u64) -> (r: u128) + ensures + r == clmul(a, b as u128), +; + +/// PMULL2: the carry-less product of the high lanes. +pub assume_specification[ vmull_high_p64 ](a: poly64x2_t, b: poly64x2_t) -> (r: u128) + ensures + r == clmul(p64x2(a)[1], p64x2(b)[1] as u128), +; + +pub assume_specification[ vdupq_n_u64 ](value: u64) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == value, + u64x2(r)[1] == value, +; + +pub assume_specification[ veorq_u64 ](a: uint64x2_t, b: uint64x2_t) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == u64x2(a)[0] ^ u64x2(b)[0], + u64x2(r)[1] == u64x2(a)[1] ^ u64x2(b)[1], +; + +/// EOR3: the three-way XOR. +pub assume_specification[ veor3q_u64 ](a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == u64x2(a)[0] ^ u64x2(b)[0] ^ u64x2(c)[0], + u64x2(r)[1] == u64x2(a)[1] ^ u64x2(b)[1] ^ u64x2(c)[1], +; + +/// ZIP1: the low lanes of `a` and `b`. +pub assume_specification[ vzip1q_u64 ](a: uint64x2_t, b: uint64x2_t) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == u64x2(a)[0], + u64x2(r)[1] == u64x2(b)[0], +; + +pub assume_specification[ vgetq_lane_u64:: ](v: uint64x2_t) -> (r: u64) + ensures + 0 <= IMM5 < 2 ==> r == u64x2(v)[IMM5 as int], +; + +} // verus! diff --git a/verification/verus/src/intrinsics/aarch64_bits.rs b/verification/verus/src/intrinsics/aarch64_bits.rs new file mode 100644 index 0000000000..7d471e0057 --- /dev/null +++ b/verification/verus/src/intrinsics/aarch64_bits.rs @@ -0,0 +1,214 @@ +//! Specifications of the AArch64 intrinsics of the bit transposes (the four-register table lookup, the 64-bit +//! lane shifts and AND, the reinterprets between byte and word lanes), and the byte loads and stores of their +//! kernel. +//! +//! A `uint8x16_t` is viewed as its 16 bytes, lane 0 first ([`u8x16`]), a `uint64x2_t` as its two words +//! ([`super::aarch64::u64x2`]). `tests/equivalence/intrinsics_aarch64_bits.rs` runs each against the hardware. +#[cfg(verus_keep_ghost)] +use super::aarch64::u64x2; +#[cfg(verus_keep_ghost)] +use super::transmuted; +use core::arch::aarch64::*; +use vstd::prelude::*; + +verus! { + +#[verifier::external_type_specification] +#[verifier::external_body] +pub struct ExUint8x16(uint8x16_t); + +#[verifier::external_type_specification] +pub struct ExUint8x16x4(uint8x16x4_t); + +// --------------------------------------------------------------------------------------------- +// Views +// --------------------------------------------------------------------------------------------- +/// The 16 bytes of a `uint8x16_t`, lane 0 first. +pub open spec fn u8x16(v: uint8x16_t) -> [u8; 16] { + transmuted::(v) +} + +/// Byte `k` of a sequence of 64-bit lanes, little-endian. +pub open spec fn u64x2_byte(w: Seq, k: int) -> u8 { + (w[k / 8] >> ((8 * (k % 8)) as u64)) as u8 +} + +// --------------------------------------------------------------------------------------------- +// Lane semantics +// --------------------------------------------------------------------------------------------- +/// Byte `i` of the 64-byte table of `vqtbl4q_u8`, the four registers in order; zero from 64 on. +pub open spec fn tbl4_byte(t0: Seq, t1: Seq, t2: Seq, t3: Seq, i: int) -> u8 { + if i < 16 { + t0[i] + } else if i < 32 { + t1[i - 16] + } else if i < 48 { + t2[i - 32] + } else if i < 64 { + t3[i - 48] + } else { + 0 + } +} + +/// A logical right shift of a 64-bit lane by `vshrq_n_u64`'s immediate (1 to 64): zero at 64. +pub open spec fn shr_n(a: u64, n: int) -> u64 { + if 0 <= n < 64 { + a >> (n as u64) + } else { + 0 + } +} + +/// A left shift of a 64-bit lane by `vshlq_n_u64`'s immediate (0 to 63). +pub open spec fn shl_n(a: u64, n: int) -> u64 { + if 0 <= n < 64 { + a << (n as u64) + } else { + 0 + } +} + +// --------------------------------------------------------------------------------------------- +// The specifications +// --------------------------------------------------------------------------------------------- +/// TBL with four table registers: byte `k` is byte `b[k]` of the 64-byte table, or zero if `b[k] >= 64`. +pub assume_specification[ vqtbl4q_u8 ](a: uint8x16x4_t, b: uint8x16_t) -> (r: uint8x16_t) + ensures + forall|k: int| + 0 <= k < 16 ==> #[trigger] u8x16(r)[k] == tbl4_byte( + u8x16(a.0)@, + u8x16(a.1)@, + u8x16(a.2)@, + u8x16(a.3)@, + u8x16(b)[k] as int, + ), +; + +/// The 16 bytes read as two little-endian words. +pub assume_specification[ vreinterpretq_u64_u8 ](a: uint8x16_t) -> (r: uint64x2_t) + ensures + forall|k: int| 0 <= k < 16 ==> #[trigger] u64x2_byte(u64x2(r)@, k) == u8x16(a)[k], +; + +/// The two words read as 16 bytes, little-endian. +pub assume_specification[ vreinterpretq_u8_u64 ](a: uint64x2_t) -> (r: uint8x16_t) + ensures + forall|k: int| 0 <= k < 16 ==> #[trigger] u8x16(r)[k] == u64x2_byte(u64x2(a)@, k), +; + +pub assume_specification[ vandq_u64 ](a: uint64x2_t, b: uint64x2_t) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == u64x2(a)[0] & u64x2(b)[0], + u64x2(r)[1] == u64x2(a)[1] & u64x2(b)[1], +; + +pub assume_specification[ vshrq_n_u64:: ](a: uint64x2_t) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == shr_n(u64x2(a)[0], N as int), + u64x2(r)[1] == shr_n(u64x2(a)[1], N as int), +; + +pub assume_specification[ vshlq_n_u64:: ](a: uint64x2_t) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == shl_n(u64x2(a)[0], N as int), + u64x2(r)[1] == shl_n(u64x2(a)[1], N as int), +; + +// --------------------------------------------------------------------------------------------- +// Loads and stores of byte arrays +// --------------------------------------------------------------------------------------------- +/// `vld1q_u8` of a 16-byte array. +#[verifier::external_body] +#[inline(always)] +pub fn vld1q_u8_16(x: &[u8; 16]) -> (r: uint8x16_t) + ensures + u8x16(r) == *x, +{ + // SAFETY: the load reads the 16 bytes of `x`. + unsafe { vld1q_u8(x.as_ptr()) } +} + +/// `vld1q_u8` of the 16 bytes of a 64-byte array from `offset` on. +#[verifier::external_body] +#[inline(always)] +pub fn vld1q_u8_at(x: &[u8; 64], offset: usize) -> (r: uint8x16_t) + requires + offset <= 48, + ensures + forall|k: int| 0 <= k < 16 ==> #[trigger] u8x16(r)[k] == x[offset + k], +{ + // SAFETY: `offset + 16 <= 64`: the load reads within `x`. + unsafe { vld1q_u8(x.as_ptr().add(offset)) } +} + +/// `vst1q_u8` into the 16 bytes of a 64-byte array from `offset` on. +#[verifier::external_body] +#[inline(always)] +pub fn vst1q_u8_at(x: &mut [u8; 64], offset: usize, v: uint8x16_t) + requires + offset <= 48, + ensures + forall|k: int| + 0 <= k < 64 ==> #[trigger] final(x)[k] == if offset <= k < offset + 16 { + u8x16(v)[k - offset] + } else { + old(x)[k] + }, +{ + // SAFETY: `offset + 16 <= 64`: the store writes within `x`. + unsafe { vst1q_u8(x.as_mut_ptr().add(offset), v) } +} + +// --------------------------------------------------------------------------------------------- +// Executable twins of the lane semantics, for the differential tests +// --------------------------------------------------------------------------------------------- +/// [`tbl4_byte`]. +pub fn model_tbl4_byte(t: &[u8], i: u8) -> (r: u8) + requires + t.len() == 64, + ensures + r == tbl4_byte(t@.subrange(0, 16), t@.subrange(16, 32), t@.subrange(32, 48), t@.subrange(48, 64), i as int), +{ + if i < 64 { + t[i as usize] + } else { + 0 + } +} + +/// [`u64x2_byte`]. +pub fn model_u64x2_byte(w: &[u64], k: usize) -> (r: u8) + requires + k < 8 * w.len(), + ensures + r == u64x2_byte(w@, k as int), +{ + (w[k / 8] >> (8 * (k % 8)) as u64) as u8 +} + +/// [`shr_n`]. +pub fn model_shr_n(a: u64, n: i32) -> (r: u64) + ensures + r == shr_n(a, n as int), +{ + if 0 <= n && n < 64 { + a >> (n as u64) + } else { + 0 + } +} + +/// [`shl_n`]. +pub fn model_shl_n(a: u64, n: i32) -> (r: u64) + ensures + r == shl_n(a, n as int), +{ + if 0 <= n && n < 64 { + a << (n as u64) + } else { + 0 + } +} + +} // verus! diff --git a/verification/verus/src/intrinsics/aarch64_gfneon.rs b/verification/verus/src/intrinsics/aarch64_gfneon.rs new file mode 100644 index 0000000000..cfd074e321 --- /dev/null +++ b/verification/verus/src/intrinsics/aarch64_gfneon.rs @@ -0,0 +1,265 @@ +//! Specifications of the AArch64 intrinsics the `E = GF(2^192)` NEON kernels (`gf2_64x3/aarch64.rs`) and the +//! `GF(2^8)` NEON helpers (`gf2_8.rs`) call, beyond the ones in [`super::aarch64`]. +//! +//! Same conventions: a register is the array of its lanes, lane 0 first ([`u8x16`], [`u8x8`], [`p8x8`], +//! [`u16x8`], [`p16x8`], [`u64x1`]); each result is given lane by lane, through an open spec function where it +//! is more than a move or a XOR, with an executable twin `model_` that `tests/equivalence/ +//! intrinsics_aarch64_gfneon.rs` runs against the hardware. +#[cfg(verus_keep_ghost)] +use super::aarch64::{p64x2, u64x2}; +#[cfg(verus_keep_ghost)] +use super::aarch64_bits::u8x16; +#[cfg(verus_keep_ghost)] +use super::transmuted; +#[cfg(verus_keep_ghost)] +use crate::clmul::clmul; +use core::arch::aarch64::*; +use vstd::prelude::*; + +verus! { + +#[verifier::external_type_specification] +#[verifier::external_body] +pub struct ExUint8x8(uint8x8_t); + +#[verifier::external_type_specification] +#[verifier::external_body] +pub struct ExPoly8x8(poly8x8_t); + +#[verifier::external_type_specification] +#[verifier::external_body] +pub struct ExUint16x8(uint16x8_t); + +#[verifier::external_type_specification] +#[verifier::external_body] +pub struct ExPoly16x8(poly16x8_t); + +#[verifier::external_type_specification] +#[verifier::external_body] +pub struct ExUint64x1(uint64x1_t); + +// --------------------------------------------------------------------------------------------- +// Views and layout +// --------------------------------------------------------------------------------------------- +/// The 8 bytes of a `uint8x8_t`, lane 0 first. +pub open spec fn u8x8(v: uint8x8_t) -> [u8; 8] { + transmuted::(v) +} + +/// The 8 lanes of a `poly8x8_t`, lane 0 first. +pub open spec fn p8x8(v: poly8x8_t) -> [u8; 8] { + transmuted::(v) +} + +/// The 8 lanes of a `uint16x8_t`, lane 0 first. +pub open spec fn u16x8(v: uint16x8_t) -> [u16; 8] { + transmuted::(v) +} + +/// The 8 lanes of a `poly16x8_t`, lane 0 first. +pub open spec fn p16x8(v: poly16x8_t) -> [u16; 8] { + transmuted::(v) +} + +/// The one lane of a `uint64x1_t`. +pub open spec fn u64x1(v: uint64x1_t) -> [u64; 1] { + transmuted::(v) +} + +/// Layout: a `uint8x8_t` read as a `poly8x8_t` has the same lanes. +pub broadcast axiom fn axiom_u8x8_as_p8x8(v: uint8x8_t) + ensures + #[trigger] p8x8(transmuted::(v)) == u8x8(v), +; + +/// Layout: a `u64` read as a `poly8x8_t` is its bytes, little-endian. +pub broadcast axiom fn axiom_u64_as_p8x8(x: u64, i: int) + requires + 0 <= i < 8, + ensures + #[trigger] p8x8(transmuted::(x))[i] == (x >> ((8 * i) as u64)) as u8, +; + +/// Layout: `[u64; 2]` read as a `uint64x2_t` has those lanes. +pub broadcast axiom fn axiom_words_as_u64x2(w: [u64; 2]) + ensures + #[trigger] u64x2(transmuted::<[u64; 2], uint64x2_t>(w)) == w, +; + +// --------------------------------------------------------------------------------------------- +// Lane semantics, shared by the specifications and their executable twins +// --------------------------------------------------------------------------------------------- +/// Lane `i` of PMULL on bytes: the carry-less product of lane `i` of each operand, 15 bits in a 16-bit lane. +pub open spec fn vmull_p8_lane(a: u8, b: u8) -> u16 { + clmul(a as u64, b as u128) as u16 +} + +/// Byte `k` of `UZP1` (`odd == 0`) or `UZP2` (`odd == 1`) on bytes: the even (odd) bytes of `a`, then those of +/// `b`. +pub open spec fn uzp_u8_lane(a: Seq, b: Seq, odd: int, k: int) -> u8 { + if k < 8 { + a[2 * k + odd] + } else { + b[2 * (k - 8) + odd] + } +} + +/// Lane `i` of `EXT` on 64-bit lanes: lane `i + n` of the concatenation `a`, `b`. +pub open spec fn ext_u64_lane(a: [u64; 2], b: [u64; 2], n: int, i: int) -> u64 { + if i + n < 2 { + a[i + n] + } else { + b[i + n - 2] + } +} + +/// Byte `k` of 16-bit lanes read as bytes: the low byte of lane `k / 2` at even `k`, its high byte at odd `k`. +pub open spec fn u16_byte(w: Seq, k: int) -> u8 { + (w[k / 2] >> ((8 * (k % 2)) as u16)) as u8 +} + +// --------------------------------------------------------------------------------------------- +// The specifications +// --------------------------------------------------------------------------------------------- +pub assume_specification[ vreinterpretq_p64_u64 ](a: uint64x2_t) -> (r: poly64x2_t) + ensures + p64x2(r) == u64x2(a), +; + +pub assume_specification[ vcreate_u64 ](a: u64) -> (r: uint64x1_t) + ensures + u64x1(r)[0] == a, +; + +pub assume_specification[ vcombine_u64 ](a: uint64x1_t, b: uint64x1_t) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == u64x1(a)[0], + u64x2(r)[1] == u64x1(b)[0], +; + +/// EXT: the two lanes from lane `N` on of the concatenation `a`, `b`. +pub assume_specification[ vextq_u64:: ](a: uint64x2_t, b: uint64x2_t) -> (r: uint64x2_t) + ensures + 0 <= N < 2 ==> forall|i: int| 0 <= i < 2 ==> #[trigger] u64x2(r)[i] == ext_u64_lane(u64x2(a), u64x2(b), N as int, i), +; + +/// DUP (element): lane `N` in both lanes. +pub assume_specification[ vdupq_laneq_u64:: ](a: uint64x2_t) -> (r: uint64x2_t) + ensures + 0 <= N < 2 ==> u64x2(r)[0] == u64x2(a)[N as int] && u64x2(r)[1] == u64x2(a)[N as int], +; + +pub assume_specification[ vdup_n_p8 ](value: u8) -> (r: poly8x8_t) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] p8x8(r)[i] == value, +; + +/// PMULL on bytes: eight carry-less products of byte lanes, each in a 16-bit lane. +pub assume_specification[ vmull_p8 ](a: poly8x8_t, b: poly8x8_t) -> (r: poly16x8_t) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] p16x8(r)[i] == vmull_p8_lane(p8x8(a)[i], p8x8(b)[i]), +; + +pub assume_specification[ vreinterpretq_u16_p16 ](a: poly16x8_t) -> (r: uint16x8_t) + ensures + u16x8(r) == p16x8(a), +; + +/// The 16-bit lanes as bytes, little-endian within each lane. +pub assume_specification[ vreinterpretq_u8_u16 ](a: uint16x8_t) -> (r: uint8x16_t) + ensures + forall|k: int| 0 <= k < 16 ==> #[trigger] u8x16(r)[k] == u16_byte(u16x8(a)@, k), +; + +pub assume_specification[ vgetq_lane_u16:: ](v: uint16x8_t) -> (r: u16) + ensures + 0 <= IMM5 < 8 ==> r == u16x8(v)[IMM5 as int], +; + +/// SHL (immediate) on 16-bit lanes. +pub assume_specification[ vshlq_n_u16:: ](a: uint16x8_t) -> (r: uint16x8_t) + ensures + 0 <= N < 16 ==> forall|i: int| 0 <= i < 8 ==> #[trigger] u16x8(r)[i] == u16x8(a)[i] << (N as u16), +; + +pub assume_specification[ vget_low_u8 ](a: uint8x16_t) -> (r: uint8x8_t) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] u8x8(r)[i] == u8x16(a)[i], +; + +pub assume_specification[ vget_high_u8 ](a: uint8x16_t) -> (r: uint8x8_t) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] u8x8(r)[i] == u8x16(a)[i + 8], +; + +/// UZP1: the even bytes of `a`, then those of `b`. +pub assume_specification[ vuzp1q_u8 ](a: uint8x16_t, b: uint8x16_t) -> (r: uint8x16_t) + ensures + forall|k: int| 0 <= k < 16 ==> #[trigger] u8x16(r)[k] == uzp_u8_lane(u8x16(a)@, u8x16(b)@, 0, k), +; + +/// UZP2: the odd bytes of `a`, then those of `b`. +pub assume_specification[ vuzp2q_u8 ](a: uint8x16_t, b: uint8x16_t) -> (r: uint8x16_t) + ensures + forall|k: int| 0 <= k < 16 ==> #[trigger] u8x16(r)[k] == uzp_u8_lane(u8x16(a)@, u8x16(b)@, 1, k), +; + +pub assume_specification[ veorq_u8 ](a: uint8x16_t, b: uint8x16_t) -> (r: uint8x16_t) + ensures + forall|k: int| 0 <= k < 16 ==> #[trigger] u8x16(r)[k] == u8x16(a)[k] ^ u8x16(b)[k], +; + +// --------------------------------------------------------------------------------------------- +// Executable twins +// --------------------------------------------------------------------------------------------- +/// [`vmull_p8_lane`]. +pub fn model_vmull_p8_lane(a: u8, b: u8) -> (r: u16) + ensures + r == vmull_p8_lane(a, b), +{ + crate::gf2_8::clmul8_software(a, b) +} + +/// [`uzp_u8_lane`]. +pub fn model_uzp_u8_lane(a: &[u8], b: &[u8], odd: usize, k: usize) -> (r: u8) + requires + a.len() == 16, + b.len() == 16, + odd < 2, + k < 16, + ensures + r == uzp_u8_lane(a@, b@, odd as int, k as int), +{ + if k < 8 { + a[2 * k + odd] + } else { + b[2 * (k - 8) + odd] + } +} + +/// [`ext_u64_lane`]. +pub fn model_ext_u64_lane(a: [u64; 2], b: [u64; 2], n: usize, i: usize) -> (r: u64) + requires + n < 2, + i < 2, + ensures + r == ext_u64_lane(a, b, n as int, i as int), +{ + if i + n < 2 { + a[i + n] + } else { + b[i + n - 2] + } +} + +/// [`u16_byte`]. +pub fn model_u16_byte(w: &[u16], k: usize) -> (r: u8) + requires + k < 2 * w.len(), + ensures + r == u16_byte(w@, k as int), +{ + (w[k / 2] >> ((8 * (k % 2)) as u16)) as u8 +} + +} // verus! diff --git a/verification/verus/src/intrinsics/aarch64_nttsimd.rs b/verification/verus/src/intrinsics/aarch64_nttsimd.rs new file mode 100644 index 0000000000..714700730d --- /dev/null +++ b/verification/verus/src/intrinsics/aarch64_nttsimd.rs @@ -0,0 +1,50 @@ +//! The memory helpers of the NEON NTT butterfly kernels (`ntt_simd`): their loads and stores, with trusted +//! specifications. +//! +//! Production's kernels load and store through raw pointers into the two rows. Verus cannot tie a pointer to +//! the slice it came from, so the verified copies take the rows as slices and an offset, and call these helpers, +//! whose bodies are production's `vld1q_u64` / `vst1q_u64` at that offset. +//! `tests/equivalence/intrinsics_aarch64_nttsimd.rs` checks each against the memory it reads or writes. They are +//! `unsafe fn`, as the loads and stores they wrap; their `requires` is their safety condition. +#[cfg(verus_keep_ghost)] +use super::aarch64::u64x2; +use crate::gf2_64::F64; +use core::arch::aarch64::*; +use vstd::prelude::*; + +verus! { + +/// `vld1q_u64` of the two words of `row` from `at`. +#[verifier::external_body] +#[inline(always)] +pub unsafe fn vld1q_u64_at(row: &[F64], at: usize) -> (r: uint64x2_t) + requires + at + 2 <= row.len(), + ensures + u64x2(r)[0] == row@[at as int].0, + u64x2(r)[1] == row@[at + 1].0, +{ + // SAFETY: `at + 2 <= row.len()`, `F64` is a transparent `u64`, and NEON is part of the aarch64 baseline. + unsafe { vld1q_u64(row.as_ptr().add(at).cast()) } +} + +/// `vst1q_u64` of `v` to the two words of `row` from `at`. +#[verifier::external_body] +#[inline(always)] +pub unsafe fn vst1q_u64_at(row: &mut [F64], at: usize, v: uint64x2_t) + requires + at + 2 <= old(row).len(), + ensures + final(row).len() == old(row).len(), + forall|j: int| + 0 <= j < old(row).len() ==> #[trigger] final(row)@[j] == if at <= j < at + 2 { + F64(u64x2(v)[j - at]) + } else { + old(row)@[j] + }, +{ + // SAFETY: `at + 2 <= row.len()`, `F64` is a transparent `u64`, and NEON is part of the aarch64 baseline. + unsafe { vst1q_u64(row.as_mut_ptr().add(at).cast(), v) } +} + +} // verus! diff --git a/verification/verus/src/intrinsics/mod.rs b/verification/verus/src/intrinsics/mod.rs new file mode 100644 index 0000000000..b55c660ca5 --- /dev/null +++ b/verification/verus/src/intrinsics/mod.rs @@ -0,0 +1,53 @@ +//! Specifications of the `core::arch` intrinsics the SIMD kernels call: the trust base of their proofs. +//! +//! Verus cannot see into an intrinsic, so each one used by a verified kernel gets an `assume_specification` +//! stating its lane-level semantics. A vector register is viewed as the array of words a `transmute` gives +//! (`__m512i` as `[u64; 8]`, `uint8x16_t` as `[u8; 16]`, ...): [`transmuted`] is that value, and the view +//! functions in [`x86`] and [`aarch64`] name it per type. Specifications use lane-wise expressions and open +//! spec functions on those arrays, with executable models for nontrivial lane operations. The +//! `tests/equivalence/intrinsics_*` modules compare real intrinsics on edge and random inputs with these +//! models. These checks provide evidence about the assumptions, not a proof of hardware semantics. +//! +//! The other assumptions are the layout facts in this file and in the per-architecture modules (`axiom_*`): +//! reinterpreting a register as another array type or integer is the little-endian split of the same bits. +//! The tests check those too. +//! +//! Loads and stores through raw pointers are not specified: Verus has no way to tie a pointer an intrinsic +//! reads to the slice it came from. Where a kernel moves whole arrays between memory and registers, its +//! verified copy calls a helper (e.g. `load512`) with a trusted specification, whose body is the production +//! expression, and which `tests/equivalence/intrinsics_*` checks like an intrinsic. +use vstd::prelude::*; + +#[cfg(target_arch = "aarch64")] +pub mod aarch64; +#[cfg(target_arch = "aarch64")] +pub mod aarch64_bits; +#[cfg(target_arch = "aarch64")] +pub mod aarch64_gfneon; +#[cfg(target_arch = "aarch64")] +pub mod aarch64_nttsimd; +#[cfg(target_arch = "x86_64")] +pub mod x86; +#[cfg(target_arch = "x86_64")] +pub mod x86_bits; +#[cfg(target_arch = "x86_64")] +pub mod x86_gfneon; +#[cfg(target_arch = "x86_64")] +pub mod x86_gfx86; +#[cfg(target_arch = "x86_64")] +pub mod x86_nttsimd; + +verus! { + +/// The value `core::mem::transmute::(s)` returns: the bits of `s`, read as a `D`. +/// +/// Uninterpreted: what it is for each pair of types is given by the view functions and the layout axioms. +pub uninterp spec fn transmuted(s: S) -> D; + +/// `transmute` returns [`transmuted`]. +pub assume_specification[ core::mem::transmute:: ](s: S) -> (d: D) + ensures + d == transmuted::(s), +; + +} // verus! diff --git a/verification/verus/src/intrinsics/x86.rs b/verification/verus/src/intrinsics/x86.rs new file mode 100644 index 0000000000..ea1ae31c40 --- /dev/null +++ b/verification/verus/src/intrinsics/x86.rs @@ -0,0 +1,703 @@ +//! Specifications of the x86-64 intrinsics (SSE2, PCLMULQDQ, BMI2, AVX2, VPCLMULQDQ, AVX-512F). +//! +//! A register is viewed as its 64-bit words, word 0 the least significant ([`m128`], [`m256`], [`m512`]). +//! Each intrinsic's result is given lane by lane by an open spec function `_lane` over those +//! words, and each has an executable twin `model_` proven to compute the same lanes, which the +//! differential tests run against the hardware. +#[cfg(verus_keep_ghost)] +use super::transmuted; +#[cfg(verus_keep_ghost)] +use crate::clmul::{bit, clmul}; +use core::arch::x86_64::*; +use vstd::prelude::*; + +verus! { + +#[verifier::external_type_specification] +#[verifier::external_body] +pub struct ExM128i(__m128i); + +#[verifier::external_type_specification] +#[verifier::external_body] +pub struct ExM256i(__m256i); + +#[verifier::external_type_specification] +#[verifier::external_body] +pub struct ExM512i(__m512i); + +// --------------------------------------------------------------------------------------------- +// Views and layout +// --------------------------------------------------------------------------------------------- +/// The two 64-bit words of a 128-bit register, the low one first. +pub open spec fn m128(v: __m128i) -> [u64; 2] { + transmuted::<__m128i, [u64; 2]>(v) +} + +/// The four 64-bit words of a 256-bit register, the low one first. +pub open spec fn m256(v: __m256i) -> [u64; 4] { + transmuted::<__m256i, [u64; 4]>(v) +} + +/// The eight 64-bit words of a 512-bit register, the low one first. +pub open spec fn m512(v: __m512i) -> [u64; 8] { + transmuted::<__m512i, [u64; 8]>(v) +} + +/// The two words of a 128-bit register as one little-endian `u128`. +pub open spec fn m128_u128(v: __m128i) -> u128 { + (m128(v)[0] as u128) | ((m128(v)[1] as u128) << 64u128) +} + +/// The 16 bytes of a 128-bit register, the low one first. +pub open spec fn m128_bytes(v: __m128i) -> [u8; 16] { + transmuted::<__m128i, [u8; 16]>(v) +} + +/// The 32 bytes of a 256-bit register, the low one first. +pub open spec fn m256_bytes(v: __m256i) -> [u8; 32] { + transmuted::<__m256i, [u8; 32]>(v) +} + +/// The 64 bytes of a 512-bit register, the low one first. +pub open spec fn m512_bytes(v: __m512i) -> [u8; 64] { + transmuted::<__m512i, [u8; 64]>(v) +} + +/// Byte `k` of a sequence of words, little-endian. +pub open spec fn word_byte(w: Seq, k: int) -> u8 { + (w[k / 8] >> ((8 * (k % 8)) as u64)) as u8 +} + +/// Layout: the bytes of a 128-bit register are its words, little-endian. +pub broadcast axiom fn axiom_m128_bytes(v: __m128i, k: int) + requires + 0 <= k < 16, + ensures + #[trigger] m128_bytes(v)[k] == word_byte(m128(v)@, k), +; + +/// Layout: the bytes of a 256-bit register are its words, little-endian. +pub broadcast axiom fn axiom_m256_bytes(v: __m256i, k: int) + requires + 0 <= k < 32, + ensures + #[trigger] m256_bytes(v)[k] == word_byte(m256(v)@, k), +; + +/// Layout: the bytes of a 512-bit register are its words, little-endian. +pub broadcast axiom fn axiom_m512_bytes(v: __m512i, k: int) + requires + 0 <= k < 64, + ensures + #[trigger] m512_bytes(v)[k] == word_byte(m512(v)@, k), +; + +/// Layout: a 128-bit register read as a `u128` is its two words, little-endian. +pub broadcast axiom fn axiom_m128_as_u128(v: __m128i) + ensures + #[trigger] transmuted::<__m128i, u128>(v) == m128_u128(v), +; + +/// Layout: a 256-bit register read as `[[u64; 2]; 2]` is its words in order. +pub axiom fn axiom_m256_as_pairs(v: __m256i) + ensures + forall|i: int, j: int| + 0 <= i < 2 && 0 <= j < 2 ==> #[trigger] transmuted::<__m256i, [[u64; 2]; 2]>(v)[i][j] == m256(v)[2 * i + + j], +; + +/// Layout: a 512-bit register read as `[[u64; 2]; 4]` is its words in order. +pub axiom fn axiom_m512_as_pairs(v: __m512i) + ensures + forall|i: int, j: int| + 0 <= i < 4 && 0 <= j < 2 ==> #[trigger] transmuted::<__m512i, [[u64; 2]; 4]>(v)[i][j] == m512(v)[2 * i + + j], +; + +/// Layout: `[u64; 8]` read as a 512-bit register has those words. +pub axiom fn axiom_m512_from_words(w: [u64; 8]) + ensures + m512(transmuted::<[u64; 8], __m512i>(w)) == w, +; + +// --------------------------------------------------------------------------------------------- +// Lane semantics, shared by the specifications and their executable twins +// --------------------------------------------------------------------------------------------- +/// The carry-less product of word `ia` of `a` and word `ib` of `b`, as two words (PCLMULQDQ on one lane). +pub open spec fn clmul_words(a: u64, b: u64) -> [u64; 2] { + let p = clmul(a, b as u128); + [p as u64, (p >> 64u128) as u64] +} + +/// Which word of a 128-bit lane PCLMULQDQ takes from its first operand: bit 0 of the immediate. +pub open spec fn clmul_sel_a(imm8: u8) -> int { + (imm8 & 1) as int +} + +/// Which word of a 128-bit lane PCLMULQDQ takes from its second operand: bit 4 of the immediate. +pub open spec fn clmul_sel_b(imm8: u8) -> int { + ((imm8 >> 4u8) & 1) as int +} + +/// The word selections of the four PCLMULQDQ immediates. +pub proof fn lemma_clmul_sel() + ensures + clmul_sel_a(0x00i32 as u8) == 0 && clmul_sel_b(0x00i32 as u8) == 0, + clmul_sel_a(0x01i32 as u8) == 1 && clmul_sel_b(0x01i32 as u8) == 0, + clmul_sel_a(0x10i32 as u8) == 0 && clmul_sel_b(0x10i32 as u8) == 1, + clmul_sel_a(0x11i32 as u8) == 1 && clmul_sel_b(0x11i32 as u8) == 1, +{ + assert((0x00u8 & 1) == 0 && ((0x00u8 >> 4u8) & 1) == 0) by (bit_vector); + assert((0x01u8 & 1) == 1 && ((0x01u8 >> 4u8) & 1) == 0) by (bit_vector); + assert((0x10u8 & 1) == 0 && ((0x10u8 >> 4u8) & 1) == 1) by (bit_vector); + assert((0x11u8 & 1) == 1 && ((0x11u8 >> 4u8) & 1) == 1) by (bit_vector); +} + +/// Word `i` of a carry-less multiply of each 128-bit lane (`_mm*_clmulepi64_*`). +pub open spec fn clmul_lane(a: Seq, b: Seq, imm8: u8, i: int) -> u64 { + let l = 2 * (i / 2); + clmul_words(a[l + clmul_sel_a(imm8)], b[l + clmul_sel_b(imm8)])[i % 2] +} + +/// Word `i` of `unpacklo_epi64`: the low words of each 128-bit lane of `a` and `b`, interleaved. +pub open spec fn unpacklo_lane(a: Seq, b: Seq, i: int) -> u64 { + if i % 2 == 0 { + a[i] + } else { + b[i - 1] + } +} + +/// Word `i` of `unpackhi_epi64`: the high words of each 128-bit lane of `a` and `b`, interleaved. +pub open spec fn unpackhi_lane(a: Seq, b: Seq, i: int) -> u64 { + if i % 2 == 0 { + a[i + 1] + } else { + b[i] + } +} + +/// A logical left shift of a word by an immediate: zero from 64 on. +pub open spec fn shl_imm(a: u64, imm8: int) -> u64 { + if 0 <= imm8 < 64 { + a << (imm8 as u64) + } else { + 0 + } +} + +/// A logical right shift of a word by an immediate: zero from 64 on. +pub open spec fn shr_imm(a: u64, imm8: int) -> u64 { + if 0 <= imm8 < 64 { + a >> (imm8 as u64) + } else { + 0 + } +} + +/// Word `i` of `_mm256_permute4x64_epi64`: word `imm8[2i + 1 : 2i]` of `a`. +pub open spec fn permute4x64_lane(a: Seq, imm8: u8, i: int) -> u64 { + a[((imm8 >> ((2 * i) as u8)) & 3) as int] +} + +/// Word `i` of `_mm512_shuffle_i64x2`: 128-bit lanes 0 and 1 from `a`, 2 and 3 from `b`, each picked by two +/// bits of the immediate. +pub open spec fn shuffle_i64x2_lane(a: Seq, b: Seq, imm8: u8, i: int) -> u64 { + let q = i / 2; + let src = ((imm8 >> ((2 * q) as u8)) & 3) as int; + if q < 2 { + a[2 * src + i % 2] + } else { + b[2 * src + i % 2] + } +} + +/// Word `i` of `_mm512_permutex2var_epi64`: word `idx[i] & 7` of `a`, or of `b` if bit 3 of `idx[i]` is set. +pub open spec fn permutex2var_lane(a: Seq, idx: Seq, b: Seq, i: int) -> u64 { + let k = (idx[i] & 7) as int; + if idx[i] & 8 == 0 { + a[k] + } else { + b[k] + } +} + +/// Byte `k` of `_mm*_shuffle_epi8(a, b)`: zero if bit 7 of `b[k]` is set, else byte `b[k] & 15` of the 16-byte +/// lane of `a` that byte `k` is in. +pub open spec fn shuffle_epi8_lane(a: Seq, b: Seq, k: int) -> u8 { + if b[k] & 0x80 != 0 { + 0 + } else { + a[16 * (k / 16) + (b[k] & 15) as int] + } +} + +/// All ones if bit `k` of the immediate is set, else zero. +pub open spec fn imm_mask(imm8: u8, k: u8) -> u64 { + if (imm8 >> k) & 1 == 1 { + !0u64 + } else { + 0u64 + } +} + +/// `vpternlogq` on one word: bit `j` of the result is bit `4 a_j + 2 b_j + c_j` of the immediate. +pub open spec fn ternlog(imm8: u8, a: u64, b: u64, c: u64) -> u64 { + (imm_mask(imm8, 0) & !a & !b & !c) | (imm_mask(imm8, 1) & !a & !b & c) | (imm_mask(imm8, 2) & !a & b & !c) | ( + imm_mask(imm8, 3) & !a & b & c) | (imm_mask(imm8, 4) & a & !b & !c) | (imm_mask(imm8, 5) & a & !b & c) | ( + imm_mask(imm8, 6) & a & b & !c) | (imm_mask(imm8, 7) & a & b & c) +} + +/// Bit `j` of `_pdep_u64(a, mask)`: zero where `mask` is clear, else the next unused bit of `a`, counting +/// from bit 0: bit `popcount(mask & (2^j - 1))` of `a`. +pub open spec fn pdep_bit(a: u64, mask: u64, j: nat) -> bool { + (mask >> (j as u64)) & 1 == 1 && (a >> (ones_below(mask, j) as u64)) & 1 == 1 +} + +/// The number of set bits of `mask` below bit `j`. +pub open spec fn ones_below(mask: u64, j: nat) -> nat + decreases j, +{ + if j == 0 { + 0 + } else { + ones_below(mask, (j - 1) as nat) + if (mask >> ((j - 1) as u64)) & 1 == 1 { + 1nat + } else { + 0nat + } + } +} + +// --------------------------------------------------------------------------------------------- +// The specifications +// --------------------------------------------------------------------------------------------- +pub assume_specification[ _mm_cvtsi64_si128 ](a: i64) -> (r: __m128i) + ensures + m128(r)[0] == a as u64, + m128(r)[1] == 0, +; + +pub assume_specification[ _mm_cvtsi128_si64 ](a: __m128i) -> (r: i64) + ensures + r as u64 == m128(a)[0], +; + +pub assume_specification[ _mm_set_epi64x ](e1: i64, e0: i64) -> (r: __m128i) + ensures + m128(r)[0] == e0 as u64, + m128(r)[1] == e1 as u64, +; + +pub assume_specification[ _mm_xor_si128 ](a: __m128i, b: __m128i) -> (r: __m128i) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] m128(r)[i] == m128(a)[i] ^ m128(b)[i], +; + +pub assume_specification[ _mm_unpacklo_epi64 ](a: __m128i, b: __m128i) -> (r: __m128i) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] m128(r)[i] == unpacklo_lane(m128(a)@, m128(b)@, i), +; + +pub assume_specification[ _mm_unpackhi_epi64 ](a: __m128i, b: __m128i) -> (r: __m128i) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] m128(r)[i] == unpackhi_lane(m128(a)@, m128(b)@, i), +; + +pub assume_specification[ _mm_clmulepi64_si128:: ](a: __m128i, b: __m128i) -> (r: __m128i) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] m128(r)[i] == clmul_lane(m128(a)@, m128(b)@, IMM8 as u8, i), +; + +pub assume_specification[ _pdep_u64 ](a: u64, mask: u64) -> (r: u64) + ensures + forall|j: nat| j < 64 ==> #[trigger] bit(r, j) == pdep_bit(a, mask, j), +; + +pub assume_specification[ _mm256_xor_si256 ](a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == m256(a)[i] ^ m256(b)[i], +; + +pub assume_specification[ _mm256_and_si256 ](a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == m256(a)[i] & m256(b)[i], +; + +pub assume_specification[ _mm256_add_epi64 ](a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == (m256(a)[i] + m256(b)[i]) as u64, +; + +pub assume_specification[ _mm256_clmulepi64_epi128:: ](a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == clmul_lane(m256(a)@, m256(b)@, IMM8 as u8, i), +; + +pub assume_specification[ _mm256_unpacklo_epi64 ](a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == unpacklo_lane(m256(a)@, m256(b)@, i), +; + +pub assume_specification[ _mm256_unpackhi_epi64 ](a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == unpackhi_lane(m256(a)@, m256(b)@, i), +; + +pub assume_specification[ _mm256_slli_epi64:: ](a: __m256i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == shl_imm(m256(a)[i], IMM8 as int), +; + +pub assume_specification[ _mm256_srli_epi64:: ](a: __m256i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == shr_imm(m256(a)[i], IMM8 as int), +; + +pub assume_specification[ _mm256_set1_epi64x ](a: i64) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == a as u64, +; + +pub assume_specification[ _mm256_set_epi64x ](e3: i64, e2: i64, e1: i64, e0: i64) -> (r: __m256i) + ensures + m256(r)[0] == e0 as u64, + m256(r)[1] == e1 as u64, + m256(r)[2] == e2 as u64, + m256(r)[3] == e3 as u64, +; + +pub assume_specification[ _mm256_shuffle_epi8 ](a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == shuffle_epi8_lane(m256_bytes(a)@, m256_bytes(b)@, k), +; + +pub assume_specification[ _mm256_broadcastsi128_si256 ](a: __m128i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == m128(a)[i % 2], +; + +pub assume_specification[ _mm256_permute4x64_epi64:: ](a: __m256i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == permute4x64_lane(m256(a)@, IMM8 as u8, i), +; + +pub assume_specification[ _mm512_xor_si512 ](a: __m512i, b: __m512i) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == m512(a)[i] ^ m512(b)[i], +; + +pub assume_specification[ _mm512_clmulepi64_epi128:: ](a: __m512i, b: __m512i) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == clmul_lane(m512(a)@, m512(b)@, IMM8 as u8, i), +; + +pub assume_specification[ _mm512_unpacklo_epi64 ](a: __m512i, b: __m512i) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == unpacklo_lane(m512(a)@, m512(b)@, i), +; + +pub assume_specification[ _mm512_unpackhi_epi64 ](a: __m512i, b: __m512i) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == unpackhi_lane(m512(a)@, m512(b)@, i), +; + +pub assume_specification[ _mm512_slli_epi64:: ](a: __m512i) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == shl_imm(m512(a)[i], IMM8 as int), +; + +pub assume_specification[ _mm512_srli_epi64:: ](a: __m512i) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == shr_imm(m512(a)[i], IMM8 as int), +; + +pub assume_specification[ _mm512_set1_epi64 ](a: i64) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == a as u64, +; + +pub assume_specification[ _mm512_set_epi64 ]( + e7: i64, + e6: i64, + e5: i64, + e4: i64, + e3: i64, + e2: i64, + e1: i64, + e0: i64, +) -> (r: __m512i) + ensures + m512(r)[0] == e0 as u64, + m512(r)[1] == e1 as u64, + m512(r)[2] == e2 as u64, + m512(r)[3] == e3 as u64, + m512(r)[4] == e4 as u64, + m512(r)[5] == e5 as u64, + m512(r)[6] == e6 as u64, + m512(r)[7] == e7 as u64, +; + +pub assume_specification[ _mm512_shuffle_i64x2:: ](a: __m512i, b: __m512i) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == shuffle_i64x2_lane(m512(a)@, m512(b)@, IMM8 as u8, i), +; + +pub assume_specification[ _mm512_permutex2var_epi64 ](a: __m512i, idx: __m512i, b: __m512i) -> (r: __m512i) + ensures + forall|i: int| + 0 <= i < 8 ==> #[trigger] m512(r)[i] == permutex2var_lane(m512(a)@, m512(idx)@, m512(b)@, i), +; + +pub assume_specification[ _mm512_ternarylogic_epi64:: ]( + a: __m512i, + b: __m512i, + c: __m512i, +) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == ternlog(IMM8 as u8, m512(a)[i], m512(b)[i], m512(c)[i]), +; + +// --------------------------------------------------------------------------------------------- +// Executable twins of the lane semantics, for the differential tests +// --------------------------------------------------------------------------------------------- +/// [`clmul_words`]. +pub fn model_clmul_words(a: u64, b: u64) -> (r: [u64; 2]) + ensures + r == clmul_words(a, b), +{ + let p = crate::gf2_64::software::clmul(a, b); + let r = [p as u64, (p >> 64u128) as u64]; + assert(r =~= clmul_words(a, b)); + r +} + +/// [`clmul_lane`]. +pub fn model_clmul_lane(a: &[u64], b: &[u64], imm8: u8, i: usize) -> (r: u64) + requires + a.len() == b.len(), + a.len() % 2 == 0, + i < a.len(), + ensures + r == clmul_lane(a@, b@, imm8, i as int), +{ + let l = 2 * (i / 2); + let (sa, sb) = ((imm8 & 1) as usize, ((imm8 >> 4u8) & 1) as usize); + assert((imm8 & 1) <= 1 && ((imm8 >> 4u8) & 1) <= 1) by (bit_vector); + model_clmul_words(a[l + sa], b[l + sb])[i % 2] +} + +/// [`unpacklo_lane`]. +pub fn model_unpacklo_lane(a: &[u64], b: &[u64], i: usize) -> (r: u64) + requires + a.len() == b.len(), + a.len() % 2 == 0, + i < a.len(), + ensures + r == unpacklo_lane(a@, b@, i as int), +{ + if i % 2 == 0 { + a[i] + } else { + b[i - 1] + } +} + +/// [`unpackhi_lane`]. +pub fn model_unpackhi_lane(a: &[u64], b: &[u64], i: usize) -> (r: u64) + requires + a.len() == b.len(), + a.len() % 2 == 0, + i < a.len(), + ensures + r == unpackhi_lane(a@, b@, i as int), +{ + if i % 2 == 0 { + a[i + 1] + } else { + b[i] + } +} + +/// [`shl_imm`]. +pub fn model_shl_imm(a: u64, imm8: u32) -> (r: u64) + ensures + r == shl_imm(a, imm8 as int), +{ + if imm8 < 64 { + a << imm8 + } else { + 0 + } +} + +/// [`shr_imm`]. +pub fn model_shr_imm(a: u64, imm8: u32) -> (r: u64) + ensures + r == shr_imm(a, imm8 as int), +{ + if imm8 < 64 { + a >> imm8 + } else { + 0 + } +} + +/// [`permute4x64_lane`]. +pub fn model_permute4x64_lane(a: &[u64], imm8: u8, i: usize) -> (r: u64) + requires + a.len() == 4, + i < 4, + ensures + r == permute4x64_lane(a@, imm8, i as int), +{ + let s = (2 * i) as u8; + assert((imm8 >> s) & 3 <= 3) by (bit_vector); + a[((imm8 >> s) & 3) as usize] +} + +/// [`shuffle_i64x2_lane`]. +pub fn model_shuffle_i64x2_lane(a: &[u64], b: &[u64], imm8: u8, i: usize) -> (r: u64) + requires + a.len() == 8, + b.len() == 8, + i < 8, + ensures + r == shuffle_i64x2_lane(a@, b@, imm8, i as int), +{ + let q = i / 2; + let s = (2 * q) as u8; + assert((imm8 >> s) & 3 <= 3) by (bit_vector); + let src = ((imm8 >> s) & 3) as usize; + if q < 2 { + a[2 * src + i % 2] + } else { + b[2 * src + i % 2] + } +} + +/// [`permutex2var_lane`]. +pub fn model_permutex2var_lane(a: &[u64], idx: &[u64], b: &[u64], i: usize) -> (r: u64) + requires + a.len() == 8, + idx.len() == 8, + b.len() == 8, + i < 8, + ensures + r == permutex2var_lane(a@, idx@, b@, i as int), +{ + let x = idx[i]; + assert(x & 7 <= 7) by (bit_vector); + let k = (x & 7) as usize; + if x & 8 == 0 { + a[k] + } else { + b[k] + } +} + +/// [`shuffle_epi8_lane`]. +pub fn model_shuffle_epi8_lane(a: &[u8], b: &[u8], k: usize) -> (r: u8) + requires + a.len() == b.len(), + a.len() % 16 == 0, + k < a.len(), + ensures + r == shuffle_epi8_lane(a@, b@, k as int), +{ + if b[k] & 0x80 != 0 { + 0 + } else { + let x = b[k]; + assert(x & 15 < 16) by (bit_vector); + a[16 * (k / 16) + (x & 15) as usize] + } +} + +/// [`imm_mask`]. +pub fn model_imm_mask(imm8: u8, k: u8) -> (r: u64) + requires + 0 <= k < 8, + ensures + r == imm_mask(imm8, k), +{ + if (imm8 >> k) & 1 == 1 { + !0u64 + } else { + 0u64 + } +} + +/// [`ternlog`]. +pub fn model_ternlog(imm8: u8, a: u64, b: u64, c: u64) -> (r: u64) + ensures + r == ternlog(imm8, a, b, c), +{ + let m = [ + model_imm_mask(imm8, 0), + model_imm_mask(imm8, 1), + model_imm_mask(imm8, 2), + model_imm_mask(imm8, 3), + model_imm_mask(imm8, 4), + model_imm_mask(imm8, 5), + model_imm_mask(imm8, 6), + model_imm_mask(imm8, 7), + ]; + (m[0] & !a & !b & !c) | (m[1] & !a & !b & c) | (m[2] & !a & b & !c) | (m[3] & !a & b & c) | (m[4] & a & !b & !c) + | (m[5] & a & !b & c) | (m[6] & a & b & !c) | (m[7] & a & b & c) +} + +/// [`pdep_bit`], for all 64 bits: the deposit computed bit by bit. +pub fn model_pdep(a: u64, mask: u64) -> (r: u64) + ensures + forall|j: nat| j < 64 ==> #[trigger] bit(r, j) == pdep_bit(a, mask, j), +{ + let mut r = 0u64; + let mut k = 0u64; + let mut j = 0u64; + assert forall|t: nat| t < 64 implies !#[trigger] bit(0u64, t) by { + let tt = t as u64; + assert((0u64 >> tt) & 1 == 0) by (bit_vector); + } + while j < 64 + invariant + j <= 64, + k == ones_below(mask, j as nat), + k <= j, + forall|t: nat| t < j ==> #[trigger] bit(r, t) == pdep_bit(a, mask, t), + forall|t: nat| j <= t < 64 ==> !#[trigger] bit(r, t), + decreases 64 - j, + { + let m = (mask >> j) & 1 == 1; + let set = m && (a >> k) & 1 == 1; + let ghost r0 = r; + if set { + r = r | (1u64 << j); + } + proof { + assert forall|t: nat| t < 64 implies #[trigger] bit(r, t) == if t == j as nat { + set || bit(r0, t) + } else { + bit(r0, t) + } by { + let tt = t as u64; + if set { + assert(tt < 64 ==> (((r0 | (1u64 << j)) >> tt) & 1 == 1) == (tt == j || (r0 >> tt) & 1 == 1)) + by (bit_vector); + } + } + assert(ones_below(mask, (j + 1) as nat) == ones_below(mask, j as nat) + if m { + 1nat + } else { + 0nat + }); + } + if m { + k = k + 1; + } + j = j + 1; + } + r +} + +} // verus! diff --git a/verification/verus/src/intrinsics/x86_bits.rs b/verification/verus/src/intrinsics/x86_bits.rs new file mode 100644 index 0000000000..b5d0728744 --- /dev/null +++ b/verification/verus/src/intrinsics/x86_bits.rs @@ -0,0 +1,292 @@ +//! Specifications of the x86-64 intrinsics of the bit transposes (AVX2 byte unpacks, AVX-512 VBMI's byte +//! permute, GFNI's affine map), and the byte loads and stores of their kernels. +//! +//! The views are those of [`super::x86`]: [`m256_bytes`] and [`m512_bytes`] for byte lanes, [`m256`] and +//! [`m512`] for 64-bit words. Each lane function has an executable twin `model_` proven equal to it, which +//! `tests/equivalence/intrinsics_x86_bits.rs` runs against the hardware. +#[cfg(verus_keep_ghost)] +use super::x86::{m128_bytes, m256, m256_bytes, m512, m512_bytes}; +use core::arch::x86_64::*; +use vstd::prelude::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Lane semantics +// --------------------------------------------------------------------------------------------- +/// Byte `k` of `_mm256_unpacklo_epi8(a, b)`: in each 16-byte lane, the low eight bytes of `a` and `b`, +/// interleaved (`a` at the even bytes). +pub open spec fn unpacklo_epi8_lane(a: Seq, b: Seq, k: int) -> u8 { + let j = 16 * (k / 16) + (k % 16) / 2; + if k % 2 == 0 { + a[j] + } else { + b[j] + } +} + +/// Byte `k` of `_mm256_unpackhi_epi8(a, b)`: in each 16-byte lane, the high eight bytes of `a` and `b`, +/// interleaved (`a` at the even bytes). +pub open spec fn unpackhi_epi8_lane(a: Seq, b: Seq, k: int) -> u8 { + let j = 16 * (k / 16) + 8 + (k % 16) / 2; + if k % 2 == 0 { + a[j] + } else { + b[j] + } +} + +/// Byte `k` of `_mm512_permutexvar_epi8(idx, a)`: byte `idx[k] & 63` of `a`, across the whole register. +pub open spec fn permutexvar_epi8_lane(idx: Seq, a: Seq, k: int) -> u8 { + a[(idx[k] & 63) as int] +} + +/// The parity of a byte: 1 if it has an odd number of set bits, else 0. +pub open spec fn parity8(x: u8) -> u8 { + let x = x ^ (x >> 4u8); + let x = x ^ (x >> 2u8); + (x ^ (x >> 1u8)) & 1 +} + +/// Bit `i` of a GF2P8AFFINEQB result byte, as a 0 or 1: the parity of matrix row `m & x`, XOR bit `i` of the +/// immediate. +pub open spec fn affine_bit(m: u8, x: u8, imm8: u8, i: u8) -> u8 { + parity8(m & x) ^ ((imm8 >> i) & 1) +} + +/// GF2P8AFFINEQB on one byte `x` with the matrix word `a`: bit `i` of the result is +/// `parity(byte (7 - i) of a & x) XOR bit i of imm8`. +pub open spec fn affine_byte(a: u64, x: u8, imm8: u8) -> u8 { + affine_bit((a >> 56u64) as u8, x, imm8, 0) | (affine_bit((a >> 48u64) as u8, x, imm8, 1) << 1u8) | (affine_bit( + (a >> 40u64) as u8, + x, + imm8, + 2, + ) << 2u8) | (affine_bit((a >> 32u64) as u8, x, imm8, 3) << 3u8) | (affine_bit((a >> 24u64) as u8, x, imm8, 4) + << 4u8) | (affine_bit((a >> 16u64) as u8, x, imm8, 5) << 5u8) | (affine_bit((a >> 8u64) as u8, x, imm8, 6) + << 6u8) | (affine_bit(a as u8, x, imm8, 7) << 7u8) +} + +/// Byte `k` of `_mm*_gf2p8affine_epi64_epi8::(x, a)`: byte `k` of `x` through the matrix in the 64-bit +/// word of `a` that byte `k` is in. +pub open spec fn gf2p8affine_lane(x: Seq, a: Seq, imm8: u8, k: int) -> u8 { + affine_byte(a[k / 8], x[k], imm8) +} + +// --------------------------------------------------------------------------------------------- +// The specifications +// --------------------------------------------------------------------------------------------- +pub assume_specification[ _mm256_unpacklo_epi8 ](a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|k: int| + 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == unpacklo_epi8_lane(m256_bytes(a)@, m256_bytes(b)@, k), +; + +pub assume_specification[ _mm256_unpackhi_epi8 ](a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|k: int| + 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == unpackhi_epi8_lane(m256_bytes(a)@, m256_bytes(b)@, k), +; + +pub assume_specification[ _mm512_permutexvar_epi8 ](idx: __m512i, a: __m512i) -> (r: __m512i) + ensures + forall|k: int| + 0 <= k < 64 ==> #[trigger] m512_bytes(r)[k] == permutexvar_epi8_lane(m512_bytes(idx)@, m512_bytes(a)@, k), +; + +pub assume_specification[ _mm256_gf2p8affine_epi64_epi8:: ](x: __m256i, a: __m256i) -> (r: __m256i) + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == gf2p8affine_lane(m256_bytes(x)@, m256(a)@, B as u8, k), +; + +pub assume_specification[ _mm512_gf2p8affine_epi64_epi8:: ](x: __m512i, a: __m512i) -> (r: __m512i) + ensures + forall|k: int| 0 <= k < 64 ==> #[trigger] m512_bytes(r)[k] == gf2p8affine_lane(m512_bytes(x)@, m512(a)@, B as u8, k), +; + +// --------------------------------------------------------------------------------------------- +// Loads and stores of byte arrays +// --------------------------------------------------------------------------------------------- +/// `_mm_loadu_si128` of a 16-byte array. +/// +/// # Safety +/// +/// Requires the `sse2` target feature. +#[verifier::external_body] +#[inline] +#[target_feature(enable = "sse2")] +pub unsafe fn load128_bytes(x: &[u8; 16]) -> (r: __m128i) + ensures + m128_bytes(r) == *x, +{ + unsafe { _mm_loadu_si128(x.as_ptr().cast()) } +} + +/// `_mm256_loadu_si256` of the 32 bytes of a 64-byte array from `offset` on. +/// +/// # Safety +/// +/// Requires the `avx` target feature. +#[verifier::external_body] +#[inline] +#[target_feature(enable = "avx")] +pub unsafe fn load256_bytes_at(x: &[u8; 64], offset: usize) -> (r: __m256i) + requires + offset <= 32, + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == x[offset + k], +{ + unsafe { _mm256_loadu_si256(x.as_ptr().add(offset).cast()) } +} + +/// `_mm256_storeu_si256` into the 32 bytes of a 64-byte array from `offset` on. +/// +/// # Safety +/// +/// Requires the `avx` target feature. +#[verifier::external_body] +#[inline] +#[target_feature(enable = "avx")] +pub unsafe fn store256_bytes_at(x: &mut [u8; 64], offset: usize, v: __m256i) + requires + offset <= 32, + ensures + forall|k: int| + 0 <= k < 64 ==> #[trigger] final(x)[k] == if offset <= k < offset + 32 { + m256_bytes(v)[k - offset] + } else { + old(x)[k] + }, +{ + unsafe { _mm256_storeu_si256(x.as_mut_ptr().add(offset).cast(), v) } +} + +/// `_mm512_loadu_si512` of a 64-byte array. +/// +/// # Safety +/// +/// Requires the `avx512f` target feature. +#[verifier::external_body] +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn load512_bytes(x: &[u8; 64]) -> (r: __m512i) + ensures + m512_bytes(r) == *x, +{ + unsafe { _mm512_loadu_si512(x.as_ptr().cast()) } +} + +/// `_mm512_storeu_si512` into a 64-byte array. +/// +/// # Safety +/// +/// Requires the `avx512f` target feature. +#[verifier::external_body] +#[inline] +#[target_feature(enable = "avx512f")] +pub unsafe fn store512_bytes(x: &mut [u8; 64], v: __m512i) + ensures + *final(x) == m512_bytes(v), +{ + unsafe { _mm512_storeu_si512(x.as_mut_ptr().cast(), v) } +} + +// --------------------------------------------------------------------------------------------- +// Executable twins of the lane semantics, for the differential tests +// --------------------------------------------------------------------------------------------- +/// [`unpacklo_epi8_lane`]. +pub fn model_unpacklo_epi8_lane(a: &[u8], b: &[u8], k: usize) -> (r: u8) + requires + a.len() == b.len(), + a.len() % 16 == 0, + k < a.len(), + ensures + r == unpacklo_epi8_lane(a@, b@, k as int), +{ + let j = 16 * (k / 16) + (k % 16) / 2; + if k % 2 == 0 { + a[j] + } else { + b[j] + } +} + +/// [`unpackhi_epi8_lane`]. +pub fn model_unpackhi_epi8_lane(a: &[u8], b: &[u8], k: usize) -> (r: u8) + requires + a.len() == b.len(), + a.len() % 16 == 0, + k < a.len(), + ensures + r == unpackhi_epi8_lane(a@, b@, k as int), +{ + let j = 16 * (k / 16) + 8 + (k % 16) / 2; + if k % 2 == 0 { + a[j] + } else { + b[j] + } +} + +/// [`permutexvar_epi8_lane`]. +pub fn model_permutexvar_epi8_lane(idx: &[u8], a: &[u8], k: usize) -> (r: u8) + requires + idx.len() == 64, + a.len() == 64, + k < 64, + ensures + r == permutexvar_epi8_lane(idx@, a@, k as int), +{ + let x = idx[k]; + assert(x & 63 < 64) by (bit_vector); + a[(x & 63) as usize] +} + +/// [`parity8`]. +pub fn model_parity8(x: u8) -> (r: u8) + ensures + r == parity8(x), +{ + let x = x ^ (x >> 4u8); + let x = x ^ (x >> 2u8); + (x ^ (x >> 1u8)) & 1 +} + +/// [`affine_bit`]. +pub fn model_affine_bit(m: u8, x: u8, imm8: u8, i: u8) -> (r: u8) + requires + i < 8, + ensures + r == affine_bit(m, x, imm8, i), +{ + model_parity8(m & x) ^ ((imm8 >> i) & 1) +} + +/// [`affine_byte`]: result bit `i` from matrix row `7 - i`. +pub fn model_affine_byte(a: u64, x: u8, imm8: u8) -> (r: u8) + ensures + r == affine_byte(a, x, imm8), +{ + model_affine_bit((a >> 56u64) as u8, x, imm8, 0) | (model_affine_bit((a >> 48u64) as u8, x, imm8, 1) << 1u8) + | (model_affine_bit((a >> 40u64) as u8, x, imm8, 2) << 2u8) | (model_affine_bit((a >> 32u64) as u8, x, imm8, 3) + << 3u8) | (model_affine_bit((a >> 24u64) as u8, x, imm8, 4) << 4u8) | (model_affine_bit( + (a >> 16u64) as u8, + x, + imm8, + 5, + ) << 5u8) | (model_affine_bit((a >> 8u64) as u8, x, imm8, 6) << 6u8) | (model_affine_bit(a as u8, x, imm8, 7) + << 7u8) +} + +/// [`gf2p8affine_lane`]. +pub fn model_gf2p8affine_lane(x: &[u8], a: &[u64], imm8: u8, k: usize) -> (r: u8) + requires + x.len() == 8 * a.len(), + k < x.len(), + ensures + r == gf2p8affine_lane(x@, a@, imm8, k as int), +{ + model_affine_byte(a[k / 8], x[k], imm8) +} + +} // verus! diff --git a/verification/verus/src/intrinsics/x86_gfneon.rs b/verification/verus/src/intrinsics/x86_gfneon.rs new file mode 100644 index 0000000000..48149c1ad2 --- /dev/null +++ b/verification/verus/src/intrinsics/x86_gfneon.rs @@ -0,0 +1,118 @@ +//! Specifications of the AVX2 byte intrinsics `gf2_8::avx2::gf8_mul_vec32` calls, beyond the ones in [`super::x86`]. +//! +//! Same conventions: a register is viewed as its bytes ([`m256_bytes`]) or its words ([`m256`]); the signed +//! byte compare is given by [`cmpgt_epi8_lane`], with an executable twin that `tests/equivalence/ +//! intrinsics_x86_gfneon.rs` runs against the hardware. +#[cfg(verus_keep_ghost)] +use super::x86::{axiom_m256_bytes, m256, m256_bytes}; +use core::arch::x86_64::*; +use vstd::prelude::*; + +verus! { + +/// Byte `k` of `_mm256_cmpgt_epi8`: all ones where `a > b` as signed bytes, else zero. +pub open spec fn cmpgt_epi8_lane(a: u8, b: u8) -> u8 { + if (a as i8) > (b as i8) { + 0xFF + } else { + 0 + } +} + +pub assume_specification[ _mm256_setzero_si256 ]() -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == 0, +; + +pub assume_specification[ _mm256_set1_epi8 ](a: i8) -> (r: __m256i) + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == a as u8, +; + +/// Byte-wise wrapping addition. +pub assume_specification[ _mm256_add_epi8 ](a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == (m256_bytes(a)[k] + m256_bytes(b)[k]) as u8, +; + +/// Byte-wise signed compare. +pub assume_specification[ _mm256_cmpgt_epi8 ](a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == cmpgt_epi8_lane(m256_bytes(a)[k], m256_bytes(b)[k]), +; + +/// [`cmpgt_epi8_lane`]. +pub fn model_cmpgt_epi8_lane(a: u8, b: u8) -> (r: u8) + ensures + r == cmpgt_epi8_lane(a, b), +{ + if (a as i8) > (b as i8) { + 0xFF + } else { + 0 + } +} + +// --------------------------------------------------------------------------------------------- +// The word-wise operations, read bytewise +// --------------------------------------------------------------------------------------------- +/// The bytes of a word-wise AND are the ANDs of the bytes. +pub proof fn lemma_m256_and_bytes(a: __m256i, b: __m256i, r: __m256i) + requires + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == m256(a)[i] & m256(b)[i], + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == m256_bytes(a)[k] & m256_bytes(b)[k], +{ + broadcast use axiom_m256_bytes; + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(r)[k] == m256_bytes(a)[k] & m256_bytes(b)[k] by { + let (x, y) = (m256(a)[k / 8], m256(b)[k / 8]); + let s = (8 * (k % 8)) as u64; + assert(m256(r)[k / 8] == x & y); + assert(s < 64 ==> ((x & y) >> s) as u8 == ((x >> s) as u8) & ((y >> s) as u8)) by (bit_vector); + } +} + +/// The bytes of a word-wise XOR are the XORs of the bytes. +pub proof fn lemma_m256_xor_bytes(a: __m256i, b: __m256i, r: __m256i) + requires + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == m256(a)[i] ^ m256(b)[i], + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == m256_bytes(a)[k] ^ m256_bytes(b)[k], +{ + broadcast use axiom_m256_bytes; + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(r)[k] == m256_bytes(a)[k] ^ m256_bytes(b)[k] by { + let (x, y) = (m256(a)[k / 8], m256(b)[k / 8]); + let s = (8 * (k % 8)) as u64; + assert(m256(r)[k / 8] == x ^ y); + assert(s < 64 ==> ((x ^ y) >> s) as u8 == ((x >> s) as u8) ^ ((y >> s) as u8)) by (bit_vector); + } +} + +/// The bytes of the zero register are zero. +pub proof fn lemma_m256_zero_bytes(r: __m256i) + requires + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == 0, + ensures + forall|k: int| 0 <= k < 32 ==> #[trigger] m256_bytes(r)[k] == 0, +{ + broadcast use axiom_m256_bytes; + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(r)[k] == 0 by { + let s = (8 * (k % 8)) as u64; + assert(m256(r)[k / 8] == 0); + assert((0u64 >> s) as u8 == 0) by (bit_vector); + } +} + +/// A signed compare of zero against a byte is its top bit, spread over the byte. +pub proof fn lemma_cmpgt_zero(x: u8) + ensures + cmpgt_epi8_lane(0, x) == if x >= 128 { + 0xFFu8 + } else { + 0u8 + }, +{ + assert((0u8 as i8) > (x as i8) <==> x >= 128) by (bit_vector); +} + +} // verus! diff --git a/verification/verus/src/intrinsics/x86_gfx86.rs b/verification/verus/src/intrinsics/x86_gfx86.rs new file mode 100644 index 0000000000..61883c8153 --- /dev/null +++ b/verification/verus/src/intrinsics/x86_gfx86.rs @@ -0,0 +1,304 @@ +//! Specifications of the x86-64 intrinsics the GF(2^192) kernels (`gf2_64x3::x86_64`) add to [`super::x86`]: +//! 128-bit shifts and shuffles, the AVX2 lane moves and blends, the AVX-512 casts and extracts, and the all-zero +//! value of the accumulator arrays. +//! +//! As in [`super::x86`], each intrinsic's result is given word by word by an open spec function `_lane` +//! over the [`m128`], [`m256`], [`m512`] views (and [`m256d`] for the one `pd` move), and each non-trivial one has +//! an executable twin `model_` that `tests/equivalence/intrinsics_x86_gfx86.rs` runs against the hardware. +#[cfg(verus_keep_ghost)] +use super::transmuted; +#[cfg(verus_keep_ghost)] +use super::x86::{m128, m256, m512}; +use core::arch::x86_64::*; +use vstd::prelude::*; + +verus! { + +#[verifier::external_type_specification] +#[verifier::external_body] +pub struct ExM256d(__m256d); + +// --------------------------------------------------------------------------------------------- +// Views and layout +// --------------------------------------------------------------------------------------------- +/// The four 64-bit words of a 256-bit `pd` register, its bits as they are (no float reading), the low one first. +pub open spec fn m256d(v: __m256d) -> [u64; 4] { + transmuted::<__m256d, [u64; 4]>(v) +} + +/// The value `core::mem::zeroed::()` returns. +/// +/// Uninterpreted: the layout axioms below say what it is for the accumulator types the kernels zero. +pub uninterp spec fn zeroed_value() -> T; + +/// `core::mem::zeroed` returns [`zeroed_value`]. +pub assume_specification[ core::mem::zeroed:: ]() -> (r: T) + ensures + r == zeroed_value::(), +; + +/// Layout: an all-zero array of 512-bit registers has zero words. +pub axiom fn axiom_zeroed_m512() + ensures + forall|i: int, j: int| 0 <= i < N && 0 <= j < 8 ==> #[trigger] m512(zeroed_value::<[__m512i; N]>()[i])[j] == 0, +; + +/// Layout: an all-zero pair of arrays of 256-bit registers has zero words. +pub axiom fn axiom_zeroed_m256_pairs() + ensures + forall|h: int, i: int, j: int| + 0 <= h < 2 && 0 <= i < N && 0 <= j < 4 ==> #[trigger] m256(zeroed_value::<[[__m256i; N]; 2]>()[h][i])[j] + == 0, +; + +// --------------------------------------------------------------------------------------------- +// Lane semantics +// --------------------------------------------------------------------------------------------- +/// Doubleword `k` of a sequence of words, little-endian, in the low half of a word. +pub open spec fn dword(a: Seq, k: int) -> u64 { + (a[k / 2] >> ((32 * (k % 2)) as u64)) & 0xFFFF_FFFFu64 +} + +/// The two-bit field `k` of an immediate. +pub open spec fn imm2(imm8: u8, k: int) -> int { + ((imm8 >> ((2 * k) as u8)) & 3) as int +} + +/// Word `i` of `_mm*_shuffle_epi32`: doubleword `j` of each 128-bit lane is doubleword `imm8[2j + 1 : 2j]` of the +/// same lane of `a`. +pub open spec fn shuffle_epi32_lane(a: Seq, imm8: u8, i: int) -> u64 { + let l = 4 * (i / 2); + dword(a, l + imm2(imm8, 2 * (i % 2))) | (dword(a, l + imm2(imm8, 2 * (i % 2) + 1)) << 32u64) +} + +/// Word `i` of `_mm256_blend_epi32`: doubleword `k` is that of `b` if bit `k` of the immediate is set, else that +/// of `a`. +pub open spec fn blend_epi32_lane(a: Seq, b: Seq, imm8: u8, i: int) -> u64 { + let lo = if (imm8 >> ((2 * i) as u8)) & 1 == 1 { + b[i] + } else { + a[i] + }; + let hi = if (imm8 >> ((2 * i + 1) as u8)) & 1 == 1 { + b[i] + } else { + a[i] + }; + (lo & 0xFFFF_FFFFu64) | (hi & 0xFFFF_FFFF_0000_0000u64) +} + +/// Word `i` of `_mm256_permute2x128_si256`: 128-bit half `h` is zero if bit `4h + 3` of the immediate is set, +/// else half `imm8[4h + 1 : 4h]` of the concatenation `a`, `b`. +pub open spec fn permute2x128_lane(a: Seq, b: Seq, imm8: u8, i: int) -> u64 { + let ctl = (imm8 >> ((4 * (i / 2)) as u8)) & 15; + let s = (ctl & 3) as int; + if ctl & 8 != 0 { + 0 + } else if s < 2 { + a[2 * s + i % 2] + } else { + b[2 * (s - 2) + i % 2] + } +} + +/// Word `i` of `_mm256_permutevar_pd`: word `bit 1 of b[i]` of the 128-bit lane of `a` that word `i` is in. +pub open spec fn permutevar_pd_lane(a: Seq, b: Seq, i: int) -> u64 { + a[2 * (i / 2) + ((b[i] >> 1u64) & 1) as int] +} + +// --------------------------------------------------------------------------------------------- +// The specifications +// --------------------------------------------------------------------------------------------- +pub assume_specification[ _mm_setzero_si128 ]() -> (r: __m128i) + ensures + m128(r)[0] == 0, + m128(r)[1] == 0, +; + +pub assume_specification[ _mm_set1_epi64x ](a: i64) -> (r: __m128i) + ensures + m128(r)[0] == a as u64, + m128(r)[1] == a as u64, +; + +pub assume_specification[ _mm_slli_epi64:: ](a: __m128i) -> (r: __m128i) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] m128(r)[i] == super::x86::shl_imm(m128(a)[i], IMM8 as int), +; + +pub assume_specification[ _mm_srli_epi64:: ](a: __m128i) -> (r: __m128i) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] m128(r)[i] == super::x86::shr_imm(m128(a)[i], IMM8 as int), +; + +pub assume_specification[ _mm_shuffle_epi32:: ](a: __m128i) -> (r: __m128i) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] m128(r)[i] == shuffle_epi32_lane(m128(a)@, IMM8 as u8, i), +; + +pub assume_specification[ _mm256_shuffle_epi32:: ](a: __m256i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == shuffle_epi32_lane(m256(a)@, MASK as u8, i), +; + +pub assume_specification[ _mm512_shuffle_epi32:: ](a: __m512i) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == shuffle_epi32_lane(m512(a)@, MASK as u8, i), +; + +pub assume_specification[ _mm256_blend_epi32:: ](a: __m256i, b: __m256i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == blend_epi32_lane(m256(a)@, m256(b)@, IMM8 as u8, i), +; + +pub assume_specification[ _mm256_permute2x128_si256:: ](a: __m256i, b: __m256i) -> (r: + __m256i) + ensures + forall|i: int| + 0 <= i < 4 ==> #[trigger] m256(r)[i] == permute2x128_lane(m256(a)@, m256(b)@, IMM8 as u8, i), +; + +pub assume_specification[ _mm256_permutevar_pd ](a: __m256d, b: __m256i) -> (r: __m256d) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256d(r)[i] == permutevar_pd_lane(m256d(a)@, m256(b)@, i), +; + +pub assume_specification[ _mm256_castpd_si256 ](a: __m256d) -> (r: __m256i) + ensures + m256(r) == m256d(a), +; + +pub assume_specification[ _mm256_castsi256_si128 ](a: __m256i) -> (r: __m128i) + ensures + m128(r)[0] == m256(a)[0], + m128(r)[1] == m256(a)[1], +; + +pub assume_specification[ _mm256_extracti128_si256:: ](a: __m256i) -> (r: __m128i) + ensures + forall|i: int| 0 <= i < 2 ==> #[trigger] m128(r)[i] == m256(a)[2 * (IMM1 as u8 & 1) as int + i], +; + +pub assume_specification[ _mm512_setzero_si512 ]() -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == 0, +; + +pub assume_specification[ _mm512_broadcast_i32x4 ](a: __m128i) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 8 ==> #[trigger] m512(r)[i] == m128(a)[i % 2], +; + +pub assume_specification[ _mm512_castsi512_si256 ](a: __m512i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == m512(a)[i], +; + +/// The upper four words are undefined: nothing is said of them. +pub assume_specification[ _mm512_castsi256_si512 ](a: __m256i) -> (r: __m512i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m512(r)[i] == m256(a)[i], +; + +pub assume_specification[ _mm512_extracti64x4_epi64:: ](a: __m512i) -> (r: __m256i) + ensures + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(r)[i] == m512(a)[4 * (IMM1 as u8 & 1) as int + i], +; + +// --------------------------------------------------------------------------------------------- +// Executable twins of the lane semantics, for the differential tests +// --------------------------------------------------------------------------------------------- +/// [`dword`]. +pub fn model_dword(a: &[u64], k: usize) -> (r: u64) + requires + k < 2 * a.len(), + ensures + r == dword(a@, k as int), +{ + (a[k / 2] >> ((32 * (k % 2)) as u64)) & 0xFFFF_FFFF +} + +/// [`imm2`]. +pub fn model_imm2(imm8: u8, k: usize) -> (r: usize) + requires + k < 4, + ensures + r == imm2(imm8, k as int), + r < 4, +{ + let s = (2 * k) as u8; + assert((imm8 >> s) & 3 <= 3) by (bit_vector); + ((imm8 >> s) & 3) as usize +} + +/// [`shuffle_epi32_lane`]. +pub fn model_shuffle_epi32_lane(a: &[u64], imm8: u8, i: usize) -> (r: u64) + requires + a.len() % 2 == 0, + a.len() <= 8, + i < a.len(), + ensures + r == shuffle_epi32_lane(a@, imm8, i as int), +{ + let l = 4 * (i / 2); + model_dword(a, l + model_imm2(imm8, 2 * (i % 2))) | (model_dword(a, l + model_imm2(imm8, 2 * (i % 2) + 1)) << 32u64) +} + +/// [`blend_epi32_lane`]. +pub fn model_blend_epi32_lane(a: &[u64], b: &[u64], imm8: u8, i: usize) -> (r: u64) + requires + a.len() == 4, + b.len() == 4, + i < 4, + ensures + r == blend_epi32_lane(a@, b@, imm8, i as int), +{ + let lo = if (imm8 >> ((2 * i) as u8)) & 1 == 1 { + b[i] + } else { + a[i] + }; + let hi = if (imm8 >> ((2 * i + 1) as u8)) & 1 == 1 { + b[i] + } else { + a[i] + }; + (lo & 0xFFFF_FFFF) | (hi & 0xFFFF_FFFF_0000_0000) +} + +/// [`permute2x128_lane`]. +pub fn model_permute2x128_lane(a: &[u64], b: &[u64], imm8: u8, i: usize) -> (r: u64) + requires + a.len() == 4, + b.len() == 4, + i < 4, + ensures + r == permute2x128_lane(a@, b@, imm8, i as int), +{ + let ctl = (imm8 >> ((4 * (i / 2)) as u8)) & 15; + assert(ctl & 3 <= 3) by (bit_vector); + let s = (ctl & 3) as usize; + if ctl & 8 != 0 { + 0 + } else if s < 2 { + a[2 * s + i % 2] + } else { + b[2 * (s - 2) + i % 2] + } +} + +/// [`permutevar_pd_lane`]. +pub fn model_permutevar_pd_lane(a: &[u64], b: &[u64], i: usize) -> (r: u64) + requires + a.len() == 4, + b.len() == 4, + i < 4, + ensures + r == permutevar_pd_lane(a@, b@, i as int), +{ + let x = b[i]; + assert((x >> 1u64) & 1 <= 1) by (bit_vector); + a[2 * (i / 2) + ((x >> 1u64) & 1) as usize] +} + +} // verus! diff --git a/verification/verus/src/intrinsics/x86_nttsimd.rs b/verification/verus/src/intrinsics/x86_nttsimd.rs new file mode 100644 index 0000000000..871fa81294 --- /dev/null +++ b/verification/verus/src/intrinsics/x86_nttsimd.rs @@ -0,0 +1,96 @@ +//! The memory helpers of the x86-64 NTT butterfly kernels (`ntt_simd`): their loads and stores, with trusted +//! specifications. +//! +//! Production's kernels load and store through raw pointers into the two rows. Verus cannot tie a pointer to +//! the slice it came from, so the verified copies take the rows as slices and an offset, and call these helpers, +//! whose bodies are production's load or store at that offset. `tests/equivalence/intrinsics_x86_nttsimd.rs` +//! checks each against the memory it reads or writes. They are `unsafe fn`, as the loads and stores they wrap; their +//! `requires` is their safety condition. +#[cfg(verus_keep_ghost)] +use super::x86::{m128_bytes, m256, m512}; +use crate::gf2_64::F64; +use core::arch::x86_64::*; +use vstd::prelude::*; + +verus! { + +/// `_mm512_loadu_si512` of the eight words of `row` from `at`. +#[cfg(target_feature = "avx512f")] +#[verifier::external_body] +#[inline(always)] +pub unsafe fn loadu512_at(row: &[F64], at: usize) -> (r: __m512i) + requires + at + 8 <= row.len(), + ensures + forall|j: int| 0 <= j < 8 ==> #[trigger] m512(r)[j] == row@[at + j].0, +{ + // SAFETY: `at + 8 <= row.len()`, and `F64` is a transparent `u64`. + unsafe { _mm512_loadu_si512(row.as_ptr().add(at).cast()) } +} + +/// `_mm512_storeu_si512` of `v` to the eight words of `row` from `at`. +#[cfg(target_feature = "avx512f")] +#[verifier::external_body] +#[inline(always)] +pub unsafe fn storeu512_at(row: &mut [F64], at: usize, v: __m512i) + requires + at + 8 <= old(row).len(), + ensures + final(row).len() == old(row).len(), + forall|j: int| + 0 <= j < old(row).len() ==> #[trigger] final(row)@[j] == if at <= j < at + 8 { + F64(m512(v)[j - at]) + } else { + old(row)@[j] + }, +{ + // SAFETY: `at + 8 <= row.len()`, and `F64` is a transparent `u64`. + unsafe { _mm512_storeu_si512(row.as_mut_ptr().add(at).cast(), v) } +} + +/// `_mm256_loadu_si256` of the four words of `row` from `at`. +#[cfg(target_feature = "avx")] +#[verifier::external_body] +#[inline(always)] +pub unsafe fn loadu256_at(row: &[F64], at: usize) -> (r: __m256i) + requires + at + 4 <= row.len(), + ensures + forall|j: int| 0 <= j < 4 ==> #[trigger] m256(r)[j] == row@[at + j].0, +{ + // SAFETY: `at + 4 <= row.len()`, and `F64` is a transparent `u64`. + unsafe { _mm256_loadu_si256(row.as_ptr().add(at).cast()) } +} + +/// `_mm256_storeu_si256` of `v` to the four words of `row` from `at`. +#[cfg(target_feature = "avx")] +#[verifier::external_body] +#[inline(always)] +pub unsafe fn storeu256_at(row: &mut [F64], at: usize, v: __m256i) + requires + at + 4 <= old(row).len(), + ensures + final(row).len() == old(row).len(), + forall|j: int| + 0 <= j < old(row).len() ==> #[trigger] final(row)@[j] == if at <= j < at + 4 { + F64(m256(v)[j - at]) + } else { + old(row)@[j] + }, +{ + // SAFETY: `at + 4 <= row.len()`, and `F64` is a transparent `u64`. + unsafe { _mm256_storeu_si256(row.as_mut_ptr().add(at).cast(), v) } +} + +/// `_mm_loadu_si128` of 16 bytes. +#[verifier::external_body] +#[inline(always)] +pub unsafe fn loadu128_bytes(x: &[u8; 16]) -> (r: __m128i) + ensures + m128_bytes(r) == *x, +{ + // SAFETY: `x` is 16 readable bytes; SSE2 is part of the x86-64 baseline. + unsafe { _mm_loadu_si128(x.as_ptr().cast()) } +} + +} // verus! diff --git a/verification/verus/src/lib.rs b/verification/verus/src/lib.rs new file mode 100644 index 0000000000..2750886345 --- /dev/null +++ b/verification/verus/src/lib.rs @@ -0,0 +1,24 @@ +//! Verus proofs of annotated leanVM arithmetic and kernel copies, with explicit intrinsic trust contracts. +#![allow(unused_parens, unused_imports, unused_variables, dead_code, clippy::all)] + +pub mod bit_fold; +pub mod bits; +pub mod blake2s; +pub mod blake2s_batch; +pub mod clmul; +pub mod fiat_shamir; +pub mod flock_ntt; +pub mod gf2_64; +pub mod gf2_64x3; +pub mod gf2_8; +pub mod intrinsics; +pub mod multilinear; +#[cfg(target_arch = "aarch64")] +pub mod neon; +pub mod ntt; +pub mod ntt_driver; +pub mod ntt_lanes; +pub mod ntt_simd; +pub mod parallel; +pub mod phi8_tower; +pub mod skip_domain; diff --git a/verification/verus/src/multilinear.rs b/verification/verus/src/multilinear.rs new file mode 100644 index 0000000000..4f84130304 --- /dev/null +++ b/verification/verus/src/multilinear.rs @@ -0,0 +1,1914 @@ +//! The equality polynomial and its tables over the Boolean cube. +//! +//! The executable functions are the portable paths of `crates/primitives/src/multilinear.rs`, copied with the same +//! bodies where Verus accepts them; `tests/equivalence/multilinear.rs` checks the two agree. Where Verus rejects the +//! production form (iterator chains and `fold`, `as_chunks_mut`, `split_at_mut`, `MaybeUninit` and `set_len`, +//! `[x; N]`, the parallel pass), the copy spells out the same arithmetic in `while`/`for` loops over indices; each +//! such spot says what it replaces. +//! +//! Specification: over `E` ([`e_mul`], [`e_add`]; `1 - a = 1 + a` in characteristic 2), +//! +//! ```text +//! eq(r, x) = prod_i (r_i x_i + (1 + r_i)(1 + x_i)) ([`eq_poly`], [`eq_factor`]) +//! ``` +//! +//! and a table over `n` variables holds at index `x` the value at the cube point whose coordinate `i` is bit `i` +//! of `x` (LSB first, [`cube_point`], [`eq_at`]). +use crate::gf2_64::*; +use crate::gf2_64x3::*; +use crate::phi8_tower::*; +use vstd::arithmetic::div_mod::*; +use vstd::arithmetic::power2::*; +use vstd::bits::*; +use vstd::prelude::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// One coordinate's factor of `eq`: `r x + (1 + r)(1 + x)`. +pub open spec fn eq_factor(r: F192, x: F192) -> F192 { + e_add(e_mul(r, x), e_mul(e_add(F192::ONE, r), e_add(F192::ONE, x))) +} + +/// `eq(r, x) = prod_i eq_factor(r_i, x_i)`, the product taken from the first coordinate up. +pub open spec fn eq_poly(r: Seq, x: Seq) -> F192 + decreases r.len(), +{ + if r.len() == 0 { + F192::ONE + } else { + e_mul(eq_poly(r.drop_last(), x.drop_last()), eq_factor(r.last(), x.last())) + } +} + +/// Bit `i` of an index. +pub open spec fn bit(x: usize, i: int) -> bool { + (x >> (i as usize)) & 1 == 1 +} + +/// A bit as an element of `E`. +pub open spec fn e_bool(b: bool) -> F192 { + if b { + F192::ONE + } else { + F192::ZERO + } +} + +/// The point of the cube `{0, 1}^n` at index `x`: coordinate `i` is bit `i` of `x`. +pub open spec fn cube_point(n: nat, x: usize) -> Seq { + Seq::new(n, |i: int| e_bool(bit(x, i))) +} + +/// `eq(r, x)` at the cube point of index `x`. +pub open spec fn eq_at(r: Seq, x: usize) -> F192 { + eq_poly(r, cube_point(r.len(), x)) +} + +/// `t` is `seed * eq(r, .)`: `2^n` entries, entry `x` being `seed * eq(r, x)`. +pub open spec fn is_eq_table(t: Seq, r: Seq, seed: F192) -> bool { + &&& r.len() < 64 + &&& t.len() == (1usize << r.len()) + &&& forall|x: int| 0 <= x < t.len() ==> #[trigger] t[x] == e_mul(seed, eq_at(r, x as usize)) +} + +// --------------------------------------------------------------------------------------------- +// Algebra of E used below +// --------------------------------------------------------------------------------------------- +proof fn lemma_xor_ring(a: u64, b: u64, c: u64) + ensures + a ^ b == b ^ a, + (a ^ b) ^ c == a ^ (b ^ c), + a ^ 0 == a, + 0 ^ a == a, + a ^ a == 0, + a ^ (b ^ (c ^ a)) == b ^ c, +{ + assert(a ^ b == b ^ a && (a ^ b) ^ c == a ^ (b ^ c) && a ^ 0 == a && 0 ^ a == a && a ^ a == 0 && a ^ (b ^ (c + ^ a)) == b ^ c) by (bit_vector); +} + +/// `E` under `+`: commutative, associative, zero, every element its own negative. +pub proof fn lemma_e_add(a: F192, b: F192, c: F192) + ensures + e_add(a, b) == e_add(b, a), + e_add(e_add(a, b), c) == e_add(a, e_add(b, c)), + e_add(a, F192::ZERO) == a, + e_add(F192::ZERO, a) == a, + e_add(a, a) == F192::ZERO, + e_add(a, e_add(b, e_add(c, a))) == e_add(b, c), +{ + lemma_xor_ring(a.c0, b.c0, c.c0); + lemma_xor_ring(a.c1, b.c1, c.c1); + lemma_xor_ring(a.c2, b.c2, c.c2); +} + +pub proof fn lemma_e_mul_zero(a: F192) + ensures + e_mul(a, F192::ZERO) == F192::ZERO, + e_mul(F192::ZERO, a) == F192::ZERO, +{ + lemma_k_mul_zero(a.c0); + lemma_k_mul_zero(a.c1); + lemma_k_mul_zero(a.c2); + assert(0u64 ^ (0u64 ^ 0u64) == 0u64 && (0u64 ^ 0u64) ^ (0u64 ^ 0u64) ^ 0u64 == 0u64 && (0u64 ^ 0u64 ^ 0u64) + ^ 0u64 == 0u64) by (bit_vector); +} + +proof fn lemma_xor_distrib( + b00: u64, b01: u64, b02: u64, b10: u64, b11: u64, b12: u64, b20: u64, b21: u64, b22: u64, + c00: u64, c01: u64, c02: u64, c10: u64, c11: u64, c12: u64, c20: u64, c21: u64, c22: u64, +) + ensures + (b00 ^ c00) ^ ((b12 ^ c12) ^ (b21 ^ c21)) == (b00 ^ (b12 ^ b21)) ^ (c00 ^ (c12 ^ c21)), + ((b01 ^ c01) ^ (b10 ^ c10)) ^ ((b12 ^ c12) ^ (b21 ^ c21)) ^ (b22 ^ c22) == ((b01 ^ b10) ^ (b12 ^ b21) ^ b22) + ^ ((c01 ^ c10) ^ (c12 ^ c21) ^ c22), + ((b02 ^ c02) ^ (b11 ^ c11) ^ (b20 ^ c20)) ^ (b22 ^ c22) == ((b02 ^ b11 ^ b20) ^ b22) ^ ((c02 ^ c11 ^ c20) + ^ c22), +{ + assert((b00 ^ c00) ^ ((b12 ^ c12) ^ (b21 ^ c21)) == (b00 ^ (b12 ^ b21)) ^ (c00 ^ (c12 ^ c21))) by (bit_vector); + assert(((b01 ^ c01) ^ (b10 ^ c10)) ^ ((b12 ^ c12) ^ (b21 ^ c21)) ^ (b22 ^ c22) == ((b01 ^ b10) ^ (b12 ^ b21) + ^ b22) ^ ((c01 ^ c10) ^ (c12 ^ c21) ^ c22)) by (bit_vector); + assert(((b02 ^ c02) ^ (b11 ^ c11) ^ (b20 ^ c20)) ^ (b22 ^ c22) == ((b02 ^ b11 ^ b20) ^ b22) ^ ((c02 ^ c11 + ^ c20) ^ c22)) by (bit_vector); +} + +/// The product distributes over the sum. +pub proof fn lemma_e_mul_distrib(a: F192, b: F192, c: F192) + ensures + e_mul(a, e_add(b, c)) == e_add(e_mul(a, b), e_mul(a, c)), + e_mul(e_add(b, c), a) == e_add(e_mul(b, a), e_mul(c, a)), +{ + let s = e_add(b, c); + lemma_k_mul_xor_right(a.c0, b.c0, c.c0); + lemma_k_mul_xor_right(a.c0, b.c1, c.c1); + lemma_k_mul_xor_right(a.c0, b.c2, c.c2); + lemma_k_mul_xor_right(a.c1, b.c0, c.c0); + lemma_k_mul_xor_right(a.c1, b.c1, c.c1); + lemma_k_mul_xor_right(a.c1, b.c2, c.c2); + lemma_k_mul_xor_right(a.c2, b.c0, c.c0); + lemma_k_mul_xor_right(a.c2, b.c1, c.c1); + lemma_k_mul_xor_right(a.c2, b.c2, c.c2); + lemma_xor_distrib( + k_mul(a.c0, b.c0), k_mul(a.c0, b.c1), k_mul(a.c0, b.c2), + k_mul(a.c1, b.c0), k_mul(a.c1, b.c1), k_mul(a.c1, b.c2), + k_mul(a.c2, b.c0), k_mul(a.c2, b.c1), k_mul(a.c2, b.c2), + k_mul(a.c0, c.c0), k_mul(a.c0, c.c1), k_mul(a.c0, c.c2), + k_mul(a.c1, c.c0), k_mul(a.c1, c.c1), k_mul(a.c1, c.c2), + k_mul(a.c2, c.c0), k_mul(a.c2, c.c1), k_mul(a.c2, c.c2), + ); + assert(e_mul(a, s) == e_add(e_mul(a, b), e_mul(a, c))); + lemma_e_mul_comm(a, s); + lemma_e_mul_comm(a, b); + lemma_e_mul_comm(a, c); +} + +/// `v + v r = v (1 + r)`: the doubling's low child. +proof fn lemma_times_one_plus(v: F192, r: F192) + ensures + e_add(v, e_mul(v, r)) == e_mul(v, e_add(F192::ONE, r)), +{ + lemma_e_mul_distrib(v, F192::ONE, r); + lemma_e_mul_one(v); +} + +// --------------------------------------------------------------------------------------------- +// The equality polynomial +// --------------------------------------------------------------------------------------------- +/// At a cube coordinate the factor selects: `eq_factor(r, 1) = r`, `eq_factor(r, 0) = 1 + r`. +pub proof fn lemma_eq_factor_bits(r: F192) + ensures + eq_factor(r, F192::ONE) == r, + eq_factor(r, F192::ZERO) == e_add(F192::ONE, r), +{ + let s = e_add(F192::ONE, r); + assert(1u64 ^ 1u64 == 0u64 && 1u64 ^ 0u64 == 1u64 && 0u64 ^ 0u64 == 0u64) by (bit_vector); + assert(e_add(F192::ONE, F192::ONE) == F192::ZERO); + assert(e_add(F192::ONE, F192::ZERO) == F192::ONE); + lemma_e_mul_one(r); + lemma_e_mul_one(s); + lemma_e_mul_zero(r); + lemma_e_mul_zero(s); + lemma_e_add(r, r, r); + lemma_e_add(s, s, s); +} + +/// In characteristic 2 the factor is `1 + r + x` for every `x`, the form `eq_eval` computes. +pub proof fn lemma_eq_factor_sum(r: F192, x: F192) + ensures + eq_factor(r, x) == e_add(e_add(F192::ONE, r), x), +{ + let (s, t, u) = (e_add(F192::ONE, r), e_add(F192::ONE, x), e_mul(r, x)); + // (1 + r)(1 + x) = (1 + r) + (x + r x). + lemma_e_mul_distrib(s, F192::ONE, x); + lemma_e_mul_one(s); + lemma_e_mul_distrib(x, F192::ONE, r); + lemma_e_mul_one(x); + lemma_e_mul_comm(s, x); + lemma_e_mul_comm(x, r); + assert(e_mul(s, t) == e_add(s, e_add(x, u))); + lemma_e_add(u, s, x); + lemma_e_add(s, x, u); + lemma_e_add(x, u, s); + assert(e_add(u, e_add(s, e_add(x, u))) == e_add(s, x)); +} + +/// A table entry extended by one top variable: the low half picks `1 + r_i`, the high half `r_i`. +pub proof fn lemma_eq_at_high(r: Seq, j: usize) + requires + 1 <= r.len() <= 63, + j < (1usize << ((r.len() - 1) as usize)), + ensures + eq_at(r, j) == e_mul(eq_at(r.drop_last(), j), e_add(F192::ONE, r.last())), + eq_at(r, (j + (1usize << ((r.len() - 1) as usize))) as usize) == e_mul(eq_at(r.drop_last(), j), r.last()), +{ + let i = (r.len() - 1) as usize; + let j2 = (j + (1usize << i)) as usize; + lemma_bit_add_high(j, i); + let (p, p2) = (cube_point(r.len(), j), cube_point(r.len(), j2)); + assert(p.drop_last() =~= cube_point(i as nat, j)); + assert forall|k: int| 0 <= k < i implies #[trigger] bit(j2, k) == bit(j, k) by { + lemma_bit_add_low(j, i, k as usize); + } + assert(p2.drop_last() =~= cube_point(i as nat, j)); + assert(p.last() == F192::ZERO); + assert(p2.last() == F192::ONE); + lemma_eq_factor_bits(r.last()); +} + +proof fn lemma_bit_add_high(j: usize, i: usize) + requires + i < 63, + j < (1usize << i), + ensures + (j >> i) & 1 == 0, + ((j + (1usize << i)) as usize >> i) & 1 == 1, +{ + assert((j >> i) & 1 == 0 && ((j + (1usize << i)) as usize >> i) & 1 == 1) by (bit_vector) + requires + i < 63, + j < (1usize << i), + ; +} + +proof fn lemma_bit_add_low(j: usize, i: usize, k: usize) + requires + i < 63, + j < (1usize << i), + k < i, + ensures + ((j + (1usize << i)) as usize >> k) & 1 == (j >> k) & 1, +{ + assert(((j + (1usize << i)) as usize >> k) & 1 == (j >> k) & 1) by (bit_vector) + requires + i < 63, + j < (1usize << i), + k < i, + ; +} + +/// `2^(i+1) = 2 * 2^i` as shifts. +proof fn lemma_shl_double(i: usize) + requires + i < 63, + ensures + (1usize << i) * 2 == (1usize << ((i + 1) as usize)), + (1usize << i) >= 1, + (1usize << i) <= (1usize << 62usize), +{ + assert((1usize << i) * 2 == (1usize << ((i + 1) as usize)) && (1usize << i) >= 1 && (1usize << i) <= (1usize + << 62usize)) by (bit_vector) + requires + i < 63, + ; +} + +/// The bits of `(h << L) | l`, `l < 2^L`: those of `l` below `L`, those of `h` above. +proof fn lemma_bit_concat(h: usize, l: usize, low: usize, k: usize) + requires + low < 64, + l < (1usize << low), + k < 64, + ensures + k < low ==> (((h << low) | l) >> k) & 1 == (l >> k) & 1, + k >= low ==> (((h << low) | l) >> k) & 1 == (h >> ((k - low) as usize)) & 1, +{ + assert(k < low ==> (((h << low) | l) >> k) & 1 == (l >> k) & 1) by (bit_vector) + requires + low < 64, + l < (1usize << low), + k < 64, + ; + assert(k >= low ==> (((h << low) | l) >> k) & 1 == (h >> ((k - low) as usize)) & 1) by (bit_vector) + requires + low < 64, + l < (1usize << low), + k < 64, + ; +} + +/// `eq` of a concatenation is the product of the parts' `eq`. +pub proof fn lemma_eq_poly_concat(r1: Seq, x1: Seq, r2: Seq, x2: Seq) + requires + r1.len() == x1.len(), + r2.len() == x2.len(), + ensures + eq_poly(r1 + r2, x1 + x2) == e_mul(eq_poly(r1, x1), eq_poly(r2, x2)), + decreases r2.len(), +{ + if r2.len() == 0 { + assert(r1 + r2 =~= r1); + assert(x1 + x2 =~= x1); + lemma_e_mul_one(eq_poly(r1, x1)); + } else { + assert((r1 + r2).drop_last() =~= r1 + r2.drop_last()); + assert((x1 + x2).drop_last() =~= x1 + x2.drop_last()); + lemma_eq_poly_concat(r1, x1, r2.drop_last(), x2.drop_last()); + lemma_e_mul_assoc( + eq_poly(r1, x1), + eq_poly(r2.drop_last(), x2.drop_last()), + eq_factor(r2.last(), x2.last()), + ); + } +} + +/// The tensor structure of the table: `eq(r, (h << L) | l) = eq(r[..L], l) * eq(r[L..], h)` for `l < 2^L`. +pub proof fn lemma_eq_at_tensor(r: Seq, low: usize, h: usize, l: usize) + requires + low <= r.len() < 64, + l < (1usize << low), + ensures + eq_at(r, (h << low) | l) == e_mul(eq_at(r.subrange(0, low as int), l), eq_at(r.subrange(low as int, r.len() as int), h)), +{ + let n = r.len(); + let x = (h << low) | l; + let (rl, rh) = (r.subrange(0, low as int), r.subrange(low as int, n as int)); + assert forall|k: int| 0 <= k < n implies #[trigger] bit(x, k) == (if k < low { + bit(l, k) + } else { + bit(h, k - low) + }) by { + lemma_bit_concat(h, l, low, k as usize); + } + assert(cube_point(n, x) =~= cube_point(low as nat, l) + cube_point((n - low) as nat, h)); + assert(r =~= rl + rh); + lemma_eq_poly_concat(rl, cube_point(low as nat, l), rh, cube_point((n - low) as nat, h)); +} + +/// `h * 2^L + l = (h << L) | l` for `l < 2^L` and `h < 2^(n - L)`, and it is below `2^n`. +proof fn lemma_index_split(h: usize, l: usize, low: usize, n: usize) + requires + low <= n < 64, + l < (1usize << low), + h < (1usize << ((n - low) as usize)), + ensures + h * (1usize << low) + l == (h << low) | l, + h * (1usize << low) + l < (1usize << n), +{ + lemma_usize_pow2_no_overflow(low as nat); + lemma_usize_pow2_no_overflow((n - low) as nat); + lemma_usize_pow2_no_overflow(n as nat); + lemma_usize_shl_is_mul(1, low); + lemma_usize_shl_is_mul(1, (n - low) as usize); + lemma_usize_shl_is_mul(1, n); + lemma_pow2_adds((n - low) as nat, low as nat); + lemma_pow2_pos(low as nat); + assert(h * pow2(low as nat) + l < pow2(n as nat)) by (nonlinear_arith) + requires + h + 1 <= pow2((n - low) as nat), + l < pow2(low as nat), + pow2((n - low) as nat) * pow2(low as nat) == pow2(n as nat), + ; + lemma_usize_pow2_no_overflow(n as nat); + lemma_usize_shl_is_mul(h, low); + let hs = h << low; + assert(hs + l == hs | l) by (bit_vector) + requires + hs == h << low, + l < (1usize << low), + low < 64, + ; +} + +/// An index splits as `((x >> L) << L) | (x & (2^L - 1))`, its high part below `2^(n - L)`. +proof fn lemma_index_parts(x: usize, low: usize, n: usize) + requires + low <= n < 64, + x < (1usize << n), + ensures + x == ((x >> low) << low) | (x & sub(1usize << low, 1)), + (x & sub(1usize << low, 1)) < (1usize << low), + (x >> low) < (1usize << ((n - low) as usize)), + (1usize << low) >= 1, +{ + assert(x == ((x >> low) << low) | (x & sub(1usize << low, 1)) && (x & sub(1usize << low, 1)) < (1usize << low) + && (x >> low) < (1usize << ((n - low) as usize)) && (1usize << low) >= 1) by (bit_vector) + requires + low <= n < 64, + x < (1usize << n), + ; +} + +// --------------------------------------------------------------------------------------------- +// Executable code: the portable paths +// --------------------------------------------------------------------------------------------- +/// Multilinear interpolation in one variable over `E`: `lo + t·(lo+hi)`, the +/// char-2 form of `(1−t)·lo + t·hi`. +#[inline] +pub fn interp(lo: F192, hi: F192, t: F192) -> (r: F192) + ensures + r == e_add(e_mul(e_add(F192::ONE, t), lo), e_mul(t, hi)), +{ + proof { + lemma_e_mul_distrib(t, lo, hi); + lemma_e_mul_distrib(lo, F192::ONE, t); + lemma_e_mul_one(lo); + lemma_e_mul_comm(lo, e_add(F192::ONE, t)); + lemma_e_mul_comm(lo, t); + lemma_e_add(lo, e_mul(t, lo), e_mul(t, hi)); + } + lo + t * (lo + hi) +} + +/// Mixed interpolation: two `K` endpoints against an `E` parameter, one +/// `mul_base` (`lo + t·(lo+hi)` with `lo, hi ∈ K`). +#[inline] +pub fn interp_k(lo: F64, hi: F64, t: F192) -> (r: F192) + ensures + r == e_add(e_mul(e_add(F192::ONE, t), e_from_k(lo.0)), e_mul(t, e_from_k(hi.0))), +{ + proof { + let (l, h) = (e_from_k(lo.0), e_from_k(hi.0)); + assert(e_from_k(lo.0 ^ hi.0) == e_add(l, h)) by { + assert(0u64 ^ 0u64 == 0u64) by (bit_vector); + } + lemma_e_mul_distrib(t, l, h); + lemma_e_mul_distrib(l, F192::ONE, t); + lemma_e_mul_one(l); + lemma_e_mul_comm(l, e_add(F192::ONE, t)); + lemma_e_mul_comm(l, t); + lemma_e_add(l, e_mul(t, l), e_mul(t, h)); + } + F192::from(lo) + t.mul_base(lo + hi) +} + +/// `eq(r, x) = ∏_i (1 + r_i + x_i)`. For Boolean `r`, this is the indicator +/// of `x = r`; for arbitrary `r`, it is the multilinear interpolation weight. +/// +/// Production folds over `r.iter().zip(x)` after a `debug_assert_eq!` of the lengths, here a `requires`. +pub fn eq_eval(r: &[F192], x: &[F192]) -> (e: F192) + requires + r.len() == x.len(), + ensures + e == eq_poly(r@, x@), +{ + let mut acc = F192::ONE; + for i in 0..r.len() + invariant + r.len() == x.len(), + acc == eq_poly(r@.subrange(0, i as int), x@.subrange(0, i as int)), + { + proof { + lemma_eq_factor_sum(r@[i as int], x@[i as int]); + assert(r@.subrange(0, i + 1).drop_last() =~= r@.subrange(0, i as int)); + assert(x@.subrange(0, i + 1).drop_last() =~= x@.subrange(0, i as int)); + } + acc = acc * (F192::ONE + r[i] + x[i]); + } + proof { + assert(r@.subrange(0, r.len() as int) =~= r@); + assert(x@.subrange(0, x.len() as int) =~= x@); + } + acc +} + +/// The `eq(r, ·)` table over `n = r.len()` variables. See [`fill_eq_table_uninit`]. +pub fn eq_table(r: &[F192]) -> (t: Vec) + requires + r.len() < 64, + ensures + is_eq_table(t@, r@, F192::ONE), + decreases r.len(), 2nat, +{ + eq_table_seeded(r, F192::ONE) +} + +/// The table of `seed * eq(r, .)` over `n = r.len()` variables, in LSB-first order. +/// +/// Production fills the spare capacity of `Vec::with_capacity(len)` and then sets the length; the copy fills a +/// vector of zeros. The length `1 << r.len()` needs `r.len() < 64`, a `requires`. +pub fn eq_table_seeded(r: &[F192], seed: F192) -> (t: Vec) + requires + r.len() < 64, + ensures + is_eq_table(t@, r@, seed), + decreases r.len(), 1nat, +{ + // One entry per point of the cube: 2^n. + let len = 1usize << r.len(); + + let mut eq = vec![F192::ZERO; len]; + fill_eq_table_uninit(r, seed, eq.as_mut_slice()); + eq +} + +/// Fill `out` with `seed * eq(r, .)`, in LSB-first order. Every entry is written before it is read. +/// +/// A small table doubles a level at a time on the calling thread. +/// +/// A large one is a tensor product, written in one parallel pass: +/// +/// ```text +/// out[h * 2^L + l] = high[h] * low[l], low = eq(r[..L]), high = seed * eq(r[L..]) +/// ``` +/// +/// Production writes `MaybeUninit` slots; the copy writes initialized ones. Its `assert_eq!` on the length is a +/// `requires`. The parallel pass (`parallel::chunks_mut` over chunks of whole rows, each row zipped with its +/// weight `high[h]` and cut `as_chunks_mut::<4>`) becomes a loop over the rows in order: each entry is written +/// once, from `high[h]` and `low` alone, so the order of the rows does not change the table. +pub fn fill_eq_table_uninit(r: &[F192], seed: F192, out: &mut [F192]) + requires + r.len() < 64, + old(out).len() == (1usize << r.len()), + ensures + is_eq_table(final(out)@, r@, seed), + decreases r.len(), 0nat, +{ + if out.len() < EQ_PAR_LEN { + return fill_eq_doubling(r, seed, out); + } + proof { + let nn = r.len(); + assert(EQ_PAR_LEN == 65536) by (compute_only); + assert((1usize << nn) >= 65536usize ==> nn >= 16) by (bit_vector) + requires + nn < 64, + ; + } + let r_low = &r[..EQ_LOW_VARS]; + let r_high = &r[EQ_LOW_VARS..]; + let low = eq_table(r_low); + let high = eq_table_seeded(r_high, seed); + let ghost n = r.len(); + let ghost hl = (n - EQ_LOW_VARS) as usize; + proof { + assert(r_low@ == r@.subrange(0, EQ_LOW_VARS as int)); + assert(r_high@ == r@.subrange(EQ_LOW_VARS as int, n as int)); + assert(low.len() == 1024) by { + assert((1usize << 10usize) == 1024) by (bit_vector); + } + } + let mut h = 0; + while h < high.len() + invariant + n == r.len(), + n < 64, + hl == n - EQ_LOW_VARS, + out.len() == (1usize << n), + low.len() == 1024, + low.len() == (1usize << EQ_LOW_VARS), + high.len() == (1usize << hl), + is_eq_table(low@, r_low@, F192::ONE), + is_eq_table(high@, r_high@, seed), + r_low@ == r@.subrange(0, EQ_LOW_VARS as int), + r_high@ == r@.subrange(EQ_LOW_VARS as int, n as int), + h <= high.len(), + forall|x: int| 0 <= x < h * 1024 ==> #[trigger] out@[x] == e_mul(seed, eq_at(r@, x as usize)), + decreases high.len() - h, + { + let w = high[h]; + let mut c = 0; + while c < low.len() / 4 + invariant + n == r.len(), + n < 64, + hl == n - EQ_LOW_VARS, + out.len() == (1usize << n), + low.len() == 1024, + low.len() == (1usize << EQ_LOW_VARS), + high.len() == (1usize << hl), + is_eq_table(low@, r_low@, F192::ONE), + is_eq_table(high@, r_high@, seed), + r_low@ == r@.subrange(0, EQ_LOW_VARS as int), + r_high@ == r@.subrange(EQ_LOW_VARS as int, n as int), + h < high.len(), + w == high@[h as int], + c <= 256, + forall|x: int| 0 <= x < h * 1024 + 4 * c ==> #[trigger] out@[x] == e_mul(seed, eq_at(r@, x as usize)), + decreases 256 - c, + { + let src = [low[4 * c], low[4 * c + 1], low[4 * c + 2], low[4 * c + 3]]; + let p = mul4([w, w, w, w], src); + for k in 0..4 + invariant + n == r.len(), + n < 64, + hl == n - EQ_LOW_VARS, + out.len() == (1usize << n), + low.len() == 1024, + low.len() == (1usize << EQ_LOW_VARS), + high.len() == (1usize << hl), + is_eq_table(low@, r_low@, F192::ONE), + is_eq_table(high@, r_high@, seed), + r_low@ == r@.subrange(0, EQ_LOW_VARS as int), + r_high@ == r@.subrange(EQ_LOW_VARS as int, n as int), + h < high.len(), + w == high@[h as int], + c < 256, + forall|kk: int| 0 <= kk < 4 ==> #[trigger] p[kk] == e_mul(w, low@[4 * c + kk]), + forall|x: int| 0 <= x < h * 1024 + 4 * c + k ==> #[trigger] out@[x] == e_mul(seed, eq_at(r@, x as usize)), + { + let l = 4 * c + k; + proof { + lemma_index_split(h, l, EQ_LOW_VARS, n as usize); + lemma_tensor_entry(r@, seed, h, l); + lemma_e_mul_one(eq_at(r_low@, l)); + } + out[h * low.len() + l] = p[k]; + } + c += 1; + } + h += 1; + } + proof { + lemma_shl_sum(hl, EQ_LOW_VARS); + } +} + +/// `2^a * 2^b = 2^(a + b)` as shifts. +proof fn lemma_shl_sum(a: usize, b: usize) + requires + a + b < 64, + ensures + (1usize << a) * (1usize << b) == (1usize << ((a + b) as usize)), +{ + lemma_usize_pow2_no_overflow((a + b) as nat); + lemma_usize_pow2_no_overflow(a as nat); + lemma_usize_pow2_no_overflow(b as nat); + lemma_usize_shl_is_mul(1, a); + lemma_usize_shl_is_mul(1, b); + lemma_usize_shl_is_mul(1, (a + b) as usize); + lemma_pow2_adds(a as nat, b as nat); +} + +/// One entry of the tensor pass: `seed eq(r_high, h) * eq(r_low, l) = seed eq(r, h 2^L + l)`. +proof fn lemma_tensor_entry(r: Seq, seed: F192, h: usize, l: usize) + requires + EQ_LOW_VARS <= r.len() < 64, + l < 1024, + h < (1usize << ((r.len() - EQ_LOW_VARS) as usize)), + ensures + h * 1024 + l < (1usize << r.len()), + e_mul(e_mul(seed, eq_at(r.subrange(EQ_LOW_VARS as int, r.len() as int), h)), eq_at(r.subrange(0, EQ_LOW_VARS as int), l)) + == e_mul(seed, eq_at(r, (h * 1024 + l) as usize)), +{ + assert((1usize << 10usize) == 1024) by (bit_vector); + lemma_index_split(h, l, EQ_LOW_VARS, r.len() as usize); + lemma_eq_at_tensor(r, EQ_LOW_VARS, h, l); + let (a, b) = (eq_at(r.subrange(0, EQ_LOW_VARS as int), l), eq_at(r.subrange(EQ_LOW_VARS as int, r.len() as int), h)); + lemma_e_mul_assoc(seed, b, a); + lemma_e_mul_comm(b, a); +} + +/// Tables below this size are built on the calling thread. +pub const EQ_PAR_LEN: usize = 1 << 16; + +/// The variables of the L1-resident factor of a large `eq` table. +pub const EQ_LOW_VARS: usize = 10; + +/// One doubling step on entry `j` of level `i`: the high child `v r_i` and the low child `v + v r_i`. +proof fn lemma_doubling_entry(r: Seq, seed: F192, j: usize) + requires + 1 <= r.len() <= 63, + j < (1usize << ((r.len() - 1) as usize)), + ensures + ({ + let v = e_mul(seed, eq_at(r.drop_last(), j)); + &&& e_mul(r.last(), v) == e_mul(seed, eq_at(r, (j + (1usize << ((r.len() - 1) as usize))) as usize)) + &&& e_mul(v, r.last()) == e_mul(seed, eq_at(r, (j + (1usize << ((r.len() - 1) as usize))) as usize)) + &&& e_add(v, e_mul(r.last(), v)) == e_mul(seed, eq_at(r, j)) + &&& e_add(v, e_mul(v, r.last())) == e_mul(seed, eq_at(r, j)) + }), +{ + let e = eq_at(r.drop_last(), j); + let v = e_mul(seed, e); + let rk = r.last(); + lemma_eq_at_high(r, j); + lemma_e_mul_comm(rk, v); + lemma_e_mul_assoc(seed, e, rk); + lemma_times_one_plus(v, rk); + lemma_e_mul_assoc(seed, e, e_add(F192::ONE, rk)); +} + +/// The level-by-level `eq` build: each level writes the high half from the low half, then rewrites the low half. +/// +/// In characteristic 2 the low child is the high child plus the parent, so each pair costs one product. +/// +/// Production enumerates `r.iter()`, splits `out[..2 * half]` with `split_at_mut` (and casts the initialized low +/// half from `MaybeUninit`), and walks both halves `as_chunks_mut::<4>` then their tails; the copy indexes +/// `lo[j] = out[j]`, `hi[j] = out[half + j]`, and `l[k] += p[k]` is `out[..] = out[..] + p[k]`. +fn fill_eq_doubling(r: &[F192], seed: F192, out: &mut [F192]) + requires + r.len() < 64, + old(out).len() == (1usize << r.len()), + ensures + is_eq_table(final(out)@, r@, seed), +{ + let ghost n = r.len(); + proof { + assert((1usize << n) >= 1) by (bit_vector) + requires + n < 64, + ; + } + out[0] = seed; + proof { + assert((1usize << 0usize) == 1) by (bit_vector); + assert(eq_at(r@.subrange(0, 0), 0) == F192::ONE); + lemma_e_mul_one(seed); + } + for i in 0..r.len() + invariant + n == r.len(), + n < 64, + out.len() == (1usize << n), + forall|j: int| 0 <= j < (1usize << i) ==> #[trigger] out@[j] == e_mul(seed, eq_at(r@.subrange(0, i as int), j as usize)), + { + let rk = r[i]; + let half = 1usize << i; + let ghost lvl = r@.subrange(0, i as int); + let ghost nxt = r@.subrange(0, i + 1); + let ghost before = out@; + proof { + lemma_shl_double(i); + assert((1usize << ((i + 1) as usize)) <= (1usize << n)) by (bit_vector) + requires + i < n, + n < 64, + ; + assert(nxt.drop_last() =~= lvl); + assert(nxt.last() == rk); + } + let n4 = half / 4; + let mut c = 0; + while c < n4 + invariant + n == r.len(), + n < 64, + i < n, + out.len() == (1usize << n), + half == (1usize << i), + 2 * half <= out.len(), + before.len() == out.len(), + n4 == half / 4, + c <= n4, + rk == r@[i as int], + lvl == r@.subrange(0, i as int), + nxt == r@.subrange(0, i + 1), + nxt.drop_last() == lvl, + nxt.last() == rk, + forall|j: int| 0 <= j < half ==> #[trigger] before[j] == e_mul(seed, eq_at(lvl, j as usize)), + forall|j: int| 0 <= j < 4 * c ==> #[trigger] out@[j] == e_mul(seed, eq_at(nxt, j as usize)), + forall|j: int| 0 <= j < 4 * c ==> #[trigger] out@[half + j] == e_mul(seed, eq_at(nxt, (half + j) as usize)), + forall|j: int| 4 * c <= j < half ==> #[trigger] out@[j] == before[j], + decreases n4 - c, + { + let l = [out[4 * c], out[4 * c + 1], out[4 * c + 2], out[4 * c + 3]]; + let p = mul4([rk, rk, rk, rk], l); + for k in 0..4 + invariant + n == r.len(), + n < 64, + i < n, + out.len() == (1usize << n), + half == (1usize << i), + 2 * half <= out.len(), + before.len() == out.len(), + n4 == half / 4, + c < n4, + rk == r@[i as int], + lvl == r@.subrange(0, i as int), + nxt == r@.subrange(0, i + 1), + nxt.drop_last() == lvl, + nxt.last() == rk, + forall|kk: int| 0 <= kk < 4 ==> #[trigger] l[kk] == before[4 * c + kk], + forall|kk: int| 0 <= kk < 4 ==> #[trigger] p[kk] == e_mul(rk, l[kk]), + forall|j: int| 0 <= j < half ==> #[trigger] before[j] == e_mul(seed, eq_at(lvl, j as usize)), + forall|j: int| 0 <= j < 4 * c + k ==> #[trigger] out@[j] == e_mul(seed, eq_at(nxt, j as usize)), + forall|j: int| 0 <= j < 4 * c + k ==> #[trigger] out@[half + j] == e_mul(seed, eq_at(nxt, (half + j) as usize)), + forall|j: int| 4 * c + k <= j < half ==> #[trigger] out@[j] == before[j], + { + let j = 4 * c + k; + proof { + lemma_doubling_entry(nxt, seed, j); + } + out[half + j] = p[k]; + out[j] = out[j] + p[k]; + } + c += 1; + } + let mut j = 4 * n4; + while j < half + invariant + n == r.len(), + n < 64, + i < n, + out.len() == (1usize << n), + half == (1usize << i), + 2 * half <= out.len(), + before.len() == out.len(), + 4 * n4 <= j <= half, + rk == r@[i as int], + lvl == r@.subrange(0, i as int), + nxt == r@.subrange(0, i + 1), + nxt.drop_last() == lvl, + nxt.last() == rk, + forall|jj: int| 0 <= jj < half ==> #[trigger] before[jj] == e_mul(seed, eq_at(lvl, jj as usize)), + forall|jj: int| 0 <= jj < j ==> #[trigger] out@[jj] == e_mul(seed, eq_at(nxt, jj as usize)), + forall|jj: int| 0 <= jj < j ==> #[trigger] out@[half + jj] == e_mul(seed, eq_at(nxt, (half + jj) as usize)), + forall|jj: int| j <= jj < half ==> #[trigger] out@[jj] == before[jj], + decreases half - j, + { + proof { + lemma_doubling_entry(nxt, seed, j); + } + let p = out[j] * rk; + out[half + j] = p; + out[j] = out[j] + p; + j += 1; + } + proof { + assert forall|jj: int| 0 <= jj < (1usize << ((i + 1) as usize)) implies #[trigger] out@[jj] == e_mul( + seed, + eq_at(nxt, jj as usize), + ) by { + if jj >= half { + assert(out@[half + (jj - half)] == e_mul(seed, eq_at(nxt, (half + (jj - half)) as usize))); + } + } + } + } + proof { + assert(r@.subrange(0, n as int) =~= r@); + } +} + +// --------------------------------------------------------------------------------------------- +// Marginalizing a variable out of a table +// --------------------------------------------------------------------------------------------- +/// The one-variable table: `eq([r0], 0) = 1 + r0`, `eq([r0], 1) = r0`. +proof fn lemma_eq_at_one(s: Seq) + requires + s.len() == 1, + ensures + eq_at(s, 0) == e_add(F192::ONE, s[0]), + eq_at(s, 1) == s[0], +{ + assert(((0usize >> 0usize) & 1) == 0 && ((1usize >> 0usize) & 1) == 1) by (bit_vector); + assert(s.drop_last().len() == 0); + assert(eq_poly(s.drop_last(), cube_point(1, 0).drop_last()) == F192::ONE); + assert(eq_poly(s.drop_last(), cube_point(1, 1).drop_last()) == F192::ONE); + assert(cube_point(1, 0).last() == F192::ZERO); + assert(cube_point(1, 1).last() == F192::ONE); + lemma_eq_factor_bits(s[0]); + lemma_e_mul_one(e_add(F192::ONE, s[0])); + lemma_e_mul_one(s[0]); +} + +/// `2i = (i << 1) | 0` and `2i + 1 = (i << 1) | 1`. +proof fn lemma_double_index(i: usize) + requires + i < 0x4000_0000_0000_0000usize, + ensures + 2 * i == (i << 1usize) | 0usize, + 2 * i + 1 == (i << 1usize) | 1usize, + (1usize << 1usize) == 2, +{ + assert(2 * i == (i << 1usize) | 0usize && 2 * i + 1 == (i << 1usize) | 1usize && (1usize << 1usize) == 2) + by (bit_vector) + requires + i < 0x4000_0000_0000_0000usize, + ; +} + +/// The two entries of a table sharing all but the lowest variable: `eq(r, 2i) = (1 + r0) eq(r[1..], i)` and +/// `eq(r, 2i + 1) = r0 eq(r[1..], i)`. +pub proof fn lemma_eq_at_low(r: Seq, i: usize) + requires + 1 <= r.len() < 64, + i < (1usize << ((r.len() - 1) as usize)), + ensures + eq_at(r, (2 * i) as usize) == e_mul(e_add(F192::ONE, r[0]), eq_at(r.subrange(1, r.len() as int), i)), + eq_at(r, (2 * i + 1) as usize) == e_mul(r[0], eq_at(r.subrange(1, r.len() as int), i)), +{ + let m = (r.len() - 1) as usize; + assert((1usize << m) <= 0x4000_0000_0000_0000usize) by (bit_vector) + requires + m < 63, + ; + lemma_double_index(i); + lemma_eq_at_tensor(r, 1, i, 0); + lemma_eq_at_tensor(r, 1, i, 1); + lemma_eq_at_one(r.subrange(0, 1)); +} + +/// Summing the two entries of each low pair of `seed * eq(r, .)` gives `seed * eq(r[1..], .)`: `eq(r_0, 0) + +/// eq(r_0, 1) = 1`. +pub proof fn lemma_shrink_low_eq(t: Seq, r: Seq, seed: F192) + requires + is_eq_table(t, r, seed), + r.len() >= 1, + ensures + is_eq_table(Seq::new(t.len() / 2, |i: int| e_add(t[2 * i], t[2 * i + 1])), r.subrange(1, r.len() as int), seed), +{ + let n = r.len(); + let rest = r.subrange(1, n as int); + lemma_shl_double((n - 1) as usize); + assert forall|i: int| 0 <= i < t.len() / 2 implies #[trigger] e_add(t[2 * i], t[2 * i + 1]) == e_mul( + seed, + eq_at(rest, i as usize), + ) by { + lemma_eq_at_low(r, i as usize); + let e = eq_at(rest, i as usize); + let (c0, c1) = (e_add(F192::ONE, r[0]), r[0]); + lemma_e_mul_distrib(seed, e_mul(c0, e), e_mul(c1, e)); + lemma_e_mul_distrib(e, c0, c1); + lemma_e_add(F192::ONE, r[0], r[0]); + lemma_e_add(r[0], r[0], r[0]); + lemma_e_mul_one(e); + assert(e_add(c0, c1) == F192::ONE); + assert(e_add(e_mul(c0, e), e_mul(c1, e)) == e); + } +} + +/// Summing each entry of the low half of `seed * eq(r, .)` with its partner in the high half gives +/// `seed * eq(r[..n-1], .)`. +pub proof fn lemma_shrink_high_eq(t: Seq, r: Seq, seed: F192) + requires + is_eq_table(t, r, seed), + r.len() >= 1, + ensures + is_eq_table(Seq::new(t.len() / 2, |i: int| e_add(t[i], t[i + t.len() / 2])), r.drop_last(), seed), +{ + let n = r.len(); + let half = 1usize << ((n - 1) as usize); + lemma_shl_double((n - 1) as usize); + assert forall|i: int| 0 <= i < t.len() / 2 implies #[trigger] e_add(t[i], t[i + t.len() / 2]) == e_mul( + seed, + eq_at(r.drop_last(), i as usize), + ) by { + lemma_eq_at_high(r, i as usize); + let e = eq_at(r.drop_last(), i as usize); + let (c0, c1) = (e_add(F192::ONE, r.last()), r.last()); + lemma_e_mul_distrib(seed, e_mul(e, c0), e_mul(e, c1)); + lemma_e_mul_distrib(e, c0, c1); + lemma_e_add(F192::ONE, c1, c1); + lemma_e_add(c1, c1, c1); + lemma_e_mul_one(e); + assert(e_add(c0, c1) == F192::ONE); + } +} + +/// Marginalize the lowest variable out of an `eq` table (in place). `eq(r_0, 0) + +/// eq(r_0, 1) = 1`, so summing adjacent entries drops `r_0` with no multiplies, +/// versus `2^{n-1}` to rebuild the table. +/// +/// The pairwise sums of any table; on `seed * eq(r, .)` they are `seed * eq(r[1..], .)` ([`lemma_shrink_low_eq`]). +/// Production reborrows the vector as a slice to index it; the copy indexes the vector. +pub fn shrink_eq_low(table: &mut Vec) + ensures + final(table)@ == Seq::new(old(table)@.len() / 2, |i: int| e_add(old(table)@[2 * i], old(table)@[2 * i + 1])), + forall|r: Seq, seed: F192| + #[trigger] is_eq_table(old(table)@, r, seed) && r.len() >= 1 ==> is_eq_table( + final(table)@, + r.subrange(1, r.len() as int), + seed, + ), +{ + let ghost before = table@; + let half = table.len() / 2; + { + for i in 0..half + invariant + half == before.len() / 2, + table.len() == before.len(), + forall|j: int| 0 <= j < i ==> #[trigger] table@[j] == e_add(before[2 * j], before[2 * j + 1]), + forall|j: int| i <= j < before.len() ==> #[trigger] table@[j] == before[j], + { + let (a, b) = (table[2 * i], table[2 * i + 1]); + table[i] = a + b; + } + } + table.truncate(half); + proof { + assert(table@ =~= Seq::new(before.len() / 2, |i: int| e_add(before[2 * i], before[2 * i + 1]))); + assert forall|r: Seq, seed: F192| #[trigger] is_eq_table(before, r, seed) && r.len() >= 1 implies is_eq_table( + table@, + r.subrange(1, r.len() as int), + seed, + ) by { + lemma_shrink_low_eq(before, r, seed); + } + } +} + +/// Marginalize the highest variable out of an `eq` table (in place), the +/// [`shrink_eq_low`] counterpart for a top-down sumcheck. +/// +/// Each low entry plus its high partner; on `seed * eq(r, .)` that is `seed * eq(r[..n-1], .)` +/// ([`lemma_shrink_high_eq`]). Production splits the slice with `split_at_mut` and zips the halves; the copy +/// indexes `lo[i] = table[i]`, `hi[i] = table[half + i]`. +pub fn shrink_eq_high(table: &mut Vec) + ensures + final(table)@ == Seq::new(old(table)@.len() / 2, |i: int| e_add(old(table)@[i], old(table)@[i + old(table)@.len() / 2])), + forall|r: Seq, seed: F192| + #[trigger] is_eq_table(old(table)@, r, seed) && r.len() >= 1 ==> is_eq_table( + final(table)@, + r.drop_last(), + seed, + ), +{ + let ghost before = table@; + let half = table.len() / 2; + { + for i in 0..half + invariant + half == before.len() / 2, + table.len() == before.len(), + forall|j: int| 0 <= j < i ==> #[trigger] table@[j] == e_add(before[j], before[j + half]), + forall|j: int| i <= j < before.len() ==> #[trigger] table@[j] == before[j], + { + let h = table[half + i]; + table[i] = table[i] + h; + } + } + table.truncate(half); + proof { + assert(table@ =~= Seq::new(before.len() / 2, |i: int| e_add(before[i], before[i + before.len() / 2]))); + assert forall|r: Seq, seed: F192| #[trigger] is_eq_table(before, r, seed) && r.len() >= 1 implies is_eq_table( + table@, + r.drop_last(), + seed, + ) by { + lemma_shrink_high_eq(before, r, seed); + } + } +} + +// --------------------------------------------------------------------------------------------- +// The split table +// --------------------------------------------------------------------------------------------- +/// The table `eq(r, .)` as two smaller tables, `eq(r, x) = low[x mod 2^L] * high[x >> L]`. +/// +/// - The low table is `eq` over the first `L` variables of `r`, the high table over the rest. +/// - Together they hold `2^L + 2^(n - L)` entries instead of `2^n`. +/// - Products are exact, so every entry equals the full table's. +#[verifier::allow(autoderive_clone_without_spec)] +#[derive(Clone, Debug)] +pub struct SplitEq { + /// The table over the low `L` variables. + pub low: Vec, + /// The table over the remaining variables. + pub high: Vec, + /// `L`. + low_log: usize, +} + +impl SplitEq { + /// The split of `eq(r, .)` after `L = low_log` variables. + pub closed spec fn splits(self, r: Seq) -> bool { + &&& self.low_log <= r.len() < 64 + &&& is_eq_table(self.low@, r.subrange(0, self.low_log as int), F192::ONE) + &&& is_eq_table(self.high@, r.subrange(self.low_log as int, r.len() as int), F192::ONE) + } + + /// `L`. + pub closed spec fn spec_low_log(self) -> usize { + self.low_log + } + + /// The split with at most `max_low` low variables. + /// + /// Production's `r.len().min(max_low)` is an `if`. + pub fn with_low_vars(r: &[F192], max_low: usize) -> (s: Self) + requires + r.len() < 64, + ensures + s.splits(r@), + s.spec_low_log() == (if r.len() < max_low { r.len() } else { max_low }), + { + Self::at_split(r, if r.len() < max_low { r.len() } else { max_low }) + } + + /// The split with at most `max_high` high variables. + /// + /// Production's `r.len().min(max_high)` is an `if`. + pub fn with_high_vars(r: &[F192], max_high: usize) -> (s: Self) + requires + r.len() < 64, + ensures + s.splits(r@), + s.spec_low_log() == r.len() - (if r.len() < max_high { r.len() } else { max_high }), + { + Self::at_split(r, r.len() - if r.len() < max_high { r.len() } else { max_high }) + } + + fn at_split(r: &[F192], low_log: usize) -> (s: Self) + requires + low_log <= r.len() < 64, + ensures + s.splits(r@), + s.spec_low_log() == low_log, + { + Self { low: eq_table(&r[..low_log]), high: eq_table(&r[low_log..]), low_log } + } + + /// The number of low variables `L`. + pub const fn low_log(&self) -> (l: usize) + ensures + l == self.spec_low_log(), + { + self.low_log + } + + /// `eq(r, x)`. + /// + /// The split of `eq(r, .)` for some `r` with `x` on its cube, a `requires`; the result is that entry. + #[inline] + pub fn at(&self, x: usize) -> (e: F192) + requires + exists|r: Seq| #[trigger] self.splits(r) && x < (1usize << r.len()), + ensures + forall|r: Seq| #[trigger] self.splits(r) && x < (1usize << r.len()) ==> e == eq_at(r, x), + { + let ghost r0 = choose|r: Seq| #[trigger] self.splits(r) && x < (1usize << r.len()); + proof { + lemma_index_parts(x, self.low_log, r0.len() as usize); + lemma_split_at(*self, r0, x); + } + self.low[x & (self.low.len() - 1)] * self.high[x >> self.low_log] + } +} + +/// The entry `at` reads, for any `r` the split is of. +proof fn lemma_split_at(s: SplitEq, r0: Seq, x: usize) + requires + s.splits(r0), + x < (1usize << r0.len()), + ensures + s.low.len() >= 1, + (x & sub(s.low@.len() as usize, 1)) < s.low.len(), + (x >> s.spec_low_log()) < s.high.len(), + forall|r: Seq| #[trigger] s.splits(r) && x < (1usize << r.len()) ==> e_mul( + s.low@[(x & sub(s.low@.len() as usize, 1)) as int], + s.high@[(x >> s.spec_low_log()) as int], + ) == eq_at(r, x), +{ + let low = s.spec_low_log(); + lemma_index_parts(x, low, r0.len() as usize); + assert forall|r: Seq| #[trigger] s.splits(r) && x < (1usize << r.len()) implies e_mul( + s.low@[(x & sub(s.low@.len() as usize, 1)) as int], + s.high@[(x >> low) as int], + ) == eq_at(r, x) by { + lemma_index_parts(x, low, r.len() as usize); + let (l, h) = (x & sub(1usize << low, 1), x >> low); + lemma_eq_at_tensor(r, low, h, l); + lemma_e_mul_one(eq_at(r.subrange(0, low as int), l)); + lemma_e_mul_one(eq_at(r.subrange(low as int, r.len() as int), h)); + } +} + +// --------------------------------------------------------------------------------------------- +// Sums over the cube, and the multilinear extension +// --------------------------------------------------------------------------------------------- +/// `sum_{i < n} g(i)`, from the first term up. +pub open spec fn e_sum_fn(g: spec_fn(int) -> F192, n: nat) -> F192 + decreases n, +{ + if n == 0 { + F192::ZERO + } else { + e_add(e_sum_fn(g, (n - 1) as nat), g(n - 1)) + } +} + +/// The multilinear extension of the table `f` at `p`: `sum_x eq(p, x) f[x]` over the cube. +pub open spec fn mle(f: Seq, p: Seq) -> F192 { + e_sum_fn(|x: int| e_mul(eq_at(p, x as usize), f[x]), f.len()) +} + +/// A table of `K` words lifted into `E`. +pub open spec fn lift(t: Seq) -> Seq { + t.map_values(|w: F64| e_from_k(w.0)) +} + +/// `(1 + t) lo + t hi`, the line through `(0, lo)` and `(1, hi)` at `t`. +pub open spec fn interp_spec(lo: F192, hi: F192, t: F192) -> F192 { + e_add(e_mul(e_add(F192::ONE, t), lo), e_mul(t, hi)) +} + +/// The table with its lowest variable bound to `t`. +pub open spec fn fold_low(f: Seq, t: F192) -> Seq { + Seq::new(f.len() / 2, |i: int| interp_spec(f[2 * i], f[2 * i + 1], t)) +} + +pub proof fn lemma_sum_ext(g1: spec_fn(int) -> F192, g2: spec_fn(int) -> F192, n: nat) + requires + forall|i: int| 0 <= i < n ==> #[trigger] g1(i) == g2(i), + ensures + e_sum_fn(g1, n) == e_sum_fn(g2, n), + decreases n, +{ + if n > 0 { + lemma_sum_ext(g1, g2, (n - 1) as nat); + } +} + +/// A sum of pairs: `sum_{x < 2m} g(x) = sum_{i < m} (g(2i) + g(2i + 1))`. +pub proof fn lemma_sum_pairs(g: spec_fn(int) -> F192, m: nat) + ensures + e_sum_fn(g, 2 * m) == e_sum_fn(|i: int| e_add(g(2 * i), g(2 * i + 1)), m), + decreases m, +{ + if m > 0 { + lemma_sum_pairs(g, (m - 1) as nat); + let s = e_sum_fn(g, (2 * m - 2) as nat); + assert(e_sum_fn(g, (2 * m - 1) as nat) == e_add(s, g(2 * m - 2))); + lemma_e_add(s, g(2 * m - 2), g(2 * m - 1)); + } +} + +/// The terms of `g` from `m` on. +pub open spec fn shifted(g: spec_fn(int) -> F192, m: int) -> spec_fn(int) -> F192 { + |l: int| g(m + l) +} + +/// The sums of `g` over consecutive blocks of `b` terms. +pub open spec fn block_sums(g: spec_fn(int) -> F192, b: nat) -> spec_fn(int) -> F192 { + |h: int| e_sum_fn(shifted(g, h * b), b) +} + +/// A sum split after `m` terms. +pub proof fn lemma_sum_split(g: spec_fn(int) -> F192, m: nat, k: nat) + ensures + e_sum_fn(g, m + k) == e_add(e_sum_fn(g, m), e_sum_fn(shifted(g, m as int), k)), + decreases k, +{ + if k == 0 { + lemma_e_add(e_sum_fn(g, m), F192::ZERO, F192::ZERO); + } else { + lemma_sum_split(g, m, (k - 1) as nat); + assert(shifted(g, m as int)(k - 1) == g(m + k - 1)); + lemma_e_add(e_sum_fn(g, m), e_sum_fn(shifted(g, m as int), (k - 1) as nat), g(m + k - 1)); + } +} + +/// A sum over `a` blocks of `b`: `sum_{x < ab} g(x) = sum_{h < a} sum_{l < b} g(hb + l)`. +pub proof fn lemma_sum_blocks(g: spec_fn(int) -> F192, a: nat, b: nat) + ensures + e_sum_fn(g, a * b) == e_sum_fn(block_sums(g, b), a), + decreases a, +{ + if a > 0 { + lemma_sum_blocks(g, (a - 1) as nat, b); + let m = ((a - 1) * b) as nat; + assert(a * b == m + b) by (nonlinear_arith) + requires + m == (a - 1) * b, + a > 0, + ; + assert((a - 1) * b >= 0) by (nonlinear_arith) + requires + a > 0, + ; + assert(m as int == (a - 1) * b); + lemma_sum_split(g, m, b); + let bs = block_sums(g, b); + assert(bs(a - 1) == e_sum_fn(shifted(g, m as int), b)); + assert(e_sum_fn(bs, a) == e_add(e_sum_fn(bs, (a - 1) as nat), bs(a - 1))); + assert(e_sum_fn(g, ((a - 1) as nat) * b) == e_sum_fn(bs, (a - 1) as nat)); + assert(e_sum_fn(g, m) == e_sum_fn(bs, (a - 1) as nat)); + } else { + assert(a * b == 0) by (nonlinear_arith) + requires + a == 0, + ; + } +} + +/// A factor out of a sum: `c * sum_i g(i) = sum_i c g(i)`. +pub proof fn lemma_sum_scale(c: F192, g: spec_fn(int) -> F192, n: nat) + ensures + e_mul(c, e_sum_fn(g, n)) == e_sum_fn(|i: int| e_mul(c, g(i)), n), + decreases n, +{ + if n == 0 { + lemma_e_mul_zero(c); + } else { + lemma_sum_scale(c, g, (n - 1) as nat); + lemma_e_mul_distrib(c, e_sum_fn(g, (n - 1) as nat), g(n - 1)); + } +} + +/// Binding the lowest variable: `mle(f, p) = mle(fold_low(f, p_0), p[1..])`. +pub proof fn lemma_mle_fold_low(f: Seq, p: Seq) + requires + 1 <= p.len() < 64, + f.len() == (1usize << p.len()), + ensures + mle(f, p) == mle(fold_low(f, p[0]), p.subrange(1, p.len() as int)), +{ + let n = p.len(); + let rest = p.subrange(1, n as int); + let g = fold_low(f, p[0]); + let m = f.len() / 2; + lemma_shl_double((n - 1) as usize); + let big = |x: int| e_mul(eq_at(p, x as usize), f[x]); + let small = |i: int| e_mul(eq_at(rest, i as usize), g[i]); + lemma_sum_pairs(big, m as nat); + assert forall|i: int| 0 <= i < m implies #[trigger] e_add(big(2 * i), big(2 * i + 1)) == small(i) by { + lemma_eq_at_low(p, i as usize); + let e = eq_at(rest, i as usize); + let (c0, c1, a, b) = (e_add(F192::ONE, p[0]), p[0], f[2 * i], f[2 * i + 1]); + lemma_e_mul_comm(c0, e); + lemma_e_mul_comm(c1, e); + lemma_e_mul_assoc(e, c0, a); + lemma_e_mul_assoc(e, c1, b); + lemma_e_mul_distrib(e, e_mul(c0, a), e_mul(c1, b)); + } + lemma_sum_ext(|i: int| e_add(big(2 * i), big(2 * i + 1)), small, m as nat); + assert(2 * m == f.len()); +} + +/// One term of the blocked sum: `eq(p, h 2^L + l) f[h 2^L + l] = eq(p_high, h) (eq(p_low, l) f[h 2^L + l])`. +proof fn lemma_mle_block_term(f: Seq, p: Seq, low: usize, h: int, l: int) + requires + low <= p.len() < 64, + 0 <= l < (1usize << low), + 0 <= h < (1usize << ((p.len() - low) as usize)), + ensures + h * (1usize << low) + l < (1usize << p.len()), + e_mul(eq_at(p, (h * (1usize << low) + l) as usize), f[h * (1usize << low) + l]) == e_mul( + eq_at(p.subrange(low as int, p.len() as int), h as usize), + e_mul(eq_at(p.subrange(0, low as int), l as usize), f[h * (1usize << low) + l]), + ), +{ + lemma_index_split(h as usize, l as usize, low, p.len() as usize); + lemma_eq_at_tensor(p, low, h as usize, l as usize); + let (el, eh) = (eq_at(p.subrange(0, low as int), l as usize), eq_at(p.subrange(low as int, p.len() as int), h as usize)); + lemma_e_mul_comm(el, eh); + lemma_e_mul_assoc(eh, el, f[h * (1usize << low) + l]); +} + +/// One block of the blocked sum is its row's value scaled by `eq(p_high, h)`. +proof fn lemma_mle_block_row(f: Seq, p: Seq, low: usize, h: int) + requires + low <= p.len() < 64, + f.len() == (1usize << p.len()), + 0 <= h < (1usize << ((p.len() - low) as usize)), + ensures + block_sums(|x: int| e_mul(eq_at(p, x as usize), f[x]), (1usize << low) as nat)(h) == e_mul( + eq_at(p.subrange(low as int, p.len() as int), h as usize), + mle(f.subrange(h * (1usize << low), (h + 1) * (1usize << low)), p.subrange(0, low as int)), + ), +{ + let n = p.len(); + let b = (1usize << low) as nat; + let big = |x: int| e_mul(eq_at(p, x as usize), f[x]); + assert((1usize << low) >= 1) by (bit_vector) + requires + low < 64, + ; + lemma_index_split(h as usize, 0, low, n as usize); + lemma_index_split(h as usize, (b - 1) as usize, low, n as usize); + assert((h + 1) * b == h * b + b) by (nonlinear_arith); + let blk = f.subrange(h * b, (h + 1) * b); + let pl = p.subrange(0, low as int); + let eh = eq_at(p.subrange(low as int, n as int), h as usize); + let inner = |l: int| e_mul(eq_at(pl, l as usize), blk[l]); + lemma_sum_scale(eh, inner, b); + assert forall|l: int| 0 <= l < b implies #[trigger] shifted(big, h * b)(l) == e_mul(eh, inner(l)) by { + lemma_mle_block_term(f, p, low, h, l); + } + lemma_sum_ext(shifted(big, h * b), |l: int| e_mul(eh, inner(l)), b); +} + +/// Binding the low `L` variables at once: `mle(f, p) = mle(rows, p[L..])` with `rows[h] = mle(block h of f, +/// p[..L])`, the blocks being the `2^(n - L)` runs of `2^L` entries. +pub proof fn lemma_mle_blocks(f: Seq, p: Seq, low: usize, rows: Seq) + requires + low <= p.len() < 64, + f.len() == (1usize << p.len()), + rows.len() == (1usize << ((p.len() - low) as usize)), + forall|h: int| 0 <= h < rows.len() ==> #[trigger] rows[h] == mle( + f.subrange(h * (1usize << low), (h + 1) * (1usize << low)), + p.subrange(0, low as int), + ), + ensures + mle(f, p) == mle(rows, p.subrange(low as int, p.len() as int)), +{ + let n = p.len(); + let ph = p.subrange(low as int, n as int); + let b = (1usize << low) as nat; + let a = rows.len(); + lemma_shl_sum(((n - low) as usize), low); + assert(a * b == f.len()); + let big = |x: int| e_mul(eq_at(p, x as usize), f[x]); + lemma_sum_blocks(big, a, b); + assert forall|h: int| 0 <= h < a implies #[trigger] block_sums(big, b)(h) == e_mul(eq_at(ph, h as usize), rows[h]) by { + lemma_mle_block_row(f, p, low, h); + } + lemma_sum_ext(block_sums(big, b), |h: int| e_mul(eq_at(ph, h as usize), rows[h]), a); +} + +/// A one-entry table over no variables is its entry. +proof fn lemma_mle_one(f: Seq, p: Seq) + requires + f.len() == 1, + p.len() == 0, + ensures + mle(f, p) == f[0], +{ + assert(eq_at(p, 0) == F192::ONE); + lemma_e_mul_one(f[0]); + lemma_e_add(f[0], F192::ZERO, F192::ZERO); + let g = |x: int| e_mul(eq_at(p, x as usize), f[x]); + assert(e_sum_fn(g, 0) == F192::ZERO); + assert(e_sum_fn(g, 1) == e_add(F192::ZERO, g(0))); +} + +/// The mixed fold: bind the lowest variable of a `K`-table to an +/// `E`-challenge, producing the `E`-table the remaining rounds fold. One +/// `mul_base` per output entry. +/// +/// Production maps `0..table.len() / 2` and collects, after a `debug_assert_eq!` on the parity (a `requires`). +fn fold_low_k(table: &[F64], chi: F192) -> (g: Vec) + requires + table.len() % 2 == 0, + ensures + g@ == fold_low(lift(table@), chi), +{ + let mut out: Vec = Vec::new(); + for i in 0..table.len() / 2 + invariant + table.len() % 2 == 0, + out@ == fold_low(lift(table@), chi).subrange(0, i as int), + { + out.push(interp_k(table[2 * i], table[2 * i + 1], chi)); + proof { + assert(out@ =~= fold_low(lift(table@), chi).subrange(0, i + 1)); + } + } + proof { + assert(out@ =~= fold_low(lift(table@), chi)); + } + out +} + +/// Bind the remaining variables of a half-folded `E`-table, LSB-first. +/// +/// `for &p in point` is a loop over the indices, and the `mut cur` argument is rebound. The table has one entry +/// per point of the cube, a `requires`. +fn fold_ladder(cur: Vec, point: &[F192]) -> (e: F192) + requires + point.len() < 64, + cur.len() == (1usize << point.len()), + ensures + e == mle(cur@, point@), +{ + let mut cur = cur; + let ghost n = point.len(); + let ghost orig = cur@; + let mut len = cur.len(); + proof { + assert(cur@.subrange(0, len as int) =~= orig); + assert(point@.subrange(0, n as int) =~= point@); + } + for k in 0..point.len() + invariant + n == point.len(), + n < 64, + cur.len() == orig.len(), + orig.len() == (1usize << n), + len == (1usize << ((n - k) as usize)), + len <= cur.len(), + mle(cur@.subrange(0, len as int), point@.subrange(k as int, n as int)) == mle(orig, point@), + { + let p = point[k]; + let ghost before = cur@.subrange(0, len as int); + let ghost all = cur@; + proof { + lemma_shl_double((n - k - 1) as usize); + lemma_mle_fold_low(before, point@.subrange(k as int, n as int)); + assert(point@.subrange(k as int, n as int).subrange(1, (n - k) as int) =~= point@.subrange(k + 1, n as int)); + } + len /= 2; + for i in 0..len + invariant + cur.len() == all.len(), + all.len() == orig.len(), + 2 * len <= all.len(), + before == all.subrange(0, 2 * len), + forall|j: int| 0 <= j < i ==> #[trigger] cur@[j] == interp_spec(before[2 * j], before[2 * j + 1], p), + forall|j: int| i <= j < all.len() ==> #[trigger] cur@[j] == all[j], + { + cur[i] = interp(cur[2 * i], cur[2 * i + 1], p); + } + proof { + assert(cur@.subrange(0, len as int) =~= fold_low(before, p)); + } + } + proof { + assert((1usize << 0usize) == 1) by (bit_vector); + lemma_mle_one(cur@.subrange(0, 1), point@.subrange(n as int, n as int)); + } + cur[0] +} + +/// The variables of the L1-resident low `eq` table of an MLE evaluation. +pub const MLE_LOW_VARS: usize = 10; + +/// `eq(r, .)` packed eight weights at a time for [`dot_base`]. Needs `r.len() >= 3`. +/// +/// Production maps `Weights8::new` over `as_chunks::<8>` of the table and collects; the copy loops over the +/// chunks. +fn packed_eq(r: &[F192]) -> (w: Vec) + requires + 3 <= r.len() < 64, + ensures + 8 * w.len() == (1usize << r.len()), + forall|x: int| 0 <= x < (1usize << r.len()) ==> #[trigger] w8_get(w@[x / 8], x % 8) == eq_at(r@, x as usize), +{ + let t = eq_table(r); + let ghost n = r.len(); + proof { + assert((1usize << n) % 8 == 0) by (bit_vector) + requires + 3 <= n < 64, + ; + } + let mut out: Vec = Vec::new(); + for c in 0..t.len() / 8 + invariant + n == r.len(), + 3 <= n < 64, + is_eq_table(t@, r@, F192::ONE), + t.len() % 8 == 0, + out.len() == c, + forall|x: int| 0 <= x < 8 * c ==> #[trigger] w8_get(out@[x / 8], x % 8) == t@[x], + { + let chunk = [t[8 * c], t[8 * c + 1], t[8 * c + 2], t[8 * c + 3], t[8 * c + 4], t[8 * c + 5], t[8 * c + 6], t[8 * c + 7]]; + out.push(Weights8::new(&chunk)); + proof { + assert forall|x: int| 0 <= x < 8 * (c + 1) implies #[trigger] w8_get(out@[x / 8], x % 8) == t@[x] by { + if x >= 8 * c { + assert(x / 8 == c && x % 8 == x - 8 * c) by (nonlinear_arith) + requires + 8 * c <= x < 8 * c + 8, + ; + } else { + assert(x / 8 < c) by (nonlinear_arith) + requires + 0 <= x < 8 * c, + ; + } + } + } + } + proof { + assert forall|x: int| 0 <= x < (1usize << n) implies #[trigger] w8_get(out@[x / 8], x % 8) == eq_at(r@, x as usize) by { + lemma_e_mul_one(eq_at(r@, x as usize)); + } + } + out +} + +/// The packed dot product is the `eq`-weighted sum of the lifted row. +proof fn lemma_dot_is_mle(w: Seq, row: Seq, pl: Seq, k: nat) + requires + pl.len() < 64, + k <= row.len(), + row.len() == (1usize << pl.len()), + 8 * w.len() == row.len(), + forall|x: int| 0 <= x < row.len() ==> #[trigger] w8_get(w[x / 8], x % 8) == eq_at(pl, x as usize), + ensures + dot_spec(w, row, k) == e_sum_fn(|x: int| e_mul(eq_at(pl, x as usize), lift(row)[x]), k), + decreases k, +{ + if k > 0 { + lemma_dot_is_mle(w, row, pl, (k - 1) as nat); + } +} + +/// Evaluate the MLE of a `K`-valued truth table at an `E`-point (length `log2(len)`). +/// +/// The `eq` weights factor into a low and a high table: +/// +/// ```text +/// f(point) = sum_h eq(point_high, h) * sum_l eq(point_low, l) * f[h * 2^L + l] +/// ``` +/// +/// Each row's inner sum is one [`dot_base`] against the packed low table, reduced once. +/// The table is read once and never lifted into `E`. +/// +/// Production's `debug_assert_eq!` on the length is a `requires`; `match point.split_first()` tests the length; +/// `point.len().min(MLE_LOW_VARS)` is an `if`; `chunks_exact(..).map(..).collect()` is a loop pushing each row's +/// reduced dot product. +pub fn mle_eval(table: &[F64], point: &[F192]) -> (e: F192) + requires + point.len() < 64, + table.len() == (1usize << point.len()), + ensures + e == mle(lift(table@), point@), +{ + let ghost n = point.len(); + let ghost f = lift(table@); + if point.len() < 3 { + if point.len() == 0 { + proof { + assert((1usize << 0usize) == 1) by (bit_vector); + lemma_mle_one(f, point@); + } + return F192::from(table[0]); + } + let p0 = point[0]; + let rest = &point[1..]; + proof { + lemma_shl_double((n - 1) as usize); + lemma_mle_fold_low(f, point@); + assert(fold_low(f, p0).len() == table.len() / 2); + } + return fold_ladder(fold_low_k(table, p0), rest); + } + let low_vars = if point.len() < MLE_LOW_VARS { point.len() } else { MLE_LOW_VARS }; + let low = packed_eq(&point[..low_vars]); + let width = 1usize << low_vars; + let ghost pl = point@.subrange(0, low_vars as int); + let ghost hl = (n - low_vars) as usize; + proof { + lemma_shl_sum(hl, low_vars); + assert((1usize << low_vars) >= 1) by (bit_vector) + requires + low_vars < 64, + ; + lemma_div_by_multiple((1usize << hl) as int, width as int); + } + let mut rows: Vec = Vec::new(); + let mut h = 0; + while h < table.len() / width + invariant + n == point.len(), + 3 <= n < 64, + low_vars <= n, + hl == n - low_vars, + width == (1usize << low_vars), + width >= 1, + table.len() == (1usize << n), + (1usize << hl) * width == table.len(), + table.len() / width == (1usize << hl), + pl == point@.subrange(0, low_vars as int), + f == lift(table@), + 8 * low.len() == width, + forall|x: int| 0 <= x < width ==> #[trigger] w8_get(low@[x / 8], x % 8) == eq_at(pl, x as usize), + h <= table.len() / width, + rows.len() == h, + forall|j: int| 0 <= j < h ==> #[trigger] rows@[j] == mle(f.subrange(j * width, (j + 1) * width), pl), + decreases table.len() / width - h, + { + proof { + assert(h * width <= (h + 1) * width <= table.len() && (h + 1) * width - h * width == width) by (nonlinear_arith) + requires + h < (1usize << hl), + (1usize << hl) * width == table.len(), + ; + } + let row = &table[h * width..(h + 1) * width]; + assert(row@ == table@.subrange(h * width, (h + 1) * width)); + let d = dot_base(&low, row).reduce(); + proof { + lemma_dot_is_mle(low@, row@, pl, width as nat); + assert(lift(row@) =~= f.subrange(h * width, (h + 1) * width)); + } + rows.push(d); + h += 1; + } + proof { + lemma_mle_blocks(f, point@, low_vars, rows@); + } + fold_ladder(rows, &point[low_vars..]) +} + +// --------------------------------------------------------------------------------------------- +// The skip domain's shared barycentric weight +// --------------------------------------------------------------------------------------------- +/// `prod_{k=1}^{n-1} φ₈(k)` in `K`. +pub open spec fn phi8_prod(n: nat) -> u64 + decreases n, +{ + if n <= 1 { + 1 + } else { + k_mul(phi8_prod((n - 1) as nat), phi8((n - 1) as u8)) + } +} + +/// The weight of a window of `2^log` nodes: `(prod_{k=1}^{2^log - 1} φ₈(k))^(2^64 - 2)`, its inverse in `K` +/// ([`lemma_denominator_inverts`]). +pub open spec fn denominator_spec(log: nat) -> F192 { + e_from_k(k_pow(phi8_prod(pow2(log)), (pow2(64) - 2) as nat)) +} + +/// The product in `K` the table is built with. +/// +/// A `const fn` inside the `DENOMINATORS` initializer in production; Verus wants it at module level. +const fn mul(a: u64, b: u64) -> (r: u64) + ensures + r == k_mul(a, b), +{ + reduce(crate::gf2_64::software::clmul(a, b)) +} + +/// The barycentric weight every node of an aligned `size`-node window of the φ₈ table shares, +/// `1 / ∏_{k≠0} φ₈(k)`. φ₈ is F2-linear on its index, so `nodes[a] + nodes[b] = φ₈(a ^ b)` (the window's +/// offset cancels) and `b ↦ a ^ b` only permutes the window, leaving every node the same product. +/// Computed once for every window size. +/// +/// Production's `debug_assert!` that `size` is a power of two at most 256 is a `requires`; `usize::trailing_zeros` +/// (which vstd does not specify) is `u64::trailing_zeros` of the same value. +pub fn window_denominator(size: usize) -> (d: F192) + requires + exists|log: nat| log <= 8 && size == #[trigger] pow2(log), + ensures + forall|log: nat| log <= 8 && size == #[trigger] pow2(log) ==> d == denominator_spec(log), +{ + let ghost log = choose|log: nat| log <= 8 && size == #[trigger] pow2(log); + let tz = (size as u64).trailing_zeros(); + proof { + lemma_pow2_injective_upto8(log); + lemma_pow2_tz(size as u64, log); + } + DENOMINATORS[tz as usize] +} + +/// The trailing zeros of `2^log` are `log`. +proof fn lemma_pow2_tz(x: u64, log: nat) + requires + log <= 8, + x == pow2(log), + ensures + x.trailing_zeros() == log, +{ + lemma_pow2_shl(log); + let t = x.trailing_zeros(); + broadcast use vstd::std_specs::bits::axiom_u64_trailing_zeros; + let l = log as u64; + assert(x == (1u64 << l)); + lemma_pow2_pos(log); + assert(x != 0); + assert(t < 64); + assert((x >> (t as u64)) & 1u64 == 1u64); + assert(forall|j: u64| 0 <= j < t ==> #[trigger] (x >> j) & 1u64 == 0u64); + if (t as u64) < l { + assert((x >> (t as u64)) & 1u64 == 0u64) by (bit_vector) + requires + x == (1u64 << l), + (t as u64) < l, + l <= 8, + ; + } else if (t as u64) > l { + assert((x >> l) & 1u64 == 1u64) by (bit_vector) + requires + x == (1u64 << l), + l <= 8, + ; + } +} + +proof fn lemma_pow2_shl(log: nat) + requires + log <= 8, + ensures + pow2(log) == (1u64 << (log as u64)), + pow2(log) == (1usize << (log as usize)), + pow2(log) <= 256, +{ + lemma2_to64(); + let l = log as u64; + assert(l <= 8 ==> (1u64 << l) == (if l == 0 { 1u64 } else if l == 1 { 2 } else if l == 2 { 4 } else if l == 3 { 8 } else if l == 4 { 16 } else if l == 5 { 32 } else if l == 6 { 64 } else if l == 7 { 128 } else { 256 })) by (bit_vector); + let u = log as usize; + assert(u <= 8 ==> (1usize << u) == (if u == 0 { 1usize } else if u == 1 { 2 } else if u == 2 { 4 } else if u == 3 { 8 } else if u == 4 { 16 } else if u == 5 { 32 } else if u == 6 { 64 } else if u == 7 { 128 } else { 256 })) by (bit_vector); +} + +proof fn lemma_pow2_injective_upto8(log: nat) + requires + log <= 8, + ensures + forall|m: nat| m <= 8 && #[trigger] pow2(m) == pow2(log) ==> m == log, +{ + lemma2_to64(); +} + +/// [`window_denominator`] for every window size, at compile time. The nodes lie in `F64`, so the +/// product and its inverse (Fermat, `a^(2^64 - 2)`) stay there. +/// +/// Verus's `exec const` form, to state what the table holds; the body is production's. +pub exec const DENOMINATORS: [F192; 9] + ensures + forall|log: int| 0 <= log < 9 ==> #[trigger] DENOMINATORS[log] == denominator_spec(log as nat), +{ + let mut out = [F192::ZERO; 9]; + let mut log = 0; + while log < out.len() + invariant + log <= 9, + forall|m: int| 0 <= m < log ==> #[trigger] out[m] == denominator_spec(m as nat), + decreases 9 - log, + { + proof { + lemma_pow2_shl(log as nat); + lemma_pow2_pos(log as nat); + } + let mut product = 1; + let mut k = 1; + while k < 1 << log + invariant + log < 9, + (1usize << log) == pow2(log as nat), + (1usize << log) <= 256, + 1 <= k <= (1usize << log), + product == phi8_prod(k as nat), + decreases (1usize << log) - k, + { + product = mul(product, PHI_8_TABLE_192[k].c0); + k += 1; + } + let ghost p = product; + proof { + lemma2_to64(); + assert(p == k_pow(p, 1)) by { + lemma_k_mul_one(p); + reveal_with_fuel(k_pow, 2); + } + } + let (mut inverse, mut bit) = (1, 1); + while bit < 64 + invariant + 1 <= bit <= 64, + product == k_pow(p, pow2((bit - 1) as nat)), + inverse == k_pow(p, (pow2(bit as nat) - 2) as nat), + pow2(bit as nat) >= 2, + decreases 64 - bit, + { + proof { + lemma_pow2_unfold(bit as nat); + lemma_pow2_unfold((bit + 1) as nat); + lemma_k_pow_add(p, pow2((bit - 1) as nat), pow2((bit - 1) as nat)); + lemma_k_pow_add(p, (pow2(bit as nat) - 2) as nat, pow2(bit as nat)); + } + product = mul(product, product); + inverse = mul(inverse, product); + bit += 1; + } + out[log] = F192::new(inverse, 0, 0); + log += 1; + } + out +} + +/// The weight inverts the product of the nonzero nodes of its window: `D * prod_{k=1}^{2^log - 1} φ₈(k) = 1`. +pub proof fn lemma_denominator_inverts(log: nat) + requires + log <= 8, + ensures + phi8_prod(pow2(log)) != 0, + e_mul(denominator_spec(log), e_from_k(phi8_prod(pow2(log)))) == F192::ONE, +{ + lemma_pow2_shl(log); + lemma_phi8_prod_nonzero(pow2(log)); + let p = phi8_prod(pow2(log)); + let w = k_pow(p, (pow2(64) - 2) as nat); + lemma_k_inverse(p); + lemma_k_mul_comm(p, w); + lemma_e_from_k_mul(w, p); +} + +/// A product of nonzero nodes is nonzero. +pub proof fn lemma_phi8_prod_nonzero(n: nat) + requires + n <= 256, + ensures + phi8_prod(n) != 0, + decreases n, +{ + if n > 1 { + lemma_phi8_prod_nonzero((n - 1) as nat); + lemma_phi8_nonzero((n - 1) as u8); + if phi8_prod(n) == 0 { + crate::ntt::lemma_k_no_zero_divisors(phi8_prod((n - 1) as nat), phi8((n - 1) as u8)); + } + } +} + +} // verus! diff --git a/verification/verus/src/neon.rs b/verification/verus/src/neon.rs new file mode 100644 index 0000000000..61e92000e5 --- /dev/null +++ b/verification/verus/src/neon.rs @@ -0,0 +1,45 @@ +//! The AArch64 helper shared by the NEON kernels, `crates/primitives/src/field/neon.rs`, copied with the same +//! bodies. +#[cfg(verus_keep_ghost)] +use crate::intrinsics::aarch64::*; +use core::arch::aarch64::*; +use vstd::prelude::*; + +verus! { + +/// Three-way XOR of 64-bit-lane vectors. +/// +/// # Safety +/// Requires the `sha3` target feature on the EOR3 arm (statically satisfied by +/// the `cfg` gate); the fallback arm has no requirement. +#[cfg(target_feature = "sha3")] +#[inline(always)] +pub unsafe fn xor3_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == u64x2(a)[0] ^ u64x2(b)[0] ^ u64x2(c)[0], + u64x2(r)[1] == u64x2(a)[1] ^ u64x2(b)[1] ^ u64x2(c)[1], +{ + // SAFETY: `sha3` is statically enabled by the cfg gate. + unsafe { veor3q_u64(a, b, c) } +} + +/// Two-`EOR` fallback for targets without the SHA3 extension. +/// +/// # Safety +/// No requirements; `unsafe` only to match the EOR3 arm's signature. +#[cfg(not(target_feature = "sha3"))] +#[inline(always)] +pub unsafe fn xor3_u64(a: uint64x2_t, b: uint64x2_t, c: uint64x2_t) -> (r: uint64x2_t) + ensures + u64x2(r)[0] == u64x2(a)[0] ^ u64x2(b)[0] ^ u64x2(c)[0], + u64x2(r)[1] == u64x2(a)[1] ^ u64x2(b)[1] ^ u64x2(c)[1], +{ + proof { + let (a0, b0, c0, a1, b1, c1) = (u64x2(a)[0], u64x2(b)[0], u64x2(c)[0], u64x2(a)[1], u64x2(b)[1], u64x2(c)[1]); + assert(a0 ^ (b0 ^ c0) == a0 ^ b0 ^ c0 && a1 ^ (b1 ^ c1) == a1 ^ b1 ^ c1) by (bit_vector); + } + // SAFETY: NEON is part of the aarch64 baseline; registers only. + unsafe { veorq_u64(a, veorq_u64(b, c)) } +} + +} // verus! diff --git a/verification/verus/src/ntt.rs b/verification/verus/src/ntt.rs new file mode 100644 index 0000000000..0feac0d5db --- /dev/null +++ b/verification/verus/src/ntt.rs @@ -0,0 +1,3093 @@ +//! The additive NTT over `K = GF(2^64)` in the Lin-Chung-Han novel polynomial basis. +//! +//! The executable functions are the portable scalar parts of `crates/pcs/src/ntt/additive_ntt_f64.rs`, +//! copied with the same names and bodies where Verus accepts them; `tests/equivalence/ntt.rs` checks +//! the copies against production. +//! +//! Specification, following annex `d` of the leanVM document: +//! +//! - [`span`]: the subset sum `Σ_j bit_j(idx) · s[j]` of a sequence of field elements. +//! - [`subspace_poly`]: the unnormalized subspace polynomial `s_i`, by its recurrence +//! `s_0(x) = x`, `s_i(x) = s_{i-1}(x) · (s_{i-1}(x) + s_{i-1}(b_{i-1}))`, evaluated in `K`. +//! - [`normalized_subspace_poly`]: `Ŵ_i(x) = s_i(x) · s_i(b_i)^(-1)`, with the inverse `y^(2^64 - 2)`. +//! - [`layer_map`]: one layer of butterflies on a buffer of interleaved lanes, by row index; +//! [`forward_layers`] and [`inverse_layers`] compose them. +//! - [`novel_eval`]: the novel-basis polynomial `Σ_j a_j X_j(x)`, `X_j = Π_i Ŵ_i(x)^(bit_i(j))`, in its +//! even-odd form; [`lemma_novel_eval_flat`] proves it equal to the flat sum [`novel_sum`]. +//! +//! Main results: [`lemma_inverse_butterfly`], [`lemma_span_xor`], `twiddles_radix8`'s postcondition, +//! [`lemma_subspace_poly_additive`], [`lemma_twiddle_is_subspace_poly`], `generate_evals_from_subspace`'s +//! postcondition, `radix8_butterflies`' postcondition, [`lemma_inverse_after_forward`], +//! [`lemma_forward_evaluates`] and [`lemma_encode_evaluates`]. The last two hold for any basis whose table +//! rows start with one (`Ŵ_i(b_i) = 1`); [`lemma_standard_rows_start_with_one`] proves that for the standard +//! basis, so [`lemma_standard_forward_evaluates`] and [`lemma_standard_encode_evaluates`] state them for +//! `AdditiveNttF64::standard(dim)` without hypothesis. +//! +//! The parallel driver (`transform`, `gathered_pass`, `run_layers`, `fused_rows`, `replicate`, +//! `transpose_lane_major`) is not copied: it is out of scope. The SIMD butterflies `lane_butterflies` dispatches +//! to are in `crate::ntt_simd`. +// `global size_of usize` expands to a braced block. +#![allow(unused_braces)] + +use crate::gf2_64::*; +use crate::ntt_simd::*; +use core::ops::{Add, AddAssign, Mul, MulAssign}; +use vstd::arithmetic::div_mod::*; +use vstd::arithmetic::mul::*; +use vstd::arithmetic::power2::*; +use vstd::bits::*; +use vstd::prelude::*; + +verus! { + +global size_of usize == 8; + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// Bit `j` of `idx`, zero past the word. +pub open spec fn idx_bit(idx: usize, j: nat) -> bool { + j < 64 && (idx >> (j as usize)) & 1 == 1 +} + +/// `Σ_j bit_j(idx) · s[j]`: the XOR of the elements of `s` whose index is a set bit of `idx`. +pub open spec fn span(s: Seq, idx: usize) -> u64 + decreases s.len(), +{ + if s.len() == 0 { + 0 + } else { + span(s.drop_last(), idx) ^ (if idx_bit(idx, (s.len() - 1) as nat) { + s.last().0 + } else { + 0 + }) + } +} + +/// The unnormalized subspace polynomial `s_i` of the basis, evaluated at `x`: +/// `s_0(x) = x` and `s_i(x) = s_{i-1}(x) · (s_{i-1}(x) + s_{i-1}(b_{i-1}))`. +pub open spec fn subspace_poly(basis: Seq, i: nat, x: u64) -> u64 + decreases i, +{ + if i == 0 { + x + } else { + let p = subspace_poly(basis, (i - 1) as nat, x); + k_mul(p, p ^ subspace_poly(basis, (i - 1) as nat, basis[i - 1].0)) + } +} + +/// The inverse in `K`, `y^(2^64 - 2)` (zero for zero), as `F64::inv` computes it. +pub open spec fn k_inv(y: u64) -> u64 { + k_pow(y, (pow2(64) - 2) as nat) +} + +/// The normalized subspace polynomial `Ŵ_i(x) = s_i(x) / s_i(b_i)`. +pub open spec fn normalized_subspace_poly(basis: Seq, i: nat, x: u64) -> u64 { + k_mul(subspace_poly(basis, i, x), k_inv(subspace_poly(basis, i, basis[i as int].0))) +} + +/// The standard basis `{1, x, x^2, ..., x^(dim-1)}` of the domain: `b_c = x^c`, bit `c` alone. +pub open spec fn standard_basis(dim: nat) -> Seq { + Seq::new(dim, |c: int| F64(1u64 << (c as u64))) +} + +/// The twiddle of one block at one layer, as read from the table `tab`: a subset sum of row +/// `L - layer - 1` past its first entry, `L` the number of rows. +pub open spec fn twiddle_spec(tab: Seq>, layer: nat, block: usize) -> u64 { + span(tab[tab.len() - layer - 1].skip(1), block) +} + +/// The forward butterfly's top output: `u + v t`. +pub open spec fn bf_top(u: u64, v: u64, t: u64) -> u64 { + u ^ k_mul(v, t) +} + +/// The forward butterfly's bottom output: `v + (u + v t)`. +pub open spec fn bf_bot(u: u64, v: u64, t: u64) -> u64 { + v ^ bf_top(u, v, t) +} + +/// The inverse butterfly's top output: `u + (v + u) t`. +pub open spec fn ibf_top(u: u64, v: u64, t: u64) -> u64 { + u ^ k_mul(v ^ u, t) +} + +/// The inverse butterfly's bottom output: `v + u`. +pub open spec fn ibf_bot(u: u64, v: u64, t: u64) -> u64 { + v ^ u +} + +// --------------------------------------------------------------------------------------------- +// Butterflies +// --------------------------------------------------------------------------------------------- +proof fn lemma_xor_facts(a: u64, b: u64, c: u64) + ensures + a ^ a == 0, + a ^ 0 == a, + 0 ^ a == a, + a ^ b == b ^ a, + (a ^ b) ^ c == a ^ (b ^ c), + (a ^ b) ^ b == a, + (a ^ b) ^ a == b, + a ^ (a ^ b) == b, + b ^ (a ^ b) == a, +{ + assert(a ^ a == 0 && a ^ 0 == a && 0 ^ a == a && a ^ b == b ^ a && (a ^ b) ^ c == a ^ (b ^ c) && (a ^ b) ^ b == a + && (a ^ b) ^ a == b && a ^ (a ^ b) == b && b ^ (a ^ b) == a) by (bit_vector); +} + +/// The forward butterfly evaluates the line `u + v X` at `X = t` and `X = t + 1`. +pub proof fn lemma_butterfly_evaluates(u: u64, v: u64, t: u64) + ensures + bf_top(u, v, t) == u ^ k_mul(v, t), + bf_bot(u, v, t) == u ^ k_mul(v, t ^ 1), +{ + lemma_k_mul_xor_right(v, t, 1); + lemma_k_mul_one(v); + let p = k_mul(v, t); + assert(v ^ (u ^ p) == u ^ (p ^ v)) by (bit_vector); +} + +/// The inverse butterfly of `inverse_transform` undoes the forward one, for every twiddle. +pub proof fn lemma_inverse_butterfly(u: u64, v: u64, t: u64) + ensures + ibf_top(bf_top(u, v, t), bf_bot(u, v, t), t) == u, + ibf_bot(bf_top(u, v, t), bf_bot(u, v, t), t) == v, + bf_top(ibf_top(u, v, t), ibf_bot(u, v, t), t) == u, + bf_bot(ibf_top(u, v, t), ibf_bot(u, v, t), t) == v, +{ + let p = k_mul(v, t); + let a = u ^ p; + lemma_xor_facts(v, a, 0); + lemma_xor_facts(u, p, 0); + let q = k_mul(v ^ u, t); + lemma_xor_facts(u, q, 0); + lemma_xor_facts(v, u, 0); + lemma_xor_facts(u ^ q, v ^ u, 0); + assert((v ^ u) ^ (u ^ q) ^ (u ^ q) == v ^ u) by (bit_vector); + assert((v ^ u) ^ (u ^ q) == v ^ q) by (bit_vector); +} + +/// The transposed butterfly undoes the forward one with its rows swapped, input and output. +pub proof fn lemma_transposed_inverts_swapped(u: u64, v: u64, t: u64) + ensures + ({ + let (a, b) = (bf_top(u, v, t), bf_bot(u, v, t)); + let s = b ^ a; + s == v && a ^ k_mul(s, t) == u + }), +{ + let p = k_mul(v, t); + lemma_xor_facts(u, p, 0); + lemma_xor_facts(v, u ^ p, 0); +} + +/// One lane's butterfly, forward or transposed. +#[inline(always)] +pub fn butterfly_one(u: &mut F64, v: &mut F64, t: F64) + ensures + TRANSPOSED ==> final(u).0 == old(u).0 ^ old(v).0 && final(v).0 == old(v).0 ^ k_mul( + old(u).0 ^ old(v).0, + t.0, + ), + !TRANSPOSED ==> final(u).0 == bf_top(old(u).0, old(v).0, t.0) && final(v).0 == bf_bot( + old(u).0, + old(v).0, + t.0, + ), +{ + if TRANSPOSED { + let s = *u + *v; + *u = s; + *v += s * t; + } else { + *u += *v * t; + *v += *u; + } +} + +// --------------------------------------------------------------------------------------------- +// Subset sums of the basis +// --------------------------------------------------------------------------------------------- +proof fn lemma_bit_xor(i: usize, j: usize, k: usize) + requires + k < 64, + ensures + ((i ^ j) >> k) & 1 == 1 <==> (((i >> k) & 1 == 1) != ((j >> k) & 1 == 1)), +{ + assert(((i ^ j) >> k) & 1 == 1 <==> (((i >> k) & 1 == 1) != ((j >> k) & 1 == 1))) by (bit_vector) + requires + k < 64, + ; +} + +proof fn lemma_xor_select(a: u64, b: u64, x: u64) + ensures + (a ^ b) ^ 0 == (a ^ 0) ^ (b ^ 0), + (a ^ b) ^ x == (a ^ x) ^ (b ^ 0), + (a ^ b) ^ x == (a ^ 0) ^ (b ^ x), + (a ^ b) ^ 0 == (a ^ x) ^ (b ^ x), +{ + assert((a ^ b) ^ 0 == (a ^ 0) ^ (b ^ 0) && (a ^ b) ^ x == (a ^ x) ^ (b ^ 0) && (a ^ b) ^ x == (a ^ 0) ^ (b + ^ x) && (a ^ b) ^ 0 == (a ^ x) ^ (b ^ x)) by (bit_vector); +} + +/// The subset sum is F_2-linear in the index: `span(s, i ^ j) = span(s, i) + span(s, j)`. +pub proof fn lemma_span_xor(s: Seq, i: usize, j: usize) + ensures + span(s, i ^ j) == span(s, i) ^ span(s, j), + decreases s.len(), +{ + if s.len() == 0 { + assert(0u64 ^ 0u64 == 0) by (bit_vector); + } else { + let d = s.drop_last(); + let k = (s.len() - 1) as nat; + lemma_span_xor(d, i, j); + if k < 64 { + lemma_bit_xor(i, j, k as usize); + } + lemma_xor_select(span(d, i), span(d, j), s.last().0); + } +} + +/// The subset sum of the zero index is zero. +pub proof fn lemma_span_zero(s: Seq) + ensures + span(s, 0) == 0, + decreases s.len(), +{ + if s.len() > 0 { + lemma_span_zero(s.drop_last()); + let k = (s.len() - 1) as usize; + assert((0usize >> k) & 1 == 0) by (bit_vector); + assert(0u64 ^ 0u64 == 0) by (bit_vector); + } +} + +proof fn lemma_bit_double(b: usize, k: usize) + requires + b < 0x8000_0000_0000_0000usize, + k < 63, + ensures + ((b * 2) as usize >> ((k + 1) as usize)) & 1 == (b >> k) & 1, + ((b * 2 + 1) as usize >> ((k + 1) as usize)) & 1 == (b >> k) & 1, + ((b * 2) as usize >> 0usize) & 1 == 0, + ((b * 2 + 1) as usize >> 0usize) & 1 == 1, +{ + let b2 = (b * 2) as usize; + let b21 = (b * 2 + 1) as usize; + assert(b2 == b << 1usize && b21 == (b << 1usize) | 1) by (bit_vector) + requires + b < 0x8000_0000_0000_0000usize, + b2 == b * 2, + b21 == b * 2 + 1, + ; + assert(((b << 1usize) >> ((k + 1) as usize)) & 1 == (b >> k) & 1 && (((b << 1usize) | 1) >> ((k + 1) as usize)) + & 1 == (b >> k) & 1 && ((b << 1usize) >> 0usize) & 1 == 0 && (((b << 1usize) | 1) >> 0usize) & 1 == 1) + by (bit_vector) + requires + k < 63, + ; +} + +/// Doubling the index shifts the subset sum by one element; the low bit selects the first element. +pub proof fn lemma_span_double(s: Seq, b: usize) + requires + 1 <= s.len() <= 64, + b < 0x8000_0000_0000_0000usize, + ensures + span(s, (b * 2) as usize) == span(s.skip(1), b), + span(s, (b * 2 + 1) as usize) == s[0].0 ^ span(s.skip(1), b), + decreases s.len(), +{ + let n = (s.len() - 1) as nat; + let d = s.drop_last(); + if n == 0 { + lemma_bit_double(b, 0); + assert(span(d, (b * 2) as usize) == 0); + assert(span(d, (b * 2 + 1) as usize) == 0); + assert(span(s.skip(1), b) == 0); + lemma_xor_facts(s[0].0, 0, 0); + lemma_xor_facts(0, 0, 0); + assert(span(s, (b * 2) as usize) == span(d, (b * 2) as usize) ^ 0); + } else { + lemma_span_double(d, b); + lemma_bit_double(b, (n - 1) as usize); + assert(s.skip(1).drop_last() =~= d.skip(1)); + assert(s.skip(1).last() == s.last()); + let sel = if idx_bit(b, (n - 1) as nat) { + s.last().0 + } else { + 0 + }; + lemma_xor_facts(s[0].0, span(d.skip(1), b), sel); + assert(span(s.skip(1), b) == span(d.skip(1), b) ^ sel); + assert(span(s, (b * 2) as usize) == span(d, (b * 2) as usize) ^ sel); + } +} + +/// Index bits past `m` select nothing: a subset sum of an index below `2^m` reads only `s[..m]`. +pub proof fn lemma_span_low(s: Seq, b: usize, m: nat) + requires + m <= s.len(), + b < pow2(m), + ensures + span(s, b) == span(s.take(m as int), b), + decreases s.len(), +{ + if s.len() == m { + assert(s.take(m as int) =~= s); + } else { + let k = (s.len() - 1) as nat; + lemma_span_low(s.drop_last(), b, m); + assert(s.drop_last().take(m as int) =~= s.take(m as int)); + if k < 64 { + lemma_usize_shr_is_div(b, k as usize); + if m < k { + lemma_pow2_strictly_increases(m, k); + } + lemma_basic_div(b as int, pow2(k) as int); + assert((0usize & 1) == 0) by (bit_vector); + } + lemma_xor_facts(span(s.drop_last(), b), 0, 0); + } +} + +/// `Σ_j bit_j(idx) · basis[j]`. +#[inline] +pub fn span_get(basis: &[F64], idx: usize) -> (r: F64) + requires + basis.len() <= 64, + ensures + r.0 == span(basis@, idx), +{ + let mut acc = F64::ZERO; + for j in 0..basis.len() + invariant + basis.len() <= 64, + acc.0 == span(basis@.take(j as int), idx), + { + let b = basis[j]; + assert(basis@.take(j + 1).drop_last() =~= basis@.take(j as int)); + proof { + lemma_xor_facts(acc.0, 0, 0); + } + if (idx >> j) & 1 == 1 { + acc += b; + } + } + assert(basis@.take(basis.len() as int) =~= basis@); + acc +} + +// --------------------------------------------------------------------------------------------- +// Subspace polynomials +// --------------------------------------------------------------------------------------------- +/// `s_i` is F_2-linear: `s_i(x + y) = s_i(x) + s_i(y)`. +pub proof fn lemma_subspace_poly_additive(basis: Seq, i: nat, x: u64, y: u64) + ensures + subspace_poly(basis, i, x ^ y) == subspace_poly(basis, i, x) ^ subspace_poly(basis, i, y), + decreases i, +{ + if i > 0 { + let j = (i - 1) as nat; + lemma_subspace_poly_additive(basis, j, x, y); + let a = subspace_poly(basis, j, x); + let b = subspace_poly(basis, j, y); + let c = subspace_poly(basis, j, basis[j as int].0); + // (a + b)(a + b + c) = a(a + c) + a b + b(b + c) + b a + assert((a ^ b) ^ c == (a ^ c) ^ b && (a ^ b) ^ c == (b ^ c) ^ a) by (bit_vector); + lemma_k_mul_xor_left(a, b, (a ^ b) ^ c); + lemma_k_mul_xor_right(a, a ^ c, b); + lemma_k_mul_xor_right(b, b ^ c, a); + lemma_k_mul_comm(a, b); + let (p, q, r) = (k_mul(a, a ^ c), k_mul(b, b ^ c), k_mul(a, b)); + assert((p ^ r) ^ (q ^ r) == p ^ q) by (bit_vector); + } +} + +/// `s_i(0) = 0`. +pub proof fn lemma_subspace_poly_zero(basis: Seq, i: nat) + ensures + subspace_poly(basis, i, 0) == 0, +{ + lemma_subspace_poly_additive(basis, i, 0, 0); + let z = subspace_poly(basis, i, 0); + assert(0u64 ^ 0u64 == 0u64 && z ^ z == 0) by (bit_vector); +} + +/// `s_i` vanishes on the first `i` basis elements. +pub proof fn lemma_subspace_poly_vanishes(basis: Seq, i: nat, j: nat) + requires + j < i, + ensures + subspace_poly(basis, i, basis[j as int].0) == 0, + decreases i, +{ + let x = basis[j as int].0; + let p = subspace_poly(basis, (i - 1) as nat, x); + let c = subspace_poly(basis, (i - 1) as nat, basis[i - 1].0); + if j == i - 1 { + lemma_xor_facts(p, 0, 0); + lemma_k_mul_zero(p); + } else { + lemma_subspace_poly_vanishes(basis, (i - 1) as nat, j); + lemma_k_mul_zero(p ^ c); + } +} + +/// So `s_i` vanishes on the whole span of the first `i` basis elements. +pub proof fn lemma_subspace_poly_vanishes_on_span(basis: Seq, i: nat, idx: usize) + requires + i <= basis.len(), + ensures + subspace_poly(basis, i, span(basis.take(i as int), idx)) == 0, +{ + lemma_subspace_poly_kills_prefix(basis, i, basis.take(i as int), idx); +} + +proof fn lemma_subspace_poly_kills_prefix(basis: Seq, i: nat, s: Seq, idx: usize) + requires + s.len() <= i <= basis.len(), + forall|j: int| 0 <= j < s.len() ==> s[j] == basis[j], + ensures + subspace_poly(basis, i, span(s, idx)) == 0, + decreases s.len(), +{ + if s.len() == 0 { + lemma_subspace_poly_zero(basis, i); + } else { + let d = s.drop_last(); + lemma_subspace_poly_kills_prefix(basis, i, d, idx); + let sel = if idx_bit(idx, (s.len() - 1) as nat) { + s.last().0 + } else { + 0 + }; + lemma_subspace_poly_additive(basis, i, span(d, idx), sel); + if idx_bit(idx, (s.len() - 1) as nat) { + lemma_subspace_poly_vanishes(basis, i, (s.len() - 1) as nat); + } else { + lemma_subspace_poly_zero(basis, i); + } + lemma_xor_facts(0, 0, 0); + } +} + +/// The normalized `Ŵ_i` is F_2-linear too. +pub proof fn lemma_normalized_additive(basis: Seq, i: nat, x: u64, y: u64) + ensures + normalized_subspace_poly(basis, i, x ^ y) == normalized_subspace_poly(basis, i, x) + ^ normalized_subspace_poly(basis, i, y), + normalized_subspace_poly(basis, i, 0) == 0, +{ + lemma_subspace_poly_additive(basis, i, x, y); + let c = k_inv(subspace_poly(basis, i, basis[i as int].0)); + lemma_k_mul_xor_left(subspace_poly(basis, i, x), subspace_poly(basis, i, y), c); + lemma_subspace_poly_zero(basis, i); + lemma_k_mul_zero(c); +} + +/// A subset sum of `Ŵ_i` values is `Ŵ_i` of the subset sum. +pub proof fn lemma_span_normalized(basis: Seq, i: nat, s: Seq, t: Seq, idx: usize) + requires + s.len() == t.len(), + forall|j: int| 0 <= j < s.len() ==> s[j].0 == normalized_subspace_poly(basis, i, #[trigger] t[j].0), + ensures + span(s, idx) == normalized_subspace_poly(basis, i, span(t, idx)), + decreases s.len(), +{ + lemma_normalized_additive(basis, i, 0, 0); + if s.len() > 0 { + lemma_span_normalized(basis, i, s.drop_last(), t.drop_last(), idx); + let sel = if idx_bit(idx, (s.len() - 1) as nat) { + t.last().0 + } else { + 0 + }; + lemma_normalized_additive(basis, i, span(t.drop_last(), idx), sel); + } +} + +// --------------------------------------------------------------------------------------------- +// The twiddle table +// --------------------------------------------------------------------------------------------- +/// Table of the normalized subspace polynomials at the basis. +/// +/// - Row `i` holds `s_i(b_j)` for every basis element `b_j` with `j >= i`. +/// - Each row is scaled so that its first entry is one. +pub fn generate_evals_from_subspace(basis: &[F64]) -> (evals: Vec>) + requires + basis.len() >= 1, + ensures + evals.len() == basis.len(), + forall|i: int| 0 <= i < evals.len() ==> (#[trigger] evals@[i])@.len() == basis.len() - i, + forall|i: int, k: int| + 0 <= i < evals.len() && 0 <= k < evals@[i]@.len() ==> (#[trigger] evals@[i]@[k]).0 + == normalized_subspace_poly(basis@, i as nat, basis@[i + k].0), +{ + let l = basis.len(); + let mut evals: Vec> = Vec::with_capacity(l); + evals.push(vstd::slice::slice_to_vec(basis)); + for i in 1..l + invariant + l == basis.len(), + evals.len() == i, + forall|i0: int| 0 <= i0 < i ==> (#[trigger] evals@[i0])@.len() == l - i0, + forall|i0: int, k: int| + 0 <= i0 < i && 0 <= k < l - i0 ==> (#[trigger] evals@[i0]@[k]).0 == subspace_poly( + basis@, + i0 as nat, + basis@[i0 + k].0, + ), + { + assert(evals@[i - 1]@.len() == l - (i - 1)); + let mut row: Vec = Vec::with_capacity(l - i); + for k in 1..evals[i - 1].len() + invariant + l == basis.len(), + 1 <= i < l, + evals.len() == i, + evals@[i - 1]@.len() == l - (i - 1), + forall|k0: int| + 0 <= k0 < l - (i - 1) ==> (#[trigger] evals@[i - 1]@[k0]).0 == subspace_poly( + basis@, + (i - 1) as nat, + basis@[i - 1 + k0].0, + ), + row.len() == k - 1, + forall|k0: int| + 0 <= k0 < k - 1 ==> (#[trigger] row@[k0]).0 == subspace_poly(basis@, i as nat, basis@[i + k0].0), + { + let val = evals[i - 1][k] * (evals[i - 1][k] + evals[i - 1][0]); + row.push(val); + } + evals.push(row); + assert(evals@[i as int] == row); + } + // Rewritten from `for row in evals.iter_mut() { .. for v in row.iter_mut() { .. } }`. + for i in 0..l + invariant + l == basis.len(), + evals.len() == l, + forall|i0: int| 0 <= i0 < l ==> (#[trigger] evals@[i0])@.len() == l - i0, + forall|i0: int, k: int| + 0 <= i0 < l && 0 <= k < l - i0 ==> (#[trigger] evals@[i0]@[k]).0 == if i0 < i { + normalized_subspace_poly(basis@, i0 as nat, basis@[i0 + k].0) + } else { + subspace_poly(basis@, i0 as nat, basis@[i0 + k].0) + }, + { + let ghost pre = evals@; + assert(evals@[i as int]@.len() == l - i); + let inv = evals[i][0].inv(); + for k in 0..l - i + invariant + l == basis.len(), + 0 <= i < l, + evals.len() == l, + pre.len() == l, + inv.0 == k_inv(subspace_poly(basis@, i as nat, basis@[i as int].0)), + forall|i0: int| 0 <= i0 < l && i0 != i ==> #[trigger] evals@[i0] == pre[i0], + evals@[i as int]@.len() == l - i, + forall|k0: int| + 0 <= k0 < l - i ==> (#[trigger] evals@[i as int]@[k0]).0 == if k0 < k { + normalized_subspace_poly(basis@, i as nat, basis@[i + k0].0) + } else { + subspace_poly(basis@, i as nat, basis@[i + k0].0) + }, + { + evals[i][k] *= inv; + } + assert forall|i0: int| 0 <= i0 < l implies (#[trigger] evals@[i0])@.len() == l - i0 by { + if i0 != i { + assert(evals@[i0] == pre[i0]); + } + } + assert forall|i0: int, k: int| 0 <= i0 < l && 0 <= k < l - i0 implies (#[trigger] evals@[i0]@[k]).0 == if i0 + < i + 1 { + normalized_subspace_poly(basis@, i0 as nat, basis@[i0 + k].0) + } else { + subspace_poly(basis@, i0 as nat, basis@[i0 + k].0) + } by { + if i0 != i { + assert(evals@[i0] == pre[i0]); + assert(evals@[i0]@[k] == pre[i0]@[k]); + } + } + } + evals +} + +/// Additive NTT over F_{2^64} with the standard polynomial-basis subspace +/// `{1, x, x², …}`: the F_2-subspace is `{0, 1, …, 2^ℓ−1}` under the natural +/// integer encoding, exactly as in the extension-field version (whose domain already +/// lived inside this very subfield). +#[derive(Clone, Debug)] +#[verifier::allow(autoderive_clone_without_spec)] +pub struct AdditiveNttF64 { + evals: Vec>, +} + +impl AdditiveNttF64 { + /// The twiddle table, row by row. + pub closed spec fn table(&self) -> Seq> { + self.evals@.map_values(|r: Vec| r@) + } + + /// The table of a basis of `ℓ` elements: row `i` holds `Ŵ_i(b_(i+k))` for `k < ℓ - i`. + pub open spec fn is_table_of(tab: Seq>, basis: Seq) -> bool { + &&& tab.len() == basis.len() + &&& forall|i: int| 0 <= i < tab.len() ==> #[trigger] tab[i].len() == basis.len() - i + &&& forall|i: int, k: int| + 0 <= i < tab.len() && 0 <= k < tab[i].len() ==> (#[trigger] tab[i][k]).0 + == normalized_subspace_poly(basis, i as nat, basis[i + k].0) + } + + /// The twiddle table of a domain of dimension `1..=63`. + pub open spec fn well_formed(&self) -> bool { + &&& 1 <= self.table().len() <= 63 + &&& forall|i: int| + 0 <= i < self.table().len() ==> #[trigger] self.table()[i].len() == self.table().len() - i + } + + fn new(basis: &[F64]) -> (r: Self) + requires + 1 <= basis.len() <= 63, + ensures + r.well_formed(), + Self::is_table_of(r.table(), basis@), + { + let r = Self { evals: generate_evals_from_subspace(basis) }; + assert forall|i: int| 0 <= i < r.table().len() implies #[trigger] r.table()[i].len() == r.table().len() + - i by { + assert(r.table()[i] == r.evals@[i]@); + assert(r.evals@[i]@.len() == basis.len() - i); + } + assert forall|i: int, k: int| 0 <= i < r.table().len() && 0 <= k < r.table()[i].len() implies ( + #[trigger] r.table()[i][k]).0 == normalized_subspace_poly(basis@, i as nat, basis@[i + k].0) by { + assert(r.table()[i] == r.evals@[i]@); + } + r + } + + /// Standard NTT with basis `{1, x, …, x^(dim-1)}`. Requires `dim ≤ 63` so + /// the evaluation domain (and the twiddles) stay inside F_{2^64} without + /// wrap; far beyond any codeword size in use. + /// + /// Rewritten: the `assert!` is a precondition, which also excludes `dim = 0` (production panics + /// there), and the basis is built by a loop instead of `map(..).collect()`. + pub fn standard(dim: usize) -> (r: Self) + requires + 1 <= dim <= 63, + ensures + r.well_formed(), + r.table().len() == dim, + Self::is_table_of(r.table(), standard_basis(dim as nat)), + { + let mut basis: Vec = Vec::with_capacity(dim); + for i in 0..dim + invariant + dim <= 63, + basis@ == standard_basis(i as nat), + { + basis.push(F64(1u64 << i)); + assert(basis@ =~= standard_basis((i + 1) as nat)); + } + Self::new(&basis) + } + + pub const fn log_domain_size(&self) -> (r: usize) + ensures + r == self.table().len(), + { + self.evals.len() + } + + /// Twiddle of one block at one layer. + /// + /// - It is `s_i(sum_j bit_j(block) * b_(i+1+j))`, with `i = L - layer - 1` on a `2^L`-point domain. + /// - The normalized `s_i` is F_2-linear, so this is a subset sum of row `i` of the table. + pub fn twiddle(&self, layer: usize, block: usize) -> (r: F64) + requires + layer < self.table().len(), + 1 <= self.table()[self.table().len() - layer - 1].len() <= 65, + ensures + r.0 == twiddle_spec(self.table(), layer as nat, block), + { + let v = &self.evals[self.log_domain_size() - layer - 1]; + assert(v@ == self.table()[self.table().len() - layer - 1]); + span_get(&v[1..], block) + } + + /// The seven twiddles a radix-8 group needs, breadth-first: layer `layer`, + /// then `layer + 1` (one per half), then `layer + 2` (one per quarter). + /// + /// `span_get` is F_2-linear in the block index, so the six deeper twiddles are + /// the block's own contribution plus a fixed correction per sub-block index: + /// one scan of the three basis rows replaces seven. + pub fn twiddles_radix8(&self, layer: usize, block: usize) -> (r: [F64; 7]) + requires + layer + 3 <= self.table().len(), + block < pow2(layer as nat), + layer + 1 <= self.table()[self.table().len() - layer - 1].len() <= 64, + layer + 2 <= self.table()[self.table().len() - layer - 2].len() <= 64, + layer + 3 <= self.table()[self.table().len() - layer - 3].len() <= 64, + ensures + r[0].0 == twiddle_spec(self.table(), layer as nat, block), + r[1].0 == twiddle_spec(self.table(), (layer + 1) as nat, (2 * block) as usize), + r[2].0 == twiddle_spec(self.table(), (layer + 1) as nat, (2 * block + 1) as usize), + r[3].0 == twiddle_spec(self.table(), (layer + 2) as nat, (4 * block) as usize), + r[4].0 == twiddle_spec(self.table(), (layer + 2) as nat, (4 * block + 1) as usize), + r[5].0 == twiddle_spec(self.table(), (layer + 2) as nat, (4 * block + 2) as usize), + r[6].0 == twiddle_spec(self.table(), (layer + 2) as nat, (4 * block + 3) as usize), + { + let l = self.log_domain_size(); + let (v0, v1, v2) = (&self.evals[l - layer - 1], &self.evals[l - layer - 2], &self.evals[l - layer - 3]); + assert(v0@ == self.table()[l - layer - 1] && v1@ == self.table()[l - layer - 2] && v2@ == self.table()[l - layer + - 3]); + let ghost (s0, s1, s2) = (v0@.skip(1), v1@.skip(2), v2@.skip(3)); + let (mut t0, mut a, mut c) = (F64::ZERO, F64::ZERO, F64::ZERO); + for j in 0..layer + invariant + layer <= 61, + v0@.len() >= layer + 1, + v1@.len() >= layer + 2, + v2@.len() >= layer + 3, + s0 == v0@.skip(1), + s1 == v1@.skip(2), + s2 == v2@.skip(3), + t0.0 == span(s0.take(j as int), block), + a.0 == span(s1.take(j as int), block), + c.0 == span(s2.take(j as int), block), + { + assert(s0.take(j + 1).drop_last() =~= s0.take(j as int)); + assert(s1.take(j + 1).drop_last() =~= s1.take(j as int)); + assert(s2.take(j + 1).drop_last() =~= s2.take(j as int)); + proof { + lemma_xor_facts(t0.0, 0, 0); + lemma_xor_facts(a.0, 0, 0); + lemma_xor_facts(c.0, 0, 0); + } + if (block >> j) & 1 == 1 { + t0 += v0[1 + j]; + a += v1[2 + j]; + c += v2[3 + j]; + } + } + let (d, e0, e1) = (v1[1], v2[1], v2[2]); + proof { + self.lemma_twiddles_radix8(layer as nat, block, t0.0, a.0, c.0); + } + [t0, a, a + d, c, c + e0, c + e1, c + e0 + e1] + } + + proof fn lemma_twiddles_radix8(&self, layer: nat, block: usize, t0: u64, a: u64, c: u64) + requires + layer + 3 <= self.table().len(), + block < pow2(layer), + layer + 1 <= self.table()[self.table().len() - layer - 1].len() <= 64, + layer + 2 <= self.table()[self.table().len() - layer - 2].len() <= 64, + layer + 3 <= self.table()[self.table().len() - layer - 3].len() <= 64, + t0 == span(self.table()[self.table().len() - layer - 1].skip(1).take(layer as int), block), + a == span(self.table()[self.table().len() - layer - 2].skip(2).take(layer as int), block), + c == span(self.table()[self.table().len() - layer - 3].skip(3).take(layer as int), block), + ensures + ({ + let tab = self.table(); + let (v1, v2) = (tab[tab.len() - layer - 2], tab[tab.len() - layer - 3]); + &&& t0 == twiddle_spec(tab, layer, block) + &&& a == twiddle_spec(tab, layer + 1, (2 * block) as usize) + &&& a ^ v1[1].0 == twiddle_spec(tab, layer + 1, (2 * block + 1) as usize) + &&& c == twiddle_spec(tab, layer + 2, (4 * block) as usize) + &&& c ^ v2[1].0 == twiddle_spec(tab, layer + 2, (4 * block + 1) as usize) + &&& c ^ v2[2].0 == twiddle_spec(tab, layer + 2, (4 * block + 2) as usize) + &&& (c ^ v2[1].0) ^ v2[2].0 == twiddle_spec(tab, layer + 2, (4 * block + 3) as usize) + }), + { + let tab = self.table(); + let (v0, v1, v2) = (tab[tab.len() - layer - 1], tab[tab.len() - layer - 2], tab[tab.len() - layer - 3]); + lemma2_to64(); + lemma2_to64_rest(); + if layer < 61 { + lemma_pow2_strictly_increases(layer, 61); + } + assert(v1.skip(1).skip(1) =~= v1.skip(2)); + assert(v2.skip(1).skip(1) =~= v2.skip(2)); + assert(v2.skip(2).skip(1) =~= v2.skip(3)); + // Layer `layer`: the block alone. + lemma_span_low(v0.skip(1), block, layer); + // Layer `layer + 1`: sub-block `2b + h`. + lemma_span_low(v1.skip(2), block, layer); + lemma_span_double(v1.skip(1), block); + lemma_xor_facts(a, v1[1].0, 0); + // Layer `layer + 2`: sub-block `4b + 2h + q = 2(2b + h) + q`. + lemma_span_low(v2.skip(3), block, layer); + let b2 = (2 * block) as usize; + let b21 = (2 * block + 1) as usize; + lemma_span_double(v2.skip(2), block); + lemma_span_double(v2.skip(1), b2); + lemma_span_double(v2.skip(1), b21); + assert((b2 * 2) as usize == (4 * block) as usize); + assert((b2 * 2 + 1) as usize == (4 * block + 1) as usize); + assert((b21 * 2) as usize == (4 * block + 2) as usize); + assert((b21 * 2 + 1) as usize == (4 * block + 3) as usize); + let (e0, e1) = (v2[1].0, v2[2].0); + assert(e0 ^ (e1 ^ c) == (c ^ e0) ^ e1 && e1 ^ c == c ^ e1 && e0 ^ c == c ^ e0) by (bit_vector); + } +} + +/// The twiddle of a block is `Ŵ_i` of the block's point, `i = L - layer - 1`: the subset sum of the +/// basis elements `b_(i+1+j)` over the set bits `j` of the block index (the claim of `twiddle`'s +/// documentation). +pub proof fn lemma_twiddle_is_subspace_poly(tab: Seq>, basis: Seq, layer: nat, block: usize) + requires + AdditiveNttF64::is_table_of(tab, basis), + layer < tab.len(), + ensures + ({ + let i = (tab.len() - layer - 1) as nat; + twiddle_spec(tab, layer, block) == normalized_subspace_poly(basis, i, span(basis.skip((i + 1) as int), block)) + }), +{ + let i = (tab.len() - layer - 1) as nat; + let s = tab[i as int].skip(1); + let t = basis.skip((i + 1) as int); + assert forall|j: int| 0 <= j < s.len() implies s[j].0 == normalized_subspace_poly(basis, i, #[trigger] t[j].0) by { + assert(tab[i as int][j + 1] == s[j]); + } + lemma_span_normalized(basis, i, s, t, block); +} + + +// --------------------------------------------------------------------------------------------- +// Butterflies on every lane of a row pair (portable path) +// --------------------------------------------------------------------------------------------- +/// One lane's butterfly outputs `(new_u, new_v)`, forward or transposed. +pub open spec fn butterfly_spec(transposed: bool, u: u64, v: u64, t: u64) -> (u64, u64) { + if transposed { + (u ^ v, v ^ k_mul(u ^ v, t)) + } else { + (bf_top(u, v, t), bf_bot(u, v, t)) + } +} + +/// Butterfly all `num_ntts` lanes of one (top row, bottom row) pair with a +/// shared twiddle: new_u = u + v*t; new_v = v + new_u. +#[inline] +pub fn butterfly_lanes(top: &mut [F64], bot: &mut [F64], twiddle: F64) + requires + old(top).len() == old(bot).len(), + ensures + final(top).len() == old(top).len(), + final(bot).len() == old(bot).len(), + forall|i: int| + 0 <= i < old(top).len() ==> ((#[trigger] final(top)@[i]).0, final(bot)@[i].0) == butterfly_spec( + false, + old(top)@[i].0, + old(bot)@[i].0, + twiddle.0, + ), +{ + lane_butterflies::(top, bot, twiddle); +} + +/// The transposed butterfly on every lane of a row pair: s = u + v; new_u = s; new_v = v + s*t. +/// +/// It is the inverse of the forward butterfly with the rows swapped. +#[inline] +pub fn transposed_butterfly_lanes(top: &mut [F64], bot: &mut [F64], twiddle: F64) + requires + old(top).len() == old(bot).len(), + ensures + final(top).len() == old(top).len(), + final(bot).len() == old(bot).len(), + forall|i: int| + 0 <= i < old(top).len() ==> ((#[trigger] final(top)@[i]).0, final(bot)@[i].0) == butterfly_spec( + true, + old(top)@[i].0, + old(bot)@[i].0, + twiddle.0, + ), +{ + lane_butterflies::(top, bot, twiddle); +} + +/// The butterflies of every lane of a row pair, forward or transposed. +/// +/// On NEON this processes eight lanes per iteration. Four independent pair +/// reductions stay in the vector register file, exposing their PMULL chains +/// in parallel and amortizing the loop branch and constant setup. The pair +/// kernel handles a short even tail, and the scalar path handles an odd tail. +/// +/// Rewritten for Verus: the SIMD kernels (`crate::ntt_simd`) take the rows and the offset instead of +/// `top.as_mut_ptr().add(..)` and `bot.as_mut_ptr().add(..)`; the NEON pair loop tests `top.len() - lane >= 2` +/// instead of `lane + 2 <= top.len()` (the same for `lane <= top.len()`, and Verus has no bound on a slice's +/// length that rules out the overflow of `lane + 2`); the `zip` of the rows' `iter_mut` from `done` is an index +/// loop; `debug_assert_eq!(top.len(), bot.len())` is the `requires`. +#[inline] +pub fn lane_butterflies(top: &mut [F64], bot: &mut [F64], twiddle: F64) + requires + old(top).len() == old(bot).len(), + ensures + final(top).len() == old(top).len(), + final(bot).len() == old(bot).len(), + forall|i: int| + 0 <= i < old(top).len() ==> ((#[trigger] final(top)@[i]).0, final(bot)@[i].0) == butterfly_spec( + TRANSPOSED, + old(top)@[i].0, + old(bot)@[i].0, + twiddle.0, + ), +{ + let ghost (top0, bot0) = (top@, bot@); + #[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] + let done = { + let vectors = top.len() / 8; + // SAFETY: the target features are enabled at compile time and each + // iteration reads and writes exactly eight elements from both rows. + unsafe { + for i in 0..vectors + invariant + vectors == top0.len() / 8, + bot0.len() == top0.len(), + top0.len() <= usize::MAX, + butterflied(TRANSPOSED, top0, bot0, top@, bot@, 0, 8 * i, twiddle.0), + { + let ghost (top1, bot1) = (top@, bot@); + proof { + lemma_fundamental_div_mod(top0.len() as int, 8); + assert(8 * i + 8 <= top0.len()); + } + butterfly_lanes_avx512::(top, bot, 8 * i, twiddle.0); + proof { + lemma_butterflied_extend(TRANSPOSED, top0, bot0, top1, bot1, top@, bot@, 0, 8 * i, 8 * i + 8, twiddle.0); + } + } + } + 8 * vectors + }; + #[cfg(all( + target_arch = "x86_64", + target_feature = "vpclmulqdq", + target_feature = "avx2", + not(target_feature = "avx512f") + ))] + let done = { + let vectors = top.len() / 4; + // SAFETY: the target features are enabled at compile time and each + // iteration reads and writes exactly four elements from both rows. + unsafe { + for i in 0..vectors + invariant + vectors == top0.len() / 4, + bot0.len() == top0.len(), + top0.len() <= usize::MAX, + butterflied(TRANSPOSED, top0, bot0, top@, bot@, 0, 4 * i, twiddle.0), + { + let ghost (top1, bot1) = (top@, bot@); + proof { + lemma_fundamental_div_mod(top0.len() as int, 4); + assert(4 * i + 4 <= top0.len()); + } + butterfly_lanes_avx2::(top, bot, 4 * i, twiddle.0); + proof { + lemma_butterflied_extend(TRANSPOSED, top0, bot0, top1, bot1, top@, bot@, 0, 4 * i, 4 * i + 4, twiddle.0); + } + } + } + 4 * vectors + }; + #[cfg(all(target_arch = "aarch64", target_feature = "aes"))] + let done = { + let vectors = top.len() / 8; + let mut lane = 8 * vectors; + // SAFETY: aes target feature is enabled at compile time; the kernels + // read and write lanes [8i, 8i+8), then pairs below the rows' length. + unsafe { + for i in 0..vectors + invariant + vectors == top0.len() / 8, + lane == 8 * vectors, + bot0.len() == top0.len(), + top0.len() <= usize::MAX, + butterflied(TRANSPOSED, top0, bot0, top@, bot@, 0, 8 * i, twiddle.0), + { + let ghost (top1, bot1) = (top@, bot@); + proof { + lemma_fundamental_div_mod(top0.len() as int, 8); + assert(8 * i + 8 <= top0.len()); + } + butterfly_lanes_neon_8::(top, bot, 8 * i, twiddle.0); + proof { + lemma_butterflied_extend(TRANSPOSED, top0, bot0, top1, bot1, top@, bot@, 0, 8 * i, 8 * i + 8, twiddle.0); + } + } + proof { + lemma_fundamental_div_mod(top0.len() as int, 8); + } + while top.len() - lane >= 2 + invariant + lane <= top0.len(), + bot0.len() == top0.len(), + top0.len() <= usize::MAX, + butterflied(TRANSPOSED, top0, bot0, top@, bot@, 0, lane as int, twiddle.0), + decreases top0.len() - lane, + { + let ghost (top1, bot1) = (top@, bot@); + butterfly_lane_pair_neon::(top, bot, lane, twiddle.0); + proof { + lemma_butterflied_extend(TRANSPOSED, top0, bot0, top1, bot1, top@, bot@, 0, lane as int, lane + 2, twiddle.0); + } + lane += 2; + } + } + lane + }; + #[cfg(not(any( + all(target_arch = "aarch64", target_feature = "aes"), + all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2") + )))] + let done = 0; + let n = top.len(); + for i in done..n + invariant + n == top0.len(), + top.len() == top0.len(), + bot.len() == top0.len(), + bot0.len() == top0.len(), + forall|j: int| + 0 <= j < i ==> ((#[trigger] top@[j]).0, bot@[j].0) == butterfly_spec(TRANSPOSED, top0[j].0, bot0[j].0, twiddle.0), + forall|j: int| i <= j < top0.len() ==> #[trigger] top@[j] == top0[j] && bot@[j] == bot0[j], + { + let mut u = top[i]; + let mut v = bot[i]; + butterfly_one::(&mut u, &mut v, twiddle); + top[i] = u; + bot[i] = v; + } +} + +// --------------------------------------------------------------------------------------------- +// Layers +// --------------------------------------------------------------------------------------------- +/// Row of word `p` in a buffer of `m` interleaved lanes. +pub open spec fn row_of(p: int, m: nat) -> int { + p / (m as int) +} + +/// Block of word `p` at a layer whose blocks are `2h` rows. +pub open spec fn blk_of(p: int, m: nat, h: nat) -> int { + row_of(p, m) / ((2 * h) as int) +} + +/// Row of word `p` inside its block of `2h` rows. +pub open spec fn r_of(p: int, m: nat, h: nat) -> int { + row_of(p, m) % ((2 * h) as int) +} + +/// One layer of butterflies on a buffer of `m` interleaved lanes, word `p` being row `p / m`, lane `p % m`. +/// +/// Rows pair `h` apart inside blocks of `2h` rows: each top row `r` with bottom row `r + h`, lane by lane. +/// Block `b` takes the twiddle `tw(b)`. The forward butterfly is `(u, v) -> (u + v t, v + u + v t)`, +/// the inverse one `(u, v) -> (u + (v + u) t, v + u)`. +pub open spec fn layer_map(data: Seq, m: nat, h: nat, tw: spec_fn(int) -> u64, inverse: bool) -> Seq { + Seq::new( + data.len(), + |p: int| + { + let t = tw(blk_of(p, m, h)); + if r_of(p, m, h) < h { + let (u, v) = (data[p].0, data[p + h * m].0); + F64( + if inverse { + ibf_top(u, v, t) + } else { + bf_top(u, v, t) + }, + ) + } else { + let (u, v) = (data[p - h * m].0, data[p].0); + F64( + if inverse { + ibf_bot(u, v, t) + } else { + bf_bot(u, v, t) + }, + ) + } + }, + ) +} + +/// The twiddles of one layer, block by block, read from the table. +pub open spec fn layer_twiddles(tab: Seq>, layer: nat) -> spec_fn(int) -> u64 { + |b: int| twiddle_spec(tab, layer, b as usize) +} + +/// Half a block at one layer of a `2^log_d`-row transform: layer 0 pairs rows half the domain apart. +pub open spec fn layer_half(log_d: nat, layer: nat) -> nat { + pow2((log_d - layer - 1) as nat) +} + +/// The forward layers `start..end` of a `2^log_d`-row transform on `m` lanes, applied in order. +pub open spec fn forward_layers(tab: Seq>, data: Seq, m: nat, log_d: nat, start: nat, end: nat) -> Seq + decreases end, +{ + if end <= start { + data + } else { + layer_map( + forward_layers(tab, data, m, log_d, start, (end - 1) as nat), + m, + layer_half(log_d, (end - 1) as nat), + layer_twiddles(tab, (end - 1) as nat), + false, + ) + } +} + +/// The inverse layers `log_d - 1` down to `lo` of a `2^log_d`-row transform on `m` lanes, applied in order. +pub open spec fn inverse_layers(tab: Seq>, data: Seq, m: nat, log_d: nat, lo: nat) -> Seq + decreases log_d - lo, +{ + if lo >= log_d { + data + } else { + layer_map( + inverse_layers(tab, data, m, log_d, lo + 1), + m, + layer_half(log_d, lo), + layer_twiddles(tab, lo), + true, + ) + } +} + +pub proof fn lemma_compose(blk: int, r: int, lane: int, m: nat, h: nat) + requires + 0 <= blk, + 0 <= r < 2 * h, + 0 <= lane < m, + ensures + ({ + let q = (blk * (2 * h) + r) * m + lane; + &&& q >= 0 + &&& row_of(q, m) == blk * (2 * h) + r + &&& q % (m as int) == lane + &&& blk_of(q, m, h) == blk + &&& r_of(q, m, h) == r + }), +{ + let row = blk * (2 * h) + r; + lemma_mul_nonnegative(blk, (2 * h) as int); + lemma_mul_nonnegative(row, m as int); + lemma_fundamental_div_mod_converse(row * m + lane, m as int, row, lane); + lemma_fundamental_div_mod_converse(row, (2 * h) as int, blk, r); +} + +proof fn lemma_decompose(p: int, m: nat, h: nat) + requires + 0 <= p, + m > 0, + h > 0, + ensures + 0 <= p % (m as int) < m, + 0 <= row_of(p, m), + 0 <= blk_of(p, m, h), + 0 <= r_of(p, m, h) < 2 * h, + p == (blk_of(p, m, h) * (2 * h) + r_of(p, m, h)) * m + p % (m as int), +{ + let row = row_of(p, m); + lemma_fundamental_div_mod(p, m as int); + lemma_mod_pos_bound(p, m as int); + lemma_div_pos_is_pos(p, m as int); + lemma_fundamental_div_mod(row, (2 * h) as int); + lemma_mod_pos_bound(row, (2 * h) as int); + lemma_div_pos_is_pos(row, (2 * h) as int); + let (b, r, lane) = (blk_of(p, m, h), r_of(p, m, h), p % (m as int)); + assert(p == (b * (2 * h) + r) * m + lane) by (nonlinear_arith) + requires + p == m * row + lane, + row == (2 * h) * b + r, + ; +} + +pub proof fn lemma_bound(blk: int, r: int, lane: int, m: nat, h: nat, nb: nat) + requires + 0 <= blk < nb, + 0 <= r < 2 * h, + 0 <= lane < m, + ensures + (blk * (2 * h) + r) * m + lane < nb * (2 * h) * m, +{ + let row = blk * (2 * h) + r; + assert(row + 1 <= nb * (2 * h)) by (nonlinear_arith) + requires + row == blk * (2 * h) + r, + r < 2 * h, + blk + 1 <= nb, + ; + assert(row * m + lane < nb * (2 * h) * m) by (nonlinear_arith) + requires + row + 1 <= nb * (2 * h), + 0 <= lane < m, + 0 <= row, + ; +} + +pub proof fn lemma_blk_bound(p: int, m: nat, h: nat, nb: nat) + requires + 0 <= p < nb * (2 * h) * m, + m > 0, + h > 0, + ensures + blk_of(p, m, h) < nb, +{ + lemma_decompose(p, m, h); + let (b, r, lane) = (blk_of(p, m, h), r_of(p, m, h), p % (m as int)); + if b >= nb { + assert(p >= nb * (2 * h) * m) by (nonlinear_arith) + requires + p == (b * (2 * h) + r) * m + lane, + b >= nb, + r >= 0, + lane >= 0, + m > 0, + h > 0, + ; + } +} + +/// Word `p` has had its butterfly at the sweep position `(block, row, lane)`: butterflies run block by +/// block, then by row of the top half, then by lane. +pub open spec fn swept(p: int, m: nat, h: nat, block: int, row: int, lane: int) -> bool { + let b = blk_of(p, m, h); + let r = r_of(p, m, h); + let rr = if r < h { + r + } else { + r - h + }; + b < block || (b == block && (rr < row || (rr == row && p % (m as int) < lane))) +} + +/// Mid-sweep: the swept words hold the layer's output, the others its input. +pub open spec fn sweep_inv( + data: Seq, + prev: Seq, + target: Seq, + m: nat, + h: nat, + block: int, + row: int, + lane: int, +) -> bool { + &&& data.len() == prev.len() + &&& target.len() == prev.len() + &&& forall|p: int| + 0 <= p < data.len() ==> #[trigger] data[p] == if swept(p, m, h, block, row, lane) { + target[p] + } else { + prev[p] + } +} + +proof fn lemma_sweep_start(prev: Seq, target: Seq, m: nat, h: nat) + requires + m > 0, + h > 0, + target.len() == prev.len(), + ensures + sweep_inv(prev, prev, target, m, h, 0, 0, 0), +{ + assert forall|p: int| 0 <= p < prev.len() implies !swept(p, m, h, 0, 0, 0) by { + lemma_decompose(p, m, h); + } +} + +proof fn lemma_sweep_lane_end(data: Seq, prev: Seq, target: Seq, m: nat, h: nat, block: int, row: int) + requires + m > 0, + h > 0, + sweep_inv(data, prev, target, m, h, block, row, m as int), + ensures + sweep_inv(data, prev, target, m, h, block, row + 1, 0), +{ + assert forall|p: int| 0 <= p < data.len() implies swept(p, m, h, block, row, m as int) == swept( + p, + m, + h, + block, + row + 1, + 0, + ) by { + lemma_decompose(p, m, h); + } + assert forall|p: int| 0 <= p < data.len() implies #[trigger] data[p] == if swept(p, m, h, block, row + 1, 0) { + target[p] + } else { + prev[p] + } by { + assert(swept(p, m, h, block, row, m as int) == swept(p, m, h, block, row + 1, 0)); + } +} + +proof fn lemma_sweep_row_end(data: Seq, prev: Seq, target: Seq, m: nat, h: nat, block: int) + requires + m > 0, + h > 0, + sweep_inv(data, prev, target, m, h, block, h as int, 0), + ensures + sweep_inv(data, prev, target, m, h, block + 1, 0, 0), +{ + assert forall|p: int| 0 <= p < data.len() implies swept(p, m, h, block, h as int, 0) == swept( + p, + m, + h, + block + 1, + 0, + 0, + ) by { + lemma_decompose(p, m, h); + } + assert forall|p: int| 0 <= p < data.len() implies #[trigger] data[p] == if swept(p, m, h, block + 1, 0, 0) { + target[p] + } else { + prev[p] + } by { + assert(swept(p, m, h, block, h as int, 0) == swept(p, m, h, block + 1, 0, 0)); + } +} + +proof fn lemma_sweep_end(data: Seq, prev: Seq, target: Seq, m: nat, h: nat, nb: nat) + requires + m > 0, + h > 0, + prev.len() == nb * (2 * h) * m, + sweep_inv(data, prev, target, m, h, nb as int, 0, 0), + ensures + data == target, +{ + assert forall|p: int| 0 <= p < data.len() implies data[p] == target[p] by { + lemma_blk_bound(p, m, h, nb); + } + assert(data =~= target); +} + +/// The exec offsets of one butterfly and their bounds. +proof fn lemma_sweep_index(nb: nat, h: nat, m: nat, block: int, row: int, lane: int) + requires + 0 <= block < nb, + 0 <= row < h, + 0 <= lane < m, + ensures + block * ((2 * h) * m) + row * m + lane == (block * (2 * h) + row) * m + lane, + (block * (2 * h) + row) * m + lane + h * m < nb * (2 * h) * m, + (2 * h) * m <= nb * (2 * h) * m, + block * ((2 * h) * m) <= nb * (2 * h) * m, + row * m <= nb * (2 * h) * m, + h * m <= nb * (2 * h) * m, + h > 0 && m > 0 ==> h * m > 0, +{ + if h > 0 && m > 0 { + assert(h * m > 0) by (nonlinear_arith) + requires + h > 0, + m > 0, + ; + } + lemma_bound(block, row + h, lane, m, h, nb); + lemma_bound(block, 0, 0, m, h, nb); + lemma_bound(0, row, 0, m, h, nb); + lemma_bound(0, h as int, 0, m, h, nb); + assert(block * ((2 * h) * m) + row * m + lane == (block * (2 * h) + row) * m + lane) by (nonlinear_arith); + assert((block * (2 * h) + (row + h)) * m + lane == (block * (2 * h) + row) * m + lane + h * m) by (nonlinear_arith); + assert((2 * h) * m <= nb * (2 * h) * m) by (nonlinear_arith) + requires + nb >= 1, + ; + assert(block * ((2 * h) * m) == (block * (2 * h) + 0) * m + 0) by (nonlinear_arith); + assert(row * m == (0 * (2 * h) + row) * m + 0) by (nonlinear_arith); + assert(h * m == (0 * (2 * h) + h) * m + 0) by (nonlinear_arith); +} + +/// One butterfly of a sweep advances the sweep position by one lane. +proof fn lemma_sweep_step( + data: Seq, + new: Seq, + prev: Seq, + m: nat, + h: nat, + nb: nat, + tw: spec_fn(int) -> u64, + inverse: bool, + block: int, + row: int, + lane: int, +) + requires + m > 0, + h > 0, + prev.len() == nb * (2 * h) * m, + 0 <= block < nb, + 0 <= row < h, + 0 <= lane < m, + sweep_inv(data, prev, layer_map(prev, m, h, tw, inverse), m, h, block, row, lane), + new.len() == data.len(), + ({ + let q1 = (block * (2 * h) + row) * m + lane; + let q2 = q1 + h * m; + let (u, v, t) = (data[q1].0, data[q2].0, tw(block)); + &&& new[q1].0 == if inverse { + ibf_top(u, v, t) + } else { + bf_top(u, v, t) + } + &&& new[q2].0 == if inverse { + ibf_bot(u, v, t) + } else { + bf_bot(u, v, t) + } + &&& forall|p: int| 0 <= p < new.len() && p != q1 && p != q2 ==> #[trigger] new[p] == data[p] + }), + ensures + sweep_inv(new, prev, layer_map(prev, m, h, tw, inverse), m, h, block, row, lane + 1), +{ + let target = layer_map(prev, m, h, tw, inverse); + let q1 = (block * (2 * h) + row) * m + lane; + let q2 = q1 + h * m; + lemma_compose(block, row, lane, m, h); + lemma_compose(block, row + h, lane, m, h); + lemma_sweep_index(nb, h, m, block, row, lane); + assert((block * (2 * h) + (row + h)) * m + lane == q2) by (nonlinear_arith) + requires + q1 == (block * (2 * h) + row) * m + lane, + q2 == q1 + h * m, + ; + assert(!swept(q1, m, h, block, row, lane)); + assert(!swept(q2, m, h, block, row, lane)); + assert(data[q1] == prev[q1]); + assert(data[q2] == prev[q2]); + assert(new[q1] == target[q1]); + assert(new[q2] == target[q2]); + assert forall|p: int| 0 <= p < new.len() implies #[trigger] new[p] == if swept(p, m, h, block, row, lane + 1) { + target[p] + } else { + prev[p] + } by { + if p != q1 && p != q2 { + lemma_decompose(p, m, h); + let (b, r) = (blk_of(p, m, h), r_of(p, m, h)); + if b == block && p % (m as int) == lane { + if r == row { + assert(p == q1); + } else if r == row + h { + assert(p == q2); + } + } + assert(new[p] == data[p]); + } + } +} + +/// The exec shape of one layer of a `2^log_d`-row transform: `2^layer` blocks of `2h` rows. +pub proof fn lemma_layer_shape(log_d: nat, layer: nat, m: nat) + requires + layer < log_d, + ensures + layer_half(log_d, layer) > 0, + pow2(layer) > 0, + pow2((log_d - layer) as nat) == 2 * layer_half(log_d, layer), + pow2(layer) * (2 * layer_half(log_d, layer)) * m == m * pow2(log_d), +{ + let k = (log_d - layer) as nat; + lemma_pow2_pos(layer); + lemma_pow2_pos((k - 1) as nat); + lemma_pow2_unfold(k); + lemma_pow2_adds(layer, k); + assert(pow2(layer) * pow2(k) * m == m * pow2(log_d)) by (nonlinear_arith) + requires + pow2(layer) * pow2(k) == pow2(log_d), + ; +} + +pub proof fn lemma_layer_map_len(data: Seq, m: nat, h: nat, tw: spec_fn(int) -> u64, inverse: bool) + ensures + layer_map(data, m, h, tw, inverse).len() == data.len(), +{ +} + +pub proof fn lemma_forward_layers_len(tab: Seq>, data: Seq, m: nat, log_d: nat, start: nat, end: nat) + ensures + forward_layers(tab, data, m, log_d, start, end).len() == data.len(), + decreases end, +{ + if end > start { + lemma_forward_layers_len(tab, data, m, log_d, start, (end - 1) as nat); + } +} + +pub proof fn lemma_inverse_layers_len(tab: Seq>, data: Seq, m: nat, log_d: nat, lo: nat) + ensures + inverse_layers(tab, data, m, log_d, lo).len() == data.len(), + decreases log_d - lo, +{ + if lo < log_d { + lemma_inverse_layers_len(tab, data, m, log_d, lo + 1); + } +} + +/// The inverse layer undoes the forward layer with the same twiddles, word by word. +pub proof fn lemma_layer_roundtrip(y: Seq, m: nat, h: nat, nb: nat, tw: spec_fn(int) -> u64) + requires + m > 0, + h > 0, + y.len() == nb * (2 * h) * m, + ensures + layer_map(layer_map(y, m, h, tw, false), m, h, tw, true) == y, +{ + let z = layer_map(y, m, h, tw, false); + let w = layer_map(z, m, h, tw, true); + assert forall|p: int| 0 <= p < y.len() implies w[p] == y[p] by { + lemma_decompose(p, m, h); + lemma_blk_bound(p, m, h, nb); + let (b, r, lane) = (blk_of(p, m, h), r_of(p, m, h), p % (m as int)); + if r < h { + let q = p + h * m; + lemma_compose(b, r + h, lane, m, h); + lemma_bound(b, r + h, lane, m, h, nb); + assert((b * (2 * h) + (r + h)) * m + lane == q) by (nonlinear_arith) + requires + p == (b * (2 * h) + r) * m + lane, + q == p + h * m, + ; + lemma_inverse_butterfly(y[p].0, y[q].0, tw(b)); + } else { + let q = p - h * m; + lemma_compose(b, r - h, lane, m, h); + assert((b * (2 * h) + (r - h)) * m + lane == q) by (nonlinear_arith) + requires + p == (b * (2 * h) + r) * m + lane, + q == p - h * m, + ; + lemma_inverse_butterfly(y[q].0, y[p].0, tw(b)); + } + } + assert(w =~= y); +} + +proof fn lemma_inverse_after_forward_from(tab: Seq>, x: Seq, m: nat, log_d: nat, j: nat) + requires + m > 0, + x.len() == m * pow2(log_d), + j <= log_d, + ensures + inverse_layers(tab, forward_layers(tab, x, m, log_d, 0, log_d), m, log_d, j) == forward_layers( + tab, + x, + m, + log_d, + 0, + j, + ), + decreases log_d - j, +{ + if j < log_d { + lemma_inverse_after_forward_from(tab, x, m, log_d, j + 1); + let y = forward_layers(tab, x, m, log_d, 0, j); + lemma_forward_layers_len(tab, x, m, log_d, 0, j); + lemma_layer_shape(log_d, j, m); + lemma_layer_roundtrip(y, m, layer_half(log_d, j), pow2(j), layer_twiddles(tab, j)); + } +} + +/// The inverse transform undoes the forward one: `inverse_layers ∘ forward_layers = id` on every buffer of +/// `m` lanes of `2^log_d` rows, for any twiddle table. +pub proof fn lemma_inverse_after_forward(tab: Seq>, x: Seq, m: nat, log_d: nat) + requires + m > 0, + x.len() == m * pow2(log_d), + ensures + inverse_layers(tab, forward_layers(tab, x, m, log_d, 0, log_d), m, log_d, 0) == x, +{ + lemma_inverse_after_forward_from(tab, x, m, log_d, 0); +} + +proof fn lemma_pow2_injective(a: nat, b: nat) + requires + pow2(a) == pow2(b), + ensures + a == b, +{ + if a < b { + lemma_pow2_strictly_increases(a, b); + } else if b < a { + lemma_pow2_strictly_increases(b, a); + } +} + +/// `log2(n)` of a power of two. +/// +/// Rewritten from `primitives::log2_strict_usize`: its `assert!` is a precondition, and `trailing_zeros` +/// a search for the exponent. +pub fn log2_strict_usize(n: usize) -> (r: usize) + requires + exists|k: nat| n == pow2(k), + ensures + r < 64, + pow2(r as nat) == n, +{ + let ghost k = choose|k: nat| n == pow2(k); + proof { + lemma2_to64_rest(); + if k >= 64 { + if k > 64 { + lemma_pow2_strictly_increases(64, k); + } + } + } + let mut r: usize = 0; + while (1usize << r) < n + invariant + r <= k < 64, + n == pow2(k), + decreases k - r, + { + proof { + lemma_usize_pow2_no_overflow(r as nat); + lemma_usize_shl_is_mul(1, r); + } + r += 1; + } + proof { + lemma_usize_pow2_no_overflow(r as nat); + lemma_usize_shl_is_mul(1, r); + if r < k { + lemma_pow2_strictly_increases(r as nat, k); + } + } + r +} + +/// Scalar reference: one butterfly at a time over `num_ntts` interleaved lanes. +/// +/// The test helper of the same name in production, which its tests check against the parallel `transform`. +pub fn forward_scalar_from_layer(ntt: &AdditiveNttF64, data: &mut [F64], num_ntts: usize, start_layer: usize) + requires + ntt.well_formed(), + num_ntts > 0, + exists|k: nat| old(data).len() == num_ntts * pow2(k) && start_layer <= k <= ntt.table().len(), + ensures + forall|k: nat| + old(data).len() == num_ntts * #[trigger] pow2(k) ==> final(data)@ == forward_layers( + ntt.table(), + old(data)@, + num_ntts as nat, + k, + start_layer as nat, + k, + ), +{ + let ghost tab = ntt.table(); + let ghost orig = data@; + let ghost m = num_ntts as nat; + let ghost k0 = choose|k: nat| data.len() == num_ntts * pow2(k) && start_layer <= k <= ntt.table().len(); + let n_total = data.len(); + proof { + lemma_div_by_multiple(pow2(k0) as int, num_ntts as int); + assert(num_ntts * pow2(k0) == pow2(k0) * num_ntts) by (nonlinear_arith); + } + let log_d = log2_strict_usize(n_total / num_ntts); + proof { + lemma_pow2_injective(log_d as nat, k0); + } + for layer in start_layer..log_d + invariant + ntt.well_formed(), + tab == ntt.table(), + log_d <= tab.len(), + start_layer <= log_d, + num_ntts > 0, + m == num_ntts, + n_total == data.len(), + n_total == num_ntts * pow2(log_d as nat), + data@ == forward_layers(tab, orig, m, log_d as nat, start_layer as nat, layer as nat), + { + let ghost h = layer_half(log_d as nat, layer as nat); + let ghost nb = pow2(layer as nat); + proof { + lemma_layer_shape(log_d as nat, layer as nat, m); + lemma_usize_pow2_no_overflow(layer as nat); + lemma_usize_pow2_no_overflow((log_d - layer) as nat); + lemma_usize_shl_is_mul(1, layer); + lemma_usize_shl_is_mul(1, (log_d - layer) as usize); + lemma_usize_shr_is_div(pow2((log_d - layer) as nat) as usize, 1); + lemma2_to64(); + lemma_bound(0, 0, 0, m, h, nb); + lemma_sweep_index(nb, h, m, 0, 0, 0); + } + let num_blocks = 1usize << layer; + let block_size = 1usize << (log_d - layer); + let block_size_half = block_size >> 1; + let block_elems = block_size * num_ntts; + let ghost prev = data@; + let ghost tw = layer_twiddles(tab, layer as nat); + let ghost target = layer_map(prev, m, h, tw, false); + proof { + lemma_sweep_start(prev, target, m, h); + } + for block in 0..num_blocks + invariant + ntt.well_formed(), + tab == ntt.table(), + layer < log_d <= tab.len(), + num_ntts > 0, + m == num_ntts, + h > 0, + num_blocks == nb, + block_size_half == h, + block_elems == (2 * h) * m, + n_total == data.len(), + prev.len() == nb * (2 * h) * m, + tw == layer_twiddles(tab, layer as nat), + target == layer_map(prev, m, h, tw, false), + sweep_inv(data@, prev, target, m, h, block as int, 0, 0), + { + let twiddle = ntt.twiddle(layer, block); + proof { + lemma_sweep_index(nb, h, m, block as int, 0, 0); + } + let block_start = block * block_elems; + for row in 0..block_size_half + invariant + num_ntts > 0, + m == num_ntts, + h > 0, + block < nb, + block_size_half == h, + block_elems == (2 * h) * m, + block_start == block * ((2 * h) * m), + n_total == data.len(), + prev.len() == nb * (2 * h) * m, + twiddle.0 == tw(block as int), + target == layer_map(prev, m, h, tw, false), + sweep_inv(data@, prev, target, m, h, block as int, row as int, 0), + { + proof { + lemma_sweep_index(nb, h, m, block as int, row as int, 0); + } + let off_top = block_start + row * num_ntts; + let off_bot = off_top + block_size_half * num_ntts; + for lane in 0..num_ntts + invariant + num_ntts > 0, + m == num_ntts, + h > 0, + block < nb, + row < h, + block_size_half == h, + off_top == block * ((2 * h) * m) + row * m, + off_bot == off_top + h * m, + n_total == data.len(), + prev.len() == nb * (2 * h) * m, + twiddle.0 == tw(block as int), + target == layer_map(prev, m, h, tw, false), + sweep_inv(data@, prev, target, m, h, block as int, row as int, lane as int), + { + proof { + lemma_sweep_index(nb, h, m, block as int, row as int, lane as int); + } + let ghost d0 = data@; + let v = data[off_bot + lane]; + let new_u = data[off_top + lane] + v * twiddle; + data[off_top + lane] = new_u; + data[off_bot + lane] = v + new_u; + proof { + lemma_sweep_step(d0, data@, prev, m, h, nb, tw, false, block as int, row as int, lane as int); + } + } + proof { + lemma_sweep_lane_end(data@, prev, target, m, h, block as int, row as int); + } + } + proof { + lemma_sweep_row_end(data@, prev, target, m, h, block as int); + } + } + proof { + lemma_sweep_end(data@, prev, target, m, h, nb); + } + } + proof { + assert forall|k: nat| orig.len() == num_ntts * pow2(k) implies k == log_d by { + lemma_div_by_multiple(pow2(k) as int, num_ntts as int); + assert(num_ntts * pow2(k) == pow2(k) * num_ntts) by (nonlinear_arith); + lemma_pow2_injective(log_d as nat, k); + } + } +} + +impl AdditiveNttF64 { + /// Recover novel-basis coefficients from evaluations with a scalar inverse NTT. + /// + /// Rewritten: the `assert!` is a precondition, and `for layer in (0..log_d).rev()` a `while` loop. + pub fn inverse_transform(&self, data: &mut [F64]) + requires + self.well_formed(), + exists|k: nat| old(data).len() == pow2(k) && k <= self.table().len(), + ensures + forall|k: nat| + old(data).len() == #[trigger] pow2(k) ==> final(data)@ == inverse_layers(self.table(), old(data)@, 1, k, 0), + { + let ghost tab = self.table(); + let ghost orig = data@; + let ghost k0 = choose|k: nat| data.len() == pow2(k) && k <= self.table().len(); + let log_d = log2_strict_usize(data.len()); + proof { + lemma_pow2_injective(log_d as nat, k0); + } + let mut next = log_d; + while next > 0 + invariant + self.well_formed(), + tab == self.table(), + log_d <= tab.len(), + next <= log_d, + data.len() == pow2(log_d as nat), + data@ == inverse_layers(tab, orig, 1, log_d as nat, next as nat), + decreases next, + { + let layer = next - 1; + let ghost h = layer_half(log_d as nat, layer as nat); + let ghost nb = pow2(layer as nat); + proof { + lemma_layer_shape(log_d as nat, layer as nat, 1); + lemma_usize_pow2_no_overflow(layer as nat); + lemma_usize_pow2_no_overflow((log_d - layer - 1) as nat); + lemma_usize_shl_is_mul(1, layer); + lemma_usize_shl_is_mul(1, (log_d - layer - 1) as usize); + } + let num_blocks = 1usize << layer; + let block_size_half = 1usize << (log_d - layer - 1); + let ghost prev = data@; + let ghost tw = layer_twiddles(tab, layer as nat); + let ghost target = layer_map(prev, 1, h, tw, true); + proof { + lemma_sweep_start(prev, target, 1, h); + } + for block in 0..num_blocks + invariant + self.well_formed(), + tab == self.table(), + layer < log_d <= tab.len(), + h > 0, + num_blocks == nb, + block_size_half == h, + block_size_half == 1usize << ((log_d - layer - 1) as usize), + pow2((log_d - layer) as nat) == 2 * h, + data.len() == prev.len(), + prev.len() == nb * (2 * h) * 1, + tw == layer_twiddles(tab, layer as nat), + target == layer_map(prev, 1, h, tw, true), + sweep_inv(data@, prev, target, 1, h, block as int, 0, 0), + { + let twiddle = self.twiddle(layer, block); + proof { + lemma_sweep_index(nb, h, 1, block as int, 0, 0); + lemma_usize_shl_is_mul(block, (log_d - layer) as usize); + } + let block_start = block << (log_d - layer); + for idx0 in block_start..(block_start + block_size_half) + invariant + h > 0, + block < nb, + layer < log_d <= 63, + block_size_half == h, + block_size_half == 1usize << ((log_d - layer - 1) as usize), + block_start == block * (2 * h), + block_start == block << ((log_d - layer) as usize), + data.len() == prev.len(), + prev.len() == nb * (2 * h) * 1, + twiddle.0 == tw(block as int), + target == layer_map(prev, 1, h, tw, true), + block_start <= idx0, + sweep_inv(data@, prev, target, 1, h, block as int, idx0 - block_start, 0), + { + let ghost row = idx0 - block_start; + proof { + lemma_sweep_index(nb, h, 1, block as int, row, 0); + lemma_or_is_add(block, row as usize, (log_d - layer) as usize); + } + let idx1 = idx0 | block_size_half; + let ghost d0 = data@; + let u = data[idx0]; + let new_v = data[idx1] + u; + data[idx1] = new_v; + data[idx0] = u + new_v * twiddle; + proof { + lemma_sweep_step(d0, data@, prev, 1, h, nb, tw, true, block as int, row, 0); + lemma_sweep_lane_end(data@, prev, target, 1, h, block as int, row); + } + } + proof { + lemma_sweep_row_end(data@, prev, target, 1, h, block as int); + } + } + proof { + lemma_sweep_end(data@, prev, target, 1, h, nb); + } + next = layer; + } + proof { + assert forall|k: nat| orig.len() == pow2(k) implies k == log_d by { + lemma_pow2_injective(log_d as nat, k); + } + } + } +} + +proof fn lemma_or_is_add(block: usize, j: usize, s: usize) + requires + 1 <= s < 64, + j < (1usize << ((s - 1) as usize)), + (block << s) as int + j + (1usize << ((s - 1) as usize)) <= usize::MAX, + ensures + (((block << s) + j) as usize) | (1usize << ((s - 1) as usize)) == (block << s) + j + (1usize << ((s + - 1) as usize)), +{ + let x = block << s; + let hb = 1usize << ((s - 1) as usize); + assert(((x + j) as usize) | hb == ((x + j) as usize) + hb) by (bit_vector) + requires + x == block << s, + hb == 1usize << ((s - 1) as usize), + 1 <= s < 64, + j < hb, + ; +} + + +// --------------------------------------------------------------------------------------------- +// Fused layers on one row group +// --------------------------------------------------------------------------------------------- +/// The twelve butterflies of one radix-8 row group, with its seven twiddles breadth-first. +/// +/// Single-lane rows: row `i` is the one word `rows[i]` (a `num_ntts`-lane slice in production), so each +/// `butterfly_lanes(ri, rj, t)` is the one-lane `butterfly_one::(&mut ri, &mut rj, t)`. +/// +/// It is three successive layers on the eight rows: rows 4 apart with `t[0]`, then rows 2 apart with +/// `t[1 + half]`, then adjacent rows with `t[3 + quarter]`. With `t = twiddles_radix8(layer, block)` these +/// are the twiddles of `block` at `layer`, of its halves `2 block + h` at `layer + 1`, and of its quarters +/// `4 block + q` at `layer + 2`. +pub fn radix8_butterflies(rows: &mut [F64; 8], t: &[F64; 7]) + ensures + final(rows)@ == layer_map( + layer_map(layer_map(old(rows)@, 1, 4, |b: int| t@[0].0, false), 1, 2, |b: int| t@[1 + b].0, false), + 1, + 1, + |b: int| t@[3 + b].0, + false, + ), +{ + let ghost x = rows@; + // Rewritten from `let [r0, r1, r2, r3, r4, r5, r6, r7] = rows;`: Verus has no slice patterns. + let (mut r0, mut r1, mut r2, mut r3) = (rows[0], rows[1], rows[2], rows[3]); + let (mut r4, mut r5, mut r6, mut r7) = (rows[4], rows[5], rows[6], rows[7]); + // Layer L: rows 4 apart, one twiddle for the whole block. + butterfly_one::(&mut r0, &mut r4, t[0]); + butterfly_one::(&mut r1, &mut r5, t[0]); + butterfly_one::(&mut r2, &mut r6, t[0]); + butterfly_one::(&mut r3, &mut r7, t[0]); + let ghost y1 = seq![r0, r1, r2, r3, r4, r5, r6, r7]; + // Layer L+1: rows 2 apart, one twiddle per half. + butterfly_one::(&mut r0, &mut r2, t[1]); + butterfly_one::(&mut r1, &mut r3, t[1]); + butterfly_one::(&mut r4, &mut r6, t[2]); + butterfly_one::(&mut r5, &mut r7, t[2]); + let ghost y2 = seq![r0, r1, r2, r3, r4, r5, r6, r7]; + // Layer L+2: adjacent rows, one twiddle per quarter. + butterfly_one::(&mut r0, &mut r1, t[3]); + butterfly_one::(&mut r2, &mut r3, t[4]); + butterfly_one::(&mut r4, &mut r5, t[5]); + butterfly_one::(&mut r6, &mut r7, t[6]); + *rows = [r0, r1, r2, r3, r4, r5, r6, r7]; + proof { + let l1 = layer_map(x, 1, 4, |b: int| t@[0].0, false); + let l2 = layer_map(l1, 1, 2, |b: int| t@[1 + b].0, false); + let l3 = layer_map(l2, 1, 1, |b: int| t@[3 + b].0, false); + lemma_small_rows(8, 4); + lemma_small_rows(8, 2); + lemma_small_rows(8, 1); + assert(l1 =~= y1); + assert(l2 =~= y2); + assert(l3 =~= rows@); + } +} + +/// Layers L and L+1 on one group of four single-lane rows: the body of the closure that +/// `butterfly_interleaved_fused_2layer` runs on every row group of a layer-L block. +/// +/// It is two successive layers on the four rows: rows 2 apart with `t_outer`, then adjacent rows with +/// `t_inner_a` (first half) and `t_inner_b` (second half). +pub fn radix4_butterflies(rows: &mut [F64; 4], t_outer: F64, t_inner_a: F64, t_inner_b: F64) + ensures + final(rows)@ == layer_map( + layer_map(old(rows)@, 1, 2, |b: int| t_outer.0, false), + 1, + 1, + |b: int| + if b == 0 { + t_inner_a.0 + } else { + t_inner_b.0 + }, + false, + ), +{ + let ghost x = rows@; + // Rewritten from `let [row_a, row_b, row_c, row_d] = rows;`: Verus has no slice patterns. + let (mut row_a, mut row_b, mut row_c, mut row_d) = (rows[0], rows[1], rows[2], rows[3]); + // Layer L: rows 2 apart, one twiddle for the block. + butterfly_one::(&mut row_a, &mut row_c, t_outer); + butterfly_one::(&mut row_b, &mut row_d, t_outer); + let ghost y1 = seq![row_a, row_b, row_c, row_d]; + // Layer L+1: adjacent rows, one twiddle per half. + butterfly_one::(&mut row_a, &mut row_b, t_inner_a); + butterfly_one::(&mut row_c, &mut row_d, t_inner_b); + *rows = [row_a, row_b, row_c, row_d]; + proof { + let l1 = layer_map(x, 1, 2, |b: int| t_outer.0, false); + let l2 = layer_map( + l1, + 1, + 1, + |b: int| + if b == 0 { + t_inner_a.0 + } else { + t_inner_b.0 + }, + false, + ); + lemma_small_rows(4, 2); + lemma_small_rows(4, 1); + assert(l1 =~= y1); + assert(l2 =~= rows@); + } +} + +/// Block and in-block row of every row of a single-lane group of `n <= 8` rows. +proof fn lemma_small_rows(n: int, h: nat) + requires + 0 < n <= 8, + h == 1 || h == 2 || h == 4, + ensures + forall|p: int| + 0 <= p < n ==> #[trigger] blk_of(p, 1, h) == p / (2 * h as int) && r_of(p, 1, h) == p % (2 * h as int), +{ + assert forall|p: int| 0 <= p < n implies #[trigger] blk_of(p, 1, h) == p / (2 * h as int) && r_of(p, 1, h) == p + % (2 * h as int) by { + assert(row_of(p, 1) == p); + } +} + + +// --------------------------------------------------------------------------------------------- +// The forward transform evaluates the novel-basis polynomial +// --------------------------------------------------------------------------------------------- +/// The novel-basis polynomial with coefficients `a` (`2^d` of them), split at level `i` and residue `l`, +/// evaluated at `x`: +/// +/// `novel_eval(i, l, x) = Σ_{j < 2^(d-i)} a[l + 2^i j] · Π_{c < d-i} Ŵ_(i+c)(x)^(bit_c(j))`, +/// +/// written by its even-odd split on the lowest bit of `j` (annex `d`, Lemma "Even-odd refinement"): +/// `novel_eval(i, l, x) = novel_eval(i+1, l, x) + Ŵ_i(x) · novel_eval(i+1, l + 2^i, x)`. +/// The polynomial itself is `P(x) = novel_eval(0, 0, x) = Σ_j a_j X_j(x)` with `X_j = Π_i Ŵ_i^(bit_i(j))`; +/// [`lemma_novel_eval_flat`] proves this equality with the flat sum [`novel_sum`]. +pub open spec fn novel_eval(basis: Seq, a: Seq, d: nat, i: nat, l: int, x: u64) -> u64 + decreases d - i, +{ + if i >= d { + a[l].0 + } else { + novel_eval(basis, a, d, i + 1, l, x) ^ k_mul( + normalized_subspace_poly(basis, i, x), + novel_eval(basis, a, d, i + 1, l + pow2(i), x), + ) + } +} + +/// `novel_eval` at level `i` reads `x` only through `Ŵ_j(x)` for `j >= i`. +proof fn lemma_novel_eval_congruent(basis: Seq, a: Seq, d: nat, i: nat, l: int, x: u64, y: u64) + requires + forall|j: nat| i <= j < d ==> normalized_subspace_poly(basis, j, x) == normalized_subspace_poly(basis, j, y), + ensures + novel_eval(basis, a, d, i, l, x) == novel_eval(basis, a, d, i, l, y), + decreases d - i, +{ + if i < d { + lemma_novel_eval_congruent(basis, a, d, i + 1, l, x, y); + lemma_novel_eval_congruent(basis, a, d, i + 1, l + pow2(i), x, y); + } +} + +proof fn lemma_bit_shr(v: usize, n: usize, c: usize) + requires + n + c < 64, + ensures + ((v >> n) >> c) & 1 == (v >> ((n + c) as usize)) & 1, + (v >> n) >> 1usize == v >> ((n + 1) as usize), +{ + assert(((v >> n) >> c) & 1 == (v >> ((n + c) as usize)) & 1 && (v >> n) >> 1usize == v >> ((n + 1) as usize)) + by (bit_vector) + requires + n + c < 64, + ; +} + +/// A subset sum over a concatenation: the second part reads the index shifted past the first. +proof fn lemma_span_concat(s1: Seq, s2: Seq, v: usize) + requires + s1.len() + s2.len() <= 64, + ensures + span(s1 + s2, v) == span(s1, v) ^ span(s2, v >> (s1.len() as usize)), + decreases s2.len(), +{ + let n1 = s1.len() as usize; + if s2.len() == 0 { + assert(s1 + s2 =~= s1); + lemma_xor_facts(span(s1, v), 0, 0); + } else { + let s = s1 + s2; + lemma_span_concat(s1, s2.drop_last(), v); + assert(s.drop_last() =~= s1 + s2.drop_last()); + assert(s.last() == s2.last()); + let c = (s2.len() - 1) as usize; + lemma_bit_shr(v, n1, c); + let sel = if idx_bit(v, (s.len() - 1) as nat) { + s.last().0 + } else { + 0 + }; + lemma_xor_facts(span(s1, v), span(s2.drop_last(), v >> n1), sel); + } +} + +/// The point of index `v` splits at basis element `i`: the low part, `b_i` if bit `i` is set, and the part +/// above, which reads `v >> (i + 1)`. +proof fn lemma_span_split(basis: Seq, i: nat, v: usize) + requires + i < basis.len() <= 63, + ensures + span(basis, v) == span(basis.take(i as int), v) ^ ((if idx_bit(v, i) { + basis[i as int].0 + } else { + 0 + }) ^ span(basis.skip((i + 1) as int), v >> ((i + 1) as usize))), +{ + let (lo, hi) = (basis.take(i as int), basis.skip(i as int)); + assert(lo + hi =~= basis); + lemma_span_concat(lo, hi, v); + let w = v >> (i as usize); + let one = seq![basis[i as int]]; + assert(one + basis.skip((i + 1) as int) =~= hi); + lemma_span_concat(one, basis.skip((i + 1) as int), w); + lemma_bit_shr(v, i as usize, 0); + assert(one.drop_last().len() == 0); + assert(span(one.drop_last(), w) == 0); + assert(one.last() == basis[i as int]); + assert(w >> 0usize == w) by (bit_vector); + let sel = if idx_bit(v, i) { + basis[i as int].0 + } else { + 0 + }; + assert(span(one, w) == 0 ^ sel); + lemma_xor_facts(sel, 0, 0); +} + +/// `Ŵ_j` of a point, `j > i`, reads only the bits above `i`. +proof fn lemma_point_above(basis: Seq, i: nat, j: nat, v: usize) + requires + i < j < basis.len() <= 63, + ensures + normalized_subspace_poly(basis, j, span(basis, v)) == normalized_subspace_poly( + basis, + j, + span(basis.skip((i + 1) as int), v >> ((i + 1) as usize)), + ), +{ + lemma_span_split(basis, i, v); + let lo = span(basis.take(i as int), v); + let sel = if idx_bit(v, i) { + basis[i as int].0 + } else { + 0 + }; + let hi = span(basis.skip((i + 1) as int), v >> ((i + 1) as usize)); + lemma_normalized_additive(basis, j, lo, sel ^ hi); + lemma_normalized_additive(basis, j, sel, hi); + let c = k_inv(subspace_poly(basis, j, basis[j as int].0)); + lemma_subspace_poly_kills_prefix(basis, j, basis.take(i as int), v); + lemma_k_mul_zero(c); + if idx_bit(v, i) { + lemma_subspace_poly_vanishes(basis, j, i); + } + lemma_xor_facts(normalized_subspace_poly(basis, j, hi), 0, 0); +} + +/// `Ŵ_i` of a point is bit `i` of its index plus `Ŵ_i` of the part above, if `Ŵ_i(b_i) = 1`. +proof fn lemma_point_at(basis: Seq, i: nat, v: usize) + requires + i < basis.len() <= 63, + normalized_subspace_poly(basis, i, basis[i as int].0) == 1, + ensures + normalized_subspace_poly(basis, i, span(basis, v)) == (if idx_bit(v, i) { + 1u64 + } else { + 0u64 + }) ^ normalized_subspace_poly(basis, i, span(basis.skip((i + 1) as int), v >> ((i + 1) as usize))), +{ + lemma_span_split(basis, i, v); + let lo = span(basis.take(i as int), v); + let sel = if idx_bit(v, i) { + basis[i as int].0 + } else { + 0 + }; + let hi = span(basis.skip((i + 1) as int), v >> ((i + 1) as usize)); + lemma_normalized_additive(basis, i, lo, sel ^ hi); + lemma_normalized_additive(basis, i, sel, hi); + let c = k_inv(subspace_poly(basis, i, basis[i as int].0)); + lemma_subspace_poly_kills_prefix(basis, i, basis.take(i as int), v); + lemma_k_mul_zero(c); + lemma_xor_facts(normalized_subspace_poly(basis, i, sel ^ hi), 0, 0); +} + +/// Bit `i` of `v` against `v mod 2^(i+1)`. +proof fn lemma_bit_mod(v: usize, i: nat) + requires + i < 63, + ensures + v as int % pow2(i + 1) as int == (if idx_bit(v, i) { + pow2(i) as int + } else { + 0 + }) + v as int % pow2(i) as int, + (v as int % pow2(i) as int) < pow2(i), + (v >> ((i + 1) as usize)) == v as int / pow2(i + 1) as int, +{ + lemma_pow2_pos(i); + lemma_pow2_unfold(i + 1); + lemma_usize_shr_is_div(v, i as usize); + lemma_usize_shr_is_div(v, (i + 1) as usize); + lemma_mod_breakdown(v as int, pow2(i) as int, 2); + lemma_mod_pos_bound(v as int, pow2(i) as int); + let w = v >> (i as usize); + assert(w & 1 == 1 <==> w % 2 == 1) by (bit_vector); + assert(pow2(i + 1) == pow2(i) * 2) by (nonlinear_arith) + requires + pow2(i + 1) == 2 * pow2(i), + ; + let q = v as int / pow2(i) as int; + assert(w as int == q); + lemma_mod_pos_bound(q, 2); + let y = pow2(i) as int; + if idx_bit(v, i) { + assert(q % 2 == 1); + assert(y * (q % 2) == y); + } else { + assert(q % 2 == 0); + assert(y * (q % 2) == 0); + } +} + +/// After `k` forward layers (code layers `0..k`) of a full single-lane transform, word `v` holds the +/// level-`L - k` part of the polynomial at its point: the annex's invariant, with `i = L - k`. +proof fn lemma_forward_invariant(tab: Seq>, basis: Seq, a: Seq, k: nat) + requires + AdditiveNttF64::is_table_of(tab, basis), + 1 <= basis.len() <= 63, + a.len() == pow2(basis.len()), + forall|i: nat| i < basis.len() ==> normalized_subspace_poly(basis, i, #[trigger] basis[i as int].0) == 1, + k <= basis.len(), + ensures + ({ + let ll = basis.len(); + let y = forward_layers(tab, a, 1, ll, 0, k); + &&& y.len() == a.len() + &&& forall|v: int| + 0 <= v < a.len() ==> (#[trigger] y[v]).0 == novel_eval( + basis, + a, + ll, + (ll - k) as nat, + v % pow2((ll - k) as nat) as int, + span(basis, v as usize), + ) + }), + decreases k, +{ + let ll = basis.len(); + lemma_forward_layers_len(tab, a, 1, ll, 0, k); + if k == 0 { + assert forall|v: int| 0 <= v < a.len() implies (#[trigger] a[v]).0 == novel_eval( + basis, + a, + ll, + ll, + v % pow2(ll) as int, + span(basis, v as usize), + ) by { + lemma_small_mod(v as nat, pow2(ll)); + } + } else { + let layer = (k - 1) as nat; + let i = (ll - k) as nat; + lemma_forward_invariant(tab, basis, a, layer); + let y = forward_layers(tab, a, 1, ll, 0, layer); + let h = layer_half(ll, layer); + let tw = layer_twiddles(tab, layer); + let nb = pow2(layer); + lemma_layer_shape(ll, layer, 1); + assert(h == pow2(i)); + lemma_pow2_unfold(i + 1); + lemma2_to64(); + lemma2_to64_rest(); + lemma_pow2_strictly_increases(ll, 64); + assert forall|v: int| 0 <= v < a.len() implies (#[trigger] forward_layers(tab, a, 1, ll, 0, k)[v]).0 + == novel_eval(basis, a, ll, i, v % pow2(i) as int, span(basis, v as usize)) by { + lemma_forward_step_word(tab, basis, a, y, ll, layer, i, h, nb, v); + } + } +} + +/// One word of the inductive step of [`lemma_forward_invariant`]. +proof fn lemma_forward_step_word( + tab: Seq>, + basis: Seq, + a: Seq, + y: Seq, + ll: nat, + layer: nat, + i: nat, + h: nat, + nb: nat, + v: int, +) + requires + AdditiveNttF64::is_table_of(tab, basis), + ll == basis.len(), + 1 <= ll <= 63, + layer < ll, + i == ll - layer - 1, + h == pow2(i), + h == layer_half(ll, layer), + nb == pow2(layer), + pow2(i + 1) == 2 * h, + a.len() == pow2(ll), + a.len() < pow2(64), + pow2(64) == 0x1_0000_0000_0000_0000, + nb * (2 * h) * 1 == 1 * pow2(ll), + forall|i0: nat| i0 < ll ==> normalized_subspace_poly(basis, i0, #[trigger] basis[i0 as int].0) == 1, + y == forward_layers(tab, a, 1, ll, 0, layer), + y.len() == a.len(), + forall|w: int| + 0 <= w < a.len() ==> (#[trigger] y[w]).0 == novel_eval( + basis, + a, + ll, + i + 1, + w % pow2(i + 1) as int, + span(basis, w as usize), + ), + 0 <= v < a.len(), + ensures + forward_layers(tab, a, 1, ll, 0, layer + 1)[v].0 == novel_eval( + basis, + a, + ll, + i, + v % pow2(i) as int, + span(basis, v as usize), + ), +{ + let tw = layer_twiddles(tab, layer); + let z = forward_layers(tab, a, 1, ll, 0, layer + 1); + assert(z == layer_map(y, 1, h, tw, false)); + lemma_decompose(v, 1, h); + lemma_blk_bound(v, 1, h, nb); + let (b, r) = (blk_of(v, 1, h), r_of(v, 1, h)); + assert(row_of(v, 1) == v); + // The twiddle of the block is `Ŵ_i` of the block's point. + lemma_twiddle_is_subspace_poly(tab, basis, layer, b as usize); + let t = tw(b); + let ell = v % pow2(i) as int; + // The pair of `v`: the top word `p` and the bottom word `q = p + h`, of the same block. + let (p, q) = if r < h { + (v, v + h) + } else { + (v - h, v) + }; + let rp = if r < h { + r + } else { + r - h + }; + lemma_compose(b, rp, 0, 1, h); + lemma_compose(b, rp + h, 0, 1, h); + lemma_bound(b, rp + h, 0, 1, h, nb); + assert((b * (2 * h) + rp) * 1 + 0 == p) by (nonlinear_arith) + requires + v == (b * (2 * h) + r) * 1 + 0, + p == (if r < h { v } else { v - h }), + rp == (if r < h { r } else { r - h }), + ; + assert((b * (2 * h) + (rp + h)) * 1 + 0 == q) by (nonlinear_arith) + requires + p == (b * (2 * h) + rp) * 1 + 0, + q == p + h, + ; + // Bit `i` of the two indices, and their residues. + lemma_bit_mod(p as usize, i); + lemma_bit_mod(q as usize, i); + lemma_bit_mod(v as usize, i); + lemma_fundamental_div_mod_converse(p, (2 * h) as int, b, rp); + lemma_fundamental_div_mod_converse(q, (2 * h) as int, b, rp + h); + assert(!idx_bit(p as usize, i)); + assert(idx_bit(q as usize, i)); + assert(p % pow2(i) as int == ell && q % pow2(i) as int == ell); + assert(p % pow2(i + 1) as int == ell && q % pow2(i + 1) as int == ell + pow2(i)); + // Both points agree on every `Ŵ_j`, `j > i`, and differ by one on `Ŵ_i`. + let hi = span(basis.skip((i + 1) as int), b as usize); + assert((p as usize) >> ((i + 1) as usize) == b as usize && (q as usize) >> ((i + 1) as usize) == b as usize); + let (xp, xq, xv) = (span(basis, p as usize), span(basis, q as usize), span(basis, v as usize)); + lemma_point_at(basis, i, p as usize); + lemma_point_at(basis, i, q as usize); + assert forall|j: nat| i + 1 <= j < ll implies normalized_subspace_poly(basis, j, xp) == normalized_subspace_poly( + basis, + j, + xv, + ) && normalized_subspace_poly(basis, j, xq) == normalized_subspace_poly(basis, j, xv) by { + lemma_point_above(basis, i, j, p as usize); + lemma_point_above(basis, i, j, q as usize); + lemma_point_above(basis, i, j, v as usize); + } + lemma_novel_eval_congruent(basis, a, ll, i + 1, ell, xp, xv); + lemma_novel_eval_congruent(basis, a, ll, i + 1, ell + pow2(i), xq, xv); + let lo_part = novel_eval(basis, a, ll, i + 1, ell, xv); + let hi_part = novel_eval(basis, a, ll, i + 1, ell + pow2(i), xv); + assert(y[p].0 == lo_part && y[q].0 == hi_part); + assert(normalized_subspace_poly(basis, i, xp) == t) by { + lemma_xor_facts(t, 0, 0); + } + lemma_k_mul_comm(hi_part, t); + lemma_k_mul_comm(hi_part, t ^ 1); + lemma_butterfly_evaluates(lo_part, hi_part, t); + if r < h { + assert(z[v].0 == bf_top(y[p].0, y[q].0, t)); + } else { + assert(normalized_subspace_poly(basis, i, xv) == 1 ^ t); + assert(1u64 ^ t == t ^ 1u64) by (bit_vector); + assert(z[v].0 == bf_bot(y[p].0, y[q].0, t)); + } +} + +/// The forward transform evaluates the novel-basis polynomial on the subspace domain. +/// +/// On one lane of `2^L` words, `L` the table's dimension, word `v` of the output is +/// `P(x_v) = Σ_j a_j X_j(x_v)` (see [`novel_eval`]), where `a` is the input read as novel-basis +/// coefficients and `x_v = Σ_c bit_c(v) b_c` the point of index `v`, bit 0 first (no bit reversal). +/// +/// Hypothesis: every normalizer works, `Ŵ_i(b_i) = 1`, i.e. every row of the table starts with one +/// (`tab[i][0] = 1`); it holds when every `s_i(b_i)` is nonzero. [`lemma_standard_rows_start_with_one`] +/// discharges it for the standard basis. +pub proof fn lemma_forward_evaluates(tab: Seq>, basis: Seq, a: Seq) + requires + AdditiveNttF64::is_table_of(tab, basis), + 1 <= basis.len() <= 63, + a.len() == pow2(basis.len()), + forall|i: nat| i < basis.len() ==> normalized_subspace_poly(basis, i, #[trigger] basis[i as int].0) == 1, + ensures + forall|v: int| + 0 <= v < a.len() ==> (#[trigger] forward_layers(tab, a, 1, basis.len(), 0, basis.len())[v]).0 + == novel_eval(basis, a, basis.len(), 0, 0, span(basis, v as usize)), +{ + lemma_forward_invariant(tab, basis, a, basis.len()); + lemma2_to64(); +} + + +// --------------------------------------------------------------------------------------------- +// Reed-Solomon encoding at a rate +// --------------------------------------------------------------------------------------------- +/// The message followed by zeros, `n` words. +pub open spec fn zero_pad(msg: Seq, n: nat) -> Seq { + Seq::new( + n, + |v: int| + if v < msg.len() { + msg[v] + } else { + F64(0) + }, + ) +} + +/// `n` words of copies of the message, back to back. +pub open spec fn replicate(msg: Seq, n: nat) -> Seq { + Seq::new(n, |v: int| msg[v % msg.len() as int]) +} + +/// Layers `0..e` are layers `0..r` followed by layers `r..e`. +pub proof fn lemma_forward_layers_split(tab: Seq>, x: Seq, m: nat, log_d: nat, r: nat, e: nat) + requires + r <= e, + ensures + forward_layers(tab, forward_layers(tab, x, m, log_d, 0, r), m, log_d, r, e) == forward_layers( + tab, + x, + m, + log_d, + 0, + e, + ), + decreases e, +{ + if e > r { + lemma_forward_layers_split(tab, x, m, log_d, r, (e - 1) as nat); + } +} + +/// The first `k <= r` layers on a zero-padded message of `2^(L-r)` words leave word `v` equal to the +/// padded word `v mod 2^(L-k)`: a butterfly whose bottom input is zero copies its top input to both outputs. +proof fn lemma_zero_pad_replicates(tab: Seq>, msg: Seq, ll: nat, r: nat, k: nat) + requires + r <= ll, + k <= r, + msg.len() == pow2((ll - r) as nat), + ensures + ({ + let x = zero_pad(msg, pow2(ll)); + let y = forward_layers(tab, x, 1, ll, 0, k); + &&& y.len() == x.len() + &&& forall|v: int| 0 <= v < y.len() ==> #[trigger] y[v] == x[v % pow2((ll - k) as nat) as int] + }), + decreases k, +{ + let x = zero_pad(msg, pow2(ll)); + lemma_forward_layers_len(tab, x, 1, ll, 0, k); + if k == 0 { + assert forall|v: int| 0 <= v < x.len() implies #[trigger] x[v] == x[v % pow2(ll) as int] by { + lemma_small_mod(v as nat, pow2(ll)); + } + } else { + let layer = (k - 1) as nat; + lemma_zero_pad_replicates(tab, msg, ll, r, layer); + let y = forward_layers(tab, x, 1, ll, 0, layer); + let h = layer_half(ll, layer); + let nb = pow2(layer); + let tw = layer_twiddles(tab, layer); + lemma_layer_shape(ll, layer, 1); + assert(h == pow2((ll - k) as nat)); + if (ll - k) as nat > (ll - r) as nat { + lemma_pow2_strictly_increases((ll - r) as nat, (ll - k) as nat); + } + let z = forward_layers(tab, x, 1, ll, 0, k); + assert(z == layer_map(y, 1, h, tw, false)); + assert forall|v: int| 0 <= v < z.len() implies #[trigger] z[v] == x[v % h as int] by { + lemma_decompose(v, 1, h); + lemma_blk_bound(v, 1, h, nb); + let (b, rr) = (blk_of(v, 1, h), r_of(v, 1, h)); + assert(row_of(v, 1) == v); + let t = tw(b); + if rr < h { + let q = v + h; + lemma_compose(b, rr + h, 0, 1, h); + lemma_bound(b, rr + h, 0, 1, h, nb); + assert((b * (2 * h) + (rr + h)) * 1 + 0 == q) by (nonlinear_arith) + requires + v == (b * (2 * h) + rr) * 1 + 0, + q == v + h, + ; + lemma_fundamental_div_mod_converse(q, (2 * h) as int, b, rr + h); + lemma_fundamental_div_mod_converse(v, (2 * h) as int, b, rr); + assert(y[q] == F64(0)); + lemma_k_mul_zero(t); + lemma_xor_facts(y[v].0, 0, 0); + assert(v == (2 * b) * h + rr) by (nonlinear_arith) + requires + v == (b * (2 * h) + rr) * 1 + 0, + ; + lemma_fundamental_div_mod_converse(v, h as int, 2 * b, rr); + } else { + let p = v - h; + lemma_compose(b, rr - h, 0, 1, h); + assert((b * (2 * h) + (rr - h)) * 1 + 0 == p) by (nonlinear_arith) + requires + v == (b * (2 * h) + rr) * 1 + 0, + p == v - h, + ; + lemma_fundamental_div_mod_converse(p, (2 * h) as int, b, rr - h); + lemma_fundamental_div_mod_converse(v, (2 * h) as int, b, rr); + assert(y[v] == F64(0)); + lemma_k_mul_zero(t); + lemma_xor_facts(y[p].0, 0, 0); + assert(v == (2 * b + 1) * h + (rr - h)) by (nonlinear_arith) + requires + v == (b * (2 * h) + rr) * 1 + 0, + ; + lemma_fundamental_div_mod_converse(v, h as int, 2 * b + 1, rr - h); + } + } + } +} + +/// The encoder at rate `2^-r`: the layers `r..L` on `2^r` copies of a message of `2^(L-r)` words give the +/// evaluations, on the whole `2^L`-point domain, of the novel-basis polynomial whose coefficients are the +/// message (degree below `2^(L-r)`). So the codeword is the Reed-Solomon encoding of the message. +/// +/// Same hypothesis as [`lemma_forward_evaluates`]: every row of the table starts with one. +pub proof fn lemma_encode_evaluates(tab: Seq>, basis: Seq, msg: Seq, r: nat) + requires + AdditiveNttF64::is_table_of(tab, basis), + 1 <= basis.len() <= 63, + r <= basis.len(), + msg.len() == pow2((basis.len() - r) as nat), + forall|i: nat| i < basis.len() ==> normalized_subspace_poly(basis, i, #[trigger] basis[i as int].0) == 1, + ensures + ({ + let ll = basis.len(); + let codeword = forward_layers(tab, replicate(msg, pow2(ll)), 1, ll, r, ll); + &&& codeword.len() == pow2(ll) + &&& forall|v: int| + 0 <= v < pow2(ll) ==> (#[trigger] codeword[v]).0 == novel_eval( + basis, + zero_pad(msg, pow2(ll)), + ll, + 0, + 0, + span(basis, v as usize), + ) + }), +{ + let ll = basis.len(); + let x = zero_pad(msg, pow2(ll)); + let rep = replicate(msg, pow2(ll)); + lemma_zero_pad_replicates(tab, msg, ll, r, r); + lemma_pow2_pos((ll - r) as nat); + if r > 0 { + lemma_pow2_strictly_increases((ll - r) as nat, ll); + } + assert forall|v: int| 0 <= v < pow2(ll) implies x[v % pow2((ll - r) as nat) as int] == rep[v] by { + lemma_mod_pos_bound(v, pow2((ll - r) as nat) as int); + } + assert(forward_layers(tab, x, 1, ll, 0, r) =~= rep); + lemma_forward_layers_split(tab, x, 1, ll, r, ll); + lemma_forward_layers_len(tab, rep, 1, ll, r, ll); + lemma_forward_evaluates(tab, basis, x); +} + + +// --------------------------------------------------------------------------------------------- +// The standard basis: every row of the table starts with one +// --------------------------------------------------------------------------------------------- +/// `K` has no zero divisors. +pub proof fn lemma_k_no_zero_divisors(a: u64, b: u64) + requires + k_mul(a, b) == 0, + a != 0, + ensures + b == 0, +{ + let ai = k_pow(a, (pow2(64) - 2) as nat); + lemma_k_inverse(a); + lemma_k_mul_comm(a, ai); + lemma_k_mul_assoc(ai, a, b); + lemma_k_mul_one(b); + lemma_k_mul_zero(ai); +} + +proof fn lemma_below_pow(x: u64, k: u64) + requires + k < 63, + ensures + x < (1u64 << k) ==> x < (1u64 << ((k + 1) as u64)), + (x ^ (1u64 << k)) < (1u64 << k) ==> x < (1u64 << ((k + 1) as u64)), + !((1u64 << ((k + 1) as u64)) < (1u64 << ((k + 1) as u64))), +{ + assert((x < (1u64 << k) ==> x < (1u64 << ((k + 1) as u64))) && ((x ^ (1u64 << k)) < (1u64 << k) ==> x < (1u64 + << ((k + 1) as u64)))) by (bit_vector) + requires + k < 63, + ; +} + +/// The roots of `s_i` for the standard basis lie below `2^i`: `s_i(x) = 0` only on `{0, .., 2^i - 1}`. +pub proof fn lemma_standard_roots(dim: nat, i: nat, x: u64) + requires + i <= dim <= 63, + subspace_poly(standard_basis(dim), i, x) == 0, + ensures + x < (1u64 << (i as u64)), + decreases i, +{ + let basis = standard_basis(dim); + if i == 0 { + assert(x == 0); + assert(0u64 < (1u64 << 0u64)) by (bit_vector); + } else { + let j = (i - 1) as nat; + let b = basis[j as int].0; + assert(b == 1u64 << (j as u64)); + let p = subspace_poly(basis, j, x); + let c = subspace_poly(basis, j, b); + lemma_below_pow(x, j as u64); + if p == 0 { + lemma_standard_roots(dim, j, x); + } else { + lemma_k_no_zero_divisors(p, p ^ c); + lemma_subspace_poly_additive(basis, j, x, b); + lemma_standard_roots(dim, j, x ^ b); + } + } +} + +/// For the standard basis every normalizer works: `Ŵ_i(b_i) = 1`, so every row of the table starts with one. +pub proof fn lemma_standard_rows_start_with_one(dim: nat) + requires + dim <= 63, + ensures + forall|i: nat| + i < dim ==> normalized_subspace_poly(standard_basis(dim), i, #[trigger] standard_basis(dim)[i as int].0) + == 1, +{ + assert forall|i: nat| i < dim implies normalized_subspace_poly( + standard_basis(dim), + i, + #[trigger] standard_basis(dim)[i as int].0, + ) == 1 by { + let basis = standard_basis(dim); + let y = subspace_poly(basis, i, basis[i as int].0); + assert(basis[i as int].0 == 1u64 << (i as u64)); + if y == 0 { + lemma_standard_roots(dim, i, basis[i as int].0); + let k = i as u64; + assert(!((1u64 << k) < (1u64 << k))) by (bit_vector); + } + lemma_k_inverse(y); + } +} + +/// The point of index `v` in the standard basis is `v` itself: the domain is `{0, .., 2^dim - 1}` under the +/// natural integer encoding. +pub proof fn lemma_standard_point(n: nat, v: usize) + requires + n <= 63, + v < pow2(n), + ensures + span(standard_basis(n), v) == v as u64, +{ + lemma_standard_span(n, v); + let k = n as u64; + lemma_u64_pow2_no_overflow(n); + assert(1 * pow2(n) == pow2(n)); + lemma_u64_shl_is_mul(1, k); + let w = v as u64; + assert(w < (1u64 << k) ==> w & sub(1u64 << k, 1) == w) by (bit_vector); +} + +proof fn lemma_standard_span(n: nat, v: usize) + requires + n <= 63, + ensures + span(standard_basis(n), v) == (v as u64) & sub(1u64 << (n as u64), 1), + decreases n, +{ + let w = v as u64; + if n == 0 { + assert(w & sub(1u64 << 0u64, 1) == 0) by (bit_vector); + } else { + let k = (n - 1) as u64; + lemma_standard_span((n - 1) as nat, v); + assert(standard_basis(n).drop_last() =~= standard_basis((n - 1) as nat)); + assert(standard_basis(n).last().0 == 1u64 << k); + let kk = k as usize; + assert(((v >> kk) & 1 == 1) == ((w >> k) & 1 == 1)) by (bit_vector) + requires + w == v as u64, + kk == k as usize, + k < 63, + ; + assert(((w >> k) & 1 == 1 ==> (w & sub(1u64 << k, 1)) ^ (1u64 << k) == w & sub(1u64 << ((k + 1) as u64), 1)) + && (!((w >> k) & 1 == 1) ==> (w & sub(1u64 << k, 1)) ^ 0 == w & sub(1u64 << ((k + 1) as u64), 1))) + by (bit_vector) + requires + k < 63, + ; + } +} + +/// The forward transform of `AdditiveNttF64::standard(dim)` evaluates the novel-basis polynomial on the +/// domain `{0, .., 2^dim - 1}`: output word `v` is `P(v) = Σ_{j < 2^dim} a_j X_j(v)` ([`novel_sum`]), with +/// no hypothesis. +pub proof fn lemma_standard_forward_evaluates(tab: Seq>, dim: nat, a: Seq) + requires + AdditiveNttF64::is_table_of(tab, standard_basis(dim)), + 1 <= dim <= 63, + a.len() == pow2(dim), + ensures + forall|v: int| + 0 <= v < a.len() ==> (#[trigger] forward_layers(tab, a, 1, dim, 0, dim)[v]).0 == novel_sum( + standard_basis(dim), + a, + dim, + v as u64, + ), +{ + lemma_standard_rows_start_with_one(dim); + lemma_forward_evaluates(tab, standard_basis(dim), a); + assert forall|v: int| 0 <= v < a.len() implies novel_eval(standard_basis(dim), a, dim, 0, 0, v as u64) == #[trigger] novel_sum( + standard_basis(dim), + a, + dim, + v as u64, + ) by { + lemma_novel_eval_flat(standard_basis(dim), a, dim, v as u64); + } + assert forall|v: int| 0 <= v < a.len() implies #[trigger] span(standard_basis(dim), v as usize) == v as u64 by { + lemma2_to64(); + lemma2_to64_rest(); + lemma_pow2_strictly_increases(dim, 64); + lemma_standard_point(dim, v as usize); + } +} + +/// The encoder of `AdditiveNttF64::standard(dim)` at rate `2^-r` is the Reed-Solomon encoding of the +/// message: word `v` of the codeword is `P(v)`, `P` the novel-basis polynomial whose coefficients are the +/// message (degree below `2^(dim - r)`), with no hypothesis. +pub proof fn lemma_standard_encode_evaluates(tab: Seq>, dim: nat, msg: Seq, r: nat) + requires + AdditiveNttF64::is_table_of(tab, standard_basis(dim)), + 1 <= dim <= 63, + r <= dim, + msg.len() == pow2((dim - r) as nat), + ensures + ({ + let codeword = forward_layers(tab, replicate(msg, pow2(dim)), 1, dim, r, dim); + &&& codeword.len() == pow2(dim) + &&& forall|v: int| + 0 <= v < pow2(dim) ==> (#[trigger] codeword[v]).0 == novel_sum( + standard_basis(dim), + zero_pad(msg, pow2(dim)), + dim, + v as u64, + ) + }), +{ + lemma_standard_rows_start_with_one(dim); + lemma_encode_evaluates(tab, standard_basis(dim), msg, r); + let x = zero_pad(msg, pow2(dim)); + assert forall|v: int| 0 <= v < pow2(dim) implies novel_eval(standard_basis(dim), x, dim, 0, 0, v as u64) == #[trigger] novel_sum( + standard_basis(dim), + x, + dim, + v as u64, + ) by { + lemma_novel_eval_flat(standard_basis(dim), x, dim, v as u64); + } + assert forall|v: int| 0 <= v < pow2(dim) implies #[trigger] span(standard_basis(dim), v as usize) == v as u64 by { + lemma2_to64(); + lemma2_to64_rest(); + lemma_pow2_strictly_increases(dim, 64); + lemma_standard_point(dim, v as usize); + } +} + + +// --------------------------------------------------------------------------------------------- +// The novel-basis polynomial as a flat sum +// --------------------------------------------------------------------------------------------- +/// `XOR_{j < n} f(j)`. +pub open spec fn xor_sum(f: spec_fn(int) -> u64, n: nat) -> u64 + decreases n, +{ + if n == 0 { + 0 + } else { + xor_sum(f, (n - 1) as nat) ^ f(n - 1) + } +} + +/// `Π_{c < n} Ŵ_(i+c)(x)^(bit_c(j))`, bit `c` of `j` being `(j / 2^c) mod 2`. +pub open spec fn novel_basis_from(basis: Seq, i: nat, n: nat, j: nat, x: u64) -> u64 + decreases n, +{ + if n == 0 { + 1 + } else { + k_mul( + if j % 2 == 1 { + normalized_subspace_poly(basis, i, x) + } else { + 1 + }, + novel_basis_from(basis, i + 1, (n - 1) as nat, j / 2, x), + ) + } +} + +/// The novel basis polynomial `X_j(x) = Π_{i < d} Ŵ_i(x)^(bit_i(j))`. +pub open spec fn novel_basis(basis: Seq, d: nat, j: nat, x: u64) -> u64 { + novel_basis_from(basis, 0, d, j, x) +} + +/// `Σ_{j < 2^(d-i)} a[l + 2^i j] · Π_{c < d-i} Ŵ_(i+c)(x)^(bit_c(j))`. +pub open spec fn novel_sum_from(basis: Seq, a: Seq, d: nat, i: nat, l: int, x: u64) -> u64 { + xor_sum( + |j: int| k_mul(a[l + pow2(i) * j].0, novel_basis_from(basis, i, (d - i) as nat, j as nat, x)), + pow2((d - i) as nat), + ) +} + +/// The novel-basis polynomial with coefficients `a`, as the flat sum `Σ_{j < 2^d} a_j X_j(x)`. +pub open spec fn novel_sum(basis: Seq, a: Seq, d: nat, x: u64) -> u64 { + xor_sum(|j: int| k_mul(a[j].0, novel_basis(basis, d, j as nat, x)), pow2(d)) +} + +proof fn lemma_xor_sum_ext(f: spec_fn(int) -> u64, g: spec_fn(int) -> u64, n: nat) + requires + forall|j: int| 0 <= j < n ==> #[trigger] f(j) == g(j), + ensures + xor_sum(f, n) == xor_sum(g, n), + decreases n, +{ + if n > 0 { + lemma_xor_sum_ext(f, g, (n - 1) as nat); + } +} + +/// A sum over `2n` terms is the sum of its even terms plus the sum of its odd terms. +proof fn lemma_xor_sum_split(f: spec_fn(int) -> u64, n: nat) + ensures + xor_sum(f, 2 * n) == xor_sum(|j: int| f(2 * j), n) ^ xor_sum(|j: int| f(2 * j + 1), n), + decreases n, +{ + let fe = |j: int| f(2 * j); + let fo = |j: int| f(2 * j + 1); + if n == 0 { + lemma_xor_facts(0, 0, 0); + } else { + lemma_xor_sum_split(f, (n - 1) as nat); + let (a, b) = (xor_sum(fe, (n - 1) as nat), xor_sum(fo, (n - 1) as nat)); + let (e, o) = (f(2 * n - 2), f(2 * n - 1)); + assert(xor_sum(f, (2 * n - 1) as nat) == xor_sum(f, (2 * (n - 1)) as nat) ^ e); + assert(xor_sum(f, 2 * n) == xor_sum(f, (2 * (n - 1)) as nat) ^ e ^ o); + assert((a ^ b) ^ e ^ o == (a ^ e) ^ (b ^ o)) by (bit_vector); + } +} + +/// Multiplication distributes over a sum. +proof fn lemma_xor_sum_scale(c: u64, f: spec_fn(int) -> u64, n: nat) + ensures + k_mul(c, xor_sum(f, n)) == xor_sum(|j: int| k_mul(c, f(j)), n), + decreases n, +{ + if n == 0 { + lemma_k_mul_zero(c); + } else { + lemma_xor_sum_scale(c, f, (n - 1) as nat); + lemma_k_mul_xor_right(c, xor_sum(f, (n - 1) as nat), f(n - 1)); + } +} + +/// `a (w p) = w (a p)`. +proof fn lemma_k_mul_swap(a: u64, w: u64, p: u64) + ensures + k_mul(a, k_mul(w, p)) == k_mul(w, k_mul(a, p)), +{ + lemma_k_mul_assoc(a, w, p); + lemma_k_mul_comm(a, w); + lemma_k_mul_assoc(w, a, p); +} + +proof fn lemma_novel_eval_flat_from(basis: Seq, a: Seq, d: nat, i: nat, l: int, x: u64) + requires + i <= d, + ensures + novel_eval(basis, a, d, i, l, x) == novel_sum_from(basis, a, d, i, l, x), + decreases d - i, +{ + let n = (d - i) as nat; + let f = |j: int| k_mul(a[l + pow2(i) * j].0, novel_basis_from(basis, i, n, j as nat, x)); + if i == d { + lemma2_to64(); + assert(pow2(i) * 0 == 0); + lemma_k_mul_one(a[l].0); + lemma_xor_facts(a[l].0, 0, 0); + assert(xor_sum(f, 0) == 0); + assert(xor_sum(f, 1) == 0 ^ f(0)); + } else { + let m = (n - 1) as nat; + let half = pow2(m); + let w = normalized_subspace_poly(basis, i, x); + let li = l + pow2(i); + lemma_novel_eval_flat_from(basis, a, d, i + 1, l, x); + lemma_novel_eval_flat_from(basis, a, d, i + 1, li, x); + let g0 = |j: int| k_mul(a[l + pow2(i + 1) * j].0, novel_basis_from(basis, i + 1, m, j as nat, x)); + let g1 = |j: int| k_mul(a[li + pow2(i + 1) * j].0, novel_basis_from(basis, i + 1, m, j as nat, x)); + assert((d - (i + 1)) as nat == m); + lemma_pow2_unfold(n); + lemma_pow2_unfold(i + 1); + lemma_xor_sum_split(f, half); + let fe = |j: int| f(2 * j); + let fo = |j: int| f(2 * j + 1); + assert forall|j: int| 0 <= j < half implies #[trigger] fe(j) == g0(j) by { + assert(pow2(i) * (2 * j) == pow2(i + 1) * j) by (nonlinear_arith) + requires + pow2(i + 1) == 2 * pow2(i), + ; + assert((2 * j) % 2 == 0 && (2 * j) / 2 == j); + assert(((2 * j) as nat) / 2 == j as nat); + lemma_k_mul_one(novel_basis_from(basis, i + 1, m, j as nat, x)); + } + assert forall|j: int| 0 <= j < half implies #[trigger] fo(j) == k_mul(w, g1(j)) by { + assert(pow2(i) * (2 * j + 1) == pow2(i) + pow2(i + 1) * j) by (nonlinear_arith) + requires + pow2(i + 1) == 2 * pow2(i), + ; + assert((2 * j + 1) % 2 == 1 && (2 * j + 1) / 2 == j); + assert(((2 * j + 1) as nat) / 2 == j as nat); + lemma_k_mul_swap( + a[li + pow2(i + 1) * j].0, + w, + novel_basis_from(basis, i + 1, m, j as nat, x), + ); + } + lemma_xor_sum_ext(fe, g0, half); + lemma_xor_sum_ext(fo, |j: int| k_mul(w, g1(j)), half); + lemma_xor_sum_scale(w, g1, half); + assert(2 * half == pow2(n)); + } +} + +/// The even-odd form is the flat sum: `novel_eval(0, 0, x) = Σ_{j < 2^d} a_j X_j(x)`. +pub proof fn lemma_novel_eval_flat(basis: Seq, a: Seq, d: nat, x: u64) + ensures + novel_eval(basis, a, d, 0, 0, x) == novel_sum(basis, a, d, x), +{ + lemma_novel_eval_flat_from(basis, a, d, 0, 0, x); + lemma2_to64(); + let f = |j: int| k_mul(a[0 + pow2(0) * j].0, novel_basis_from(basis, 0, (d - 0) as nat, j as nat, x)); + let g = |j: int| k_mul(a[j].0, novel_basis(basis, d, j as nat, x)); + assert forall|j: int| 0 <= j < pow2(d) implies #[trigger] f(j) == g(j) by { + assert(0 + pow2(0) * j == j); + } + lemma_xor_sum_ext(f, g, pow2(d)); +} + +} // verus! diff --git a/verification/verus/src/ntt_driver.rs b/verification/verus/src/ntt_driver.rs new file mode 100644 index 0000000000..76e3c39cc0 --- /dev/null +++ b/verification/verus/src/ntt_driver.rs @@ -0,0 +1,3735 @@ +//! Permission-backed executable refinement of the parallel additive NTT driver. +//! `run_layers`, `fused_rows` and row groups copy the portable production operations; +//! `transform` composes a caller-supplied gathered/deep plan on populated replicas. +//! +//! Memory is held as [`crate::parallel::owns`] permissions. Raw accesses require the +//! touched elements' permissions; distinct dispatched tasks receive disjoint maps. +//! The real pool's exactly-once joined dispatch is trusted, not proved. Production's +//! cache planner, fused replication, row sinks and streaming fences are not verified. +//! Function comments document rewrites; differential tests supply finite evidence only. +//! +//! Specification: +//! +//! - [`gather`]: rows `base + r + i * step` of a buffer, as a buffer of their own. +//! - [`sub_layers`]: the forward layers `first..end` of a `2^log_d`-row transform run on one of its +//! `2^o` sub-blocks, which fixes the global block index, and so the twiddle, of each block. +//! +//! Main results: a layer commutes with a gather of rows it pairs among themselves +//! ([`lemma_gather_layer`], [`lemma_gather_sub_layers`]); `run_layers`, the fused groups and +//! `gathered_pass` run the layers they claim; `transform` computes [`forward_layers`], the specification of +//! the verified reference `forward_scalar_from_layer`. +// `global size_of usize` expands to a braced block. +#![allow(unused_braces)] + +use crate::gf2_64::*; +use crate::ntt::*; +use crate::parallel; +use crate::parallel::*; +use vstd::arithmetic::div_mod::*; +use vstd::arithmetic::mul::*; +use vstd::arithmetic::power2::*; +use vstd::bits::*; +use vstd::prelude::*; +use vstd::raw_ptr::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// The index in the buffer of word `q` of a gather: row `base + r + (q / m) * step`, lane `q % m`. +pub open spec fn gidx(m: nat, base: int, r: int, step: int, q: int) -> int { + (base + r + (q / (m as int)) * step) * (m as int) + q % (m as int) +} + +/// Rows `base + r + i * step`, `i < cnt`, of a buffer of `m` lanes, as one buffer of `cnt` rows. +pub open spec fn gather(x: Seq, m: nat, base: int, r: int, step: int, cnt: nat) -> Seq { + Seq::new(cnt * m, |q: int| x[gidx(m, base, r, step, q)]) +} + +/// The twiddles of one layer on sub-block `s` of `2^o`: block `b` of the sub-block is block +/// `s * 2^(layer - o) + b` of the domain. +pub open spec fn sub_twiddles(tab: Seq>, layer: nat, o: nat, s: int) -> spec_fn(int) -> u64 { + |b: int| twiddle_spec(tab, layer, (s * pow2((layer - o) as nat) + b) as usize) +} + +/// The forward layers `first..end` of a `2^log_d`-row transform on `m` lanes, run on its sub-block `s` +/// of `2^o` (`o <= first`), a buffer of `2^(log_d - o)` rows. +pub open spec fn sub_layers( + tab: Seq>, + x: Seq, + m: nat, + log_d: nat, + o: nat, + s: int, + first: nat, + end: nat, +) -> Seq + decreases end, +{ + if end <= first { + x + } else { + layer_map( + sub_layers(tab, x, m, log_d, o, s, first, (end - 1) as nat), + m, + layer_half(log_d, (end - 1) as nat), + sub_twiddles(tab, (end - 1) as nat, o, s), + false, + ) + } +} + +// --------------------------------------------------------------------------------------------- +// Arithmetic +// --------------------------------------------------------------------------------------------- +/// Word `lane` of row `row` is word `row * m + lane`. +pub proof fn lemma_word(row: int, lane: int, m: int) + requires + m > 0, + 0 <= lane < m, + ensures + (row * m + lane) / m == row, + (row * m + lane) % m == lane, +{ + lemma_fundamental_div_mod_converse(row * m + lane, m, row, lane); +} + +/// Every word is `row * m + lane`. +pub proof fn lemma_split_word(q: int, m: int) + requires + m > 0, + q >= 0, + ensures + q == (q / m) * m + q % m, + q / m >= 0, + 0 <= q % m < m, +{ + lemma_fundamental_div_mod(q, m); + lemma_mod_bound(q, m); + lemma_div_pos_is_pos(q, m); + lemma_mul_is_commutative(q / m, m); +} + +/// Row `base + r + i * step`, with `base` a multiple of `2H = 2 k step` and `r < step`, lies in block +/// `base / 2H + i / 2k`, at row `(i % 2k) * step + r` of it: a top row exactly when `i % 2k < k`. +pub proof fn lemma_row_split(base: int, r: int, i: int, step: int, k: int) + requires + step > 0, + 0 <= r < step, + k > 0, + i >= 0, + base >= 0, + base % (2 * k * step) == 0, + ensures + (base + r + i * step) / (2 * k * step) == base / (2 * k * step) + i / (2 * k), + (base + r + i * step) % (2 * k * step) == (i % (2 * k)) * step + r, + ((i % (2 * k)) * step + r < k * step) == (i % (2 * k) < k), +{ + let h2 = 2 * k * step; + assert(h2 > 0) by (nonlinear_arith) + requires + k > 0, + step > 0, + h2 == 2 * k * step, + ; + let a = i / (2 * k); + let j = i % (2 * k); + let c0 = base / h2; + lemma_fundamental_div_mod(i, 2 * k); + lemma_mod_bound(i, 2 * k); + lemma_fundamental_div_mod(base, h2); + assert(base + r + i * step == (c0 + a) * h2 + (j * step + r)) by (nonlinear_arith) + requires + i == (2 * k) * a + j, + base == h2 * c0, + h2 == 2 * k * step, + ; + assert(0 <= j * step + r < h2) by (nonlinear_arith) + requires + 0 <= j < 2 * k, + 0 <= r < step, + h2 == 2 * k * step, + ; + lemma_fundamental_div_mod_converse(base + r + i * step, h2, c0 + a, j * step + r); + if j < k { + assert(j * step + r < k * step) by (nonlinear_arith) + requires + 0 <= j <= k - 1, + r < step, + step > 0, + ; + } else { + assert(j * step + r >= k * step) by (nonlinear_arith) + requires + j >= k, + r >= 0, + step > 0, + ; + } +} + +/// A word of a gather below `cnt` rows, and its partner rows `k` away. +proof fn lemma_gather_rows(q: int, m: int, cnt: int, k: int) + requires + m > 0, + k > 0, + cnt % (2 * k) == 0, + 0 <= q < cnt * m, + ensures + 0 <= q / m < cnt, + (q / m) % (2 * k) < k ==> q / m + k < cnt && q + k * m < cnt * m && (q + k * m) / m == q / m + k && (q + k * m) % m + == q % m, + (q / m) % (2 * k) >= k ==> q / m >= k && q - k * m >= 0 && (q - k * m) / m == q / m - k && (q - k * m) % m == q % m, + (q / m) / (2 * k) < cnt / (2 * k), +{ + let i = q / m; + let lane = q % m; + lemma_split_word(q, m); + assert(i < cnt) by (nonlinear_arith) + requires + q == i * m + lane, + 0 <= lane, + q < cnt * m, + m > 0, + ; + let a = i / (2 * k); + let j = i % (2 * k); + let c = cnt / (2 * k); + lemma_fundamental_div_mod(i, 2 * k); + lemma_mod_bound(i, 2 * k); + lemma_fundamental_div_mod(cnt, 2 * k); + assert(a < c) by (nonlinear_arith) + requires + i == (2 * k) * a + j, + cnt == (2 * k) * c, + 0 <= j, + i < cnt, + k > 0, + ; + if j < k { + assert(i + k < cnt) by (nonlinear_arith) + requires + i == (2 * k) * a + j, + cnt == (2 * k) * c, + 0 <= j < k, + a + 1 <= c, + k > 0, + ; + assert(q + k * m == (i + k) * m + lane) by (nonlinear_arith) + requires + q == i * m + lane, + ; + lemma_word(i + k, lane, m); + assert((i + k) * m + lane < cnt * m) by (nonlinear_arith) + requires + i + k + 1 <= cnt, + lane < m, + m > 0, + ; + } else { + assert(q - k * m == (i - k) * m + lane) by (nonlinear_arith) + requires + q == i * m + lane, + ; + lemma_word(i - k, lane, m); + assert((i - k) * m + lane >= 0) by (nonlinear_arith) + requires + i >= k, + lane >= 0, + m > 0, + ; + } +} + +// --------------------------------------------------------------------------------------------- +// A layer commutes with a gather of the rows it pairs +// --------------------------------------------------------------------------------------------- +/// One layer of half `H = k * step`, gathered on rows `base + r + i * step` (`r < step`, `base` a multiple +/// of a block), is the layer of half `k` on the gathered buffer, with the twiddles of the blocks the +/// gathered rows come from. +pub proof fn lemma_gather_layer( + x: Seq, + m: nat, + h: nat, + tw: spec_fn(int) -> u64, + tw2: spec_fn(int) -> u64, + inverse: bool, + base: int, + r: int, + step: int, + cnt: nat, + k: nat, +) + requires + m > 0, + step > 0, + 0 <= r < step, + k > 0, + h == k * step, + cnt % (2 * k) == 0, + base >= 0, + base % (2 * h as int) == 0, + (base + cnt * step) * m <= x.len(), + forall|b: int| 0 <= b < cnt / (2 * k) ==> #[trigger] tw2(b) == tw(base / (2 * h as int) + b), + ensures + gather(layer_map(x, m, h, tw, inverse), m, base, r, step, cnt) == layer_map( + gather(x, m, base, r, step, cnt), + m, + k, + tw2, + inverse, + ), +{ + let y = layer_map(x, m, h, tw, inverse); + let g = gather(x, m, base, r, step, cnt); + let lhs = gather(y, m, base, r, step, cnt); + let rhs = layer_map(g, m, k, tw2, inverse); + let mi = m as int; + assert forall|q: int| 0 <= q < cnt * m implies lhs[q] == rhs[q] by { + let i = q / mi; + let lane = q % mi; + lemma_split_word(q, mi); + lemma_gather_rows(q, mi, cnt as int, k as int); + let row = base + r + i * step; + let p = row * mi + lane; + assert(gidx(m, base, r, step, q) == p); + lemma_word(row, lane, mi); + assert(2 * h as int == 2 * (k as int) * step) by (nonlinear_arith) + requires + h == k * step, + ; + lemma_row_split(base, r, i, step, k as int); + assert(row >= 0) by (nonlinear_arith) + requires + row == base + r + i * step, + base >= 0, + r >= 0, + i >= 0, + step > 0, + ; + assert(row + 1 <= base + cnt * step) by (nonlinear_arith) + requires + row == base + r + i * step, + i + 1 <= cnt, + 0 <= r < step, + step > 0, + ; + assert(p < x.len()) by (nonlinear_arith) + requires + p == row * mi + lane, + row + 1 <= base + cnt * step, + 0 <= lane < mi, + row >= 0, + (base + cnt * step) * mi <= x.len(), + ; + assert(p >= 0) by (nonlinear_arith) + requires + p == row * mi + lane, + row >= 0, + lane >= 0, + mi > 0, + ; + // The full layer at `p`: row `row`, block `base / 2H + i / 2k`, in-block row `(i % 2k) step + r`. + assert(row_of(p, m) == row); + assert(blk_of(p, m, h) == base / (2 * h as int) + i / (2 * k as int)); + assert(r_of(p, m, h) == (i % (2 * k as int)) * step + r); + // The gathered layer at `q`: row `i`, block `i / 2k`, in-block row `i % 2k`. + assert(row_of(q, m) == i); + assert(blk_of(q, m, k) == i / (2 * k as int)); + assert(r_of(q, m, k) == i % (2 * k as int)); + assert(tw2(i / (2 * k as int)) == tw(base / (2 * h as int) + i / (2 * k as int))); + if i % (2 * k as int) < k { + let q2 = q + k * m; + assert((row + k * step) * mi == row * mi + (k * step) * mi) by (nonlinear_arith); + assert((i + k) * step == i * step + k * step) by (nonlinear_arith); + assert(h * m == (k * step) * mi); + assert(gidx(m, base, r, step, q2) == p + h * m); + assert(g[q2] == x[p + h * m]); + } else { + let q2 = q - k * m; + assert((row - k * step) * mi == row * mi - (k * step) * mi) by (nonlinear_arith); + assert((i - k) * step == i * step - k * step) by (nonlinear_arith); + assert(h * m == (k * step) * mi); + assert(gidx(m, base, r, step, q2) == p - h * m); + assert(g[q2] == x[p - h * m]); + } + } + assert(lhs =~= rhs); +} + +/// Layers with twiddles that agree on every block are equal. +pub proof fn lemma_layer_map_tw(x: Seq, m: nat, h: nat, tw: spec_fn(int) -> u64, tw2: spec_fn(int) -> u64, inverse: bool) + requires + m > 0, + h > 0, + forall|b: int| 0 <= b ==> #[trigger] tw(b) == tw2(b), + ensures + layer_map(x, m, h, tw, inverse) == layer_map(x, m, h, tw2, inverse), +{ + assert forall|p: int| 0 <= p < x.len() implies layer_map(x, m, h, tw, inverse)[p] == layer_map(x, m, h, tw2, inverse)[p] by { + lemma_div_pos_is_pos(p, m as int); + lemma_div_pos_is_pos(row_of(p, m), 2 * h as int); + } + assert(layer_map(x, m, h, tw, inverse) =~= layer_map(x, m, h, tw2, inverse)); +} + +/// A gather of `cnt` consecutive rows is a subrange. +pub proof fn lemma_gather_contiguous(x: Seq, m: nat, base: int, cnt: nat) + requires + m > 0, + base >= 0, + (base + cnt) * m <= x.len(), + ensures + gather(x, m, base, 0, 1, cnt) == x.subrange(base * m, (base + cnt) * m), +{ + let mi = m as int; + assert((base + cnt) * m - base * m == cnt * m) by (nonlinear_arith); + assert(base * m >= 0) by (nonlinear_arith) + requires + base >= 0, + ; + assert forall|q: int| 0 <= q < cnt * m implies gather(x, m, base, 0, 1, cnt)[q] == x.subrange(base * m, (base + cnt) * m)[q] by { + lemma_split_word(q, mi); + assert(gidx(m, base, 0, 1, q) == base * mi + q) by (nonlinear_arith) + requires + gidx(m, base, 0, 1, q) == (base + 0 + (q / mi) * 1) * mi + q % mi, + q == (q / mi) * mi + q % mi, + ; + } + assert((base + cnt) * m - base * m == cnt * m) by (nonlinear_arith); + assert(gather(x, m, base, 0, 1, cnt) =~= x.subrange(base * m, (base + cnt) * m)); +} + +proof fn lemma_sub_layers_len(tab: Seq>, x: Seq, m: nat, log_d: nat, o: nat, s: int, first: nat, end: nat) + ensures + sub_layers(tab, x, m, log_d, o, s, first, end).len() == x.len(), + decreases end, +{ + if end > first { + lemma_sub_layers_len(tab, x, m, log_d, o, s, first, (end - 1) as nat); + } +} + +/// Layers `first..end` are layers `first..mid` followed by layers `mid..end`. +pub proof fn lemma_sub_layers_split(tab: Seq>, x: Seq, m: nat, log_d: nat, o: nat, s: int, first: nat, mid: nat, end: nat) + requires + first <= mid <= end, + ensures + sub_layers(tab, sub_layers(tab, x, m, log_d, o, s, first, mid), m, log_d, o, s, mid, end) == sub_layers( + tab, + x, + m, + log_d, + o, + s, + first, + end, + ), + decreases end, +{ + if end > mid { + lemma_sub_layers_split(tab, x, m, log_d, o, s, first, mid, (end - 1) as nat); + } +} + +/// On the whole domain (`o = 0`), the sub-block layers are [`forward_layers`]. +pub proof fn lemma_forward_is_sub(tab: Seq>, x: Seq, m: nat, log_d: nat, first: nat, end: nat) + requires + m > 0, + end <= log_d, + ensures + forward_layers(tab, x, m, log_d, first, end) == sub_layers(tab, x, m, log_d, 0, 0, first, end), + decreases end, +{ + if end > first { + let l = (end - 1) as nat; + lemma_forward_is_sub(tab, x, m, log_d, first, l); + lemma_layer_shape(log_d, l, m); + lemma_layer_map_tw( + sub_layers(tab, x, m, log_d, 0, 0, first, l), + m, + layer_half(log_d, l), + layer_twiddles(tab, l), + sub_twiddles(tab, l, 0, 0), + false, + ); + } +} + +/// The layers of a sub-block, gathered on rows they pair among themselves, are the layers of a smaller +/// domain on the gathered buffer. +/// +/// The buffer `x` is sub-block `s` of `2^o` of a `2^log_d`-row domain. The gather takes the rows +/// `base + r + i * 2^(log_d - log_d2)` of sub-block `s2` of `2^o2` of it, `i < 2^(log_d2 - o2)`; on them the +/// layers `first..end` (`o2 <= first`) are those of sub-block `s2` of a `2^log_d2`-row domain. +pub proof fn lemma_gather_sub_layers( + tab: Seq>, + x: Seq, + m: nat, + log_d: nat, + o: nat, + s: int, + log_d2: nat, + o2: nat, + s2: int, + r: int, + first: nat, + end: nat, +) + requires + m > 0, + o <= o2 <= first <= end <= log_d2 <= log_d, + s >= 0, + s * pow2((o2 - o) as nat) <= s2 < (s + 1) * pow2((o2 - o) as nat), + 0 <= r < pow2((log_d - log_d2) as nat), + x.len() == m * pow2((log_d - o) as nat), + ensures + ({ + let base = (s2 - s * pow2((o2 - o) as nat)) * pow2((log_d - o2) as nat); + let step = pow2((log_d - log_d2) as nat) as int; + let cnt = pow2((log_d2 - o2) as nat); + gather(sub_layers(tab, x, m, log_d, o, s, first, end), m, base, r, step, cnt) == sub_layers( + tab, + gather(x, m, base, r, step, cnt), + m, + log_d2, + o2, + s2, + first, + end, + ) + }), + decreases end, +{ + let rb = s2 - s * pow2((o2 - o) as nat); + let base = rb * pow2((log_d - o2) as nat); + let step = pow2((log_d - log_d2) as nat) as int; + let cnt = pow2((log_d2 - o2) as nat); + if end > first { + let l = (end - 1) as nat; + lemma_gather_sub_layers(tab, x, m, log_d, o, s, log_d2, o2, s2, r, first, l); + let y = sub_layers(tab, x, m, log_d, o, s, first, l); + lemma_sub_layers_len(tab, x, m, log_d, o, s, first, l); + let h = layer_half(log_d, l); + let k = layer_half(log_d2, l); + // H = k * step, 2k = 2^(log_d2 - l), cnt = 2k * 2^(l - o2), 2H = 2^(log_d - l). + lemma_pow2_adds((log_d2 - l - 1) as nat, (log_d - log_d2) as nat); + assert((log_d2 - l - 1) as nat + (log_d - log_d2) as nat == (log_d - l - 1) as nat); + lemma_pow2_unfold((log_d2 - l) as nat); + lemma_pow2_unfold((log_d - l) as nat); + lemma_pow2_adds((log_d2 - l) as nat, (l - o2) as nat); + assert((log_d2 - l) as nat + (l - o2) as nat == (log_d2 - o2) as nat); + lemma_pow2_adds((log_d - l) as nat, (l - o2) as nat); + assert((log_d - l) as nat + (l - o2) as nat == (log_d - o2) as nat); + lemma_pow2_adds((o2 - o) as nat, (log_d - o2) as nat); + assert((o2 - o) as nat + (log_d - o2) as nat == (log_d - o) as nat); + lemma_pow2_adds((o2 - o) as nat, (l - o2) as nat); + assert((o2 - o) as nat + (l - o2) as nat == (l - o) as nat); + lemma_pow2_pos((l - o2) as nat); + lemma_pow2_pos((log_d - l) as nat); + lemma_pow2_pos((log_d - o2) as nat); + lemma_pow2_pos((log_d2 - l - 1) as nat); + lemma_pow2_pos((log_d - log_d2) as nat); + lemma_pow2_adds((log_d2 - o2) as nat, (log_d - log_d2) as nat); + assert((log_d2 - o2) as nat + (log_d - log_d2) as nat == (log_d - o2) as nat); + let p_lo2 = pow2((l - o2) as nat) as int; + let p2h = pow2((log_d - l) as nat) as int; + assert(0 <= rb < pow2((o2 - o) as nat)) by (nonlinear_arith) + requires + rb == s2 - s * pow2((o2 - o) as nat), + s * pow2((o2 - o) as nat) <= s2 < (s + 1) * pow2((o2 - o) as nat), + ; + assert(base == (rb * p_lo2) * p2h) by (nonlinear_arith) + requires + base == rb * pow2((log_d - o2) as nat), + pow2((log_d - o2) as nat) == p2h * p_lo2, + ; + assert(2 * h == p2h); + lemma_mod_multiples_basic(rb * p_lo2, p2h); + lemma_div_multiples_vanish(rb * p_lo2, p2h); + assert(base / (2 * h as int) == rb * p_lo2) by { + lemma_mul_is_commutative(rb * p_lo2, p2h); + } + assert(base % (2 * h as int) == 0) by { + lemma_mul_is_commutative(rb * p_lo2, p2h); + } + assert(cnt as int == (2 * k) * p_lo2); + lemma_mod_multiples_basic(p_lo2, 2 * k as int); + assert(cnt % (2 * k) == 0) by { + lemma_mul_is_commutative(p_lo2, 2 * k as int); + } + assert(cnt / (2 * k) == p_lo2) by { + lemma_div_multiples_vanish(p_lo2, 2 * k as int); + lemma_mul_is_commutative(p_lo2, 2 * k as int); + } + assert((base + cnt * step) * m <= y.len()) by (nonlinear_arith) + requires + base == rb * pow2((log_d - o2) as nat), + cnt * step == pow2((log_d - o2) as nat), + rb + 1 <= pow2((o2 - o) as nat), + y.len() == m * pow2((log_d - o) as nat), + pow2((log_d - o) as nat) == pow2((o2 - o) as nat) * pow2((log_d - o2) as nat), + m > 0, + ; + let tw = sub_twiddles(tab, l, o, s); + let tw2 = sub_twiddles(tab, l, o2, s2); + assert forall|b: int| 0 <= b < cnt / (2 * k) implies #[trigger] tw2(b) == tw(base / (2 * h as int) + b) by { + assert(s * pow2((l - o) as nat) + rb * p_lo2 == s2 * p_lo2) by (nonlinear_arith) + requires + rb == s2 - s * pow2((o2 - o) as nat), + pow2((l - o) as nat) == pow2((o2 - o) as nat) * p_lo2, + ; + } + lemma_gather_layer(y, m, h, tw, tw2, false, base, r, step, cnt, k); + } +} + +// --------------------------------------------------------------------------------------------- +// Fused layers on row groups +// --------------------------------------------------------------------------------------------- +/// Rows of `m` words, as one buffer. +pub open spec fn rows_seq(rows: Seq>, m: nat) -> Seq { + Seq::new(rows.len() * m, |p: int| rows[p / (m as int)][p % (m as int)]) +} + +/// The rows `b` are one butterfly layer of half `h` on the rows `a`, lane by lane. +pub open spec fn layer_rows(a: Seq>, b: Seq>, m: nat, h: nat, tw: spec_fn(int) -> u64) -> bool { + &&& a.len() == b.len() + &&& forall|i: int| 0 <= i < a.len() ==> (#[trigger] a[i]).len() == m && b[i].len() == m + &&& forall|i: int, lane: int| + 0 <= i < a.len() && 0 <= lane < m ==> if i % (2 * h as int) < h { + (#[trigger] b[i][lane]).0 == bf_top(a[i][lane].0, a[i + h][lane].0, tw(i / (2 * h as int))) + } else { + b[i][lane].0 == bf_bot(a[i - h][lane].0, a[i][lane].0, tw(i / (2 * h as int))) + } +} + +/// Layers L, L+1 and L+2 on a block of `8e` rows: rows `4e`, `2e`, then `e` apart, with the seven +/// twiddles `t` breadth-first. +pub open spec fn layer3(x: Seq, m: nat, e: nat, t: Seq) -> Seq { + layer_map( + layer_map(layer_map(x, m, 4 * e, |b: int| t[0].0, false), m, 2 * e, |b: int| t[1 + b].0, false), + m, + e, + |b: int| t[3 + b].0, + false, + ) +} + +/// Layers L and L+1 on a block of `4e` rows: rows `2e` then `e` apart. +pub open spec fn layer2(x: Seq, m: nat, e: nat, t_outer: u64, t_inner_a: u64, t_inner_b: u64) -> Seq { + layer_map( + layer_map(x, m, 2 * e, |b: int| t_outer, false), + m, + e, + pick2(t_inner_a, t_inner_b), + false, + ) +} + +/// Twiddle `a` for block 0, `b` for block 1. +pub open spec fn pick2(a: u64, b: u64) -> spec_fn(int) -> u64 { + |x: int| + if x == 0 { + a + } else { + b + } +} + +/// `gp` holds exactly the permissions of the words `off .. off + m` of each of `n` slabs `stride` words +/// long, of the buffer at `base`. +pub open spec fn holds_group(gp: Map>, base: *mut F64, off: int, stride: int, n: nat, m: nat) -> bool { + &&& 0 <= off + &&& off + m <= stride + &&& base@.addr + n * stride * size_of::() <= usize::MAX + &&& n * stride <= usize::MAX + &&& forall|k: int| #[trigger] gp.dom().contains(k) <==> 0 <= k < n * stride && off <= k % stride < off + m + &&& forall|k: int| #[trigger] gp.dom().contains(k) ==> gp[k].ptr() == ptr_at(base, k) && gp[k].is_init() +} + +/// The words of a group, row by row, as one buffer. +pub open spec fn group_seq(gp: Map>, off: int, stride: int, n: nat, m: nat) -> Seq { + Seq::new(n * m, |q: int| gp[off + (q / (m as int)) * stride + q % (m as int)].value()) +} + +/// What `fused_rows` hands its closure: the permissions of row `off / m` of every slab, with the +/// block's values `x`. +pub open spec fn group_pre(gp: Map>, base: *mut F64, off: int, stride: int, n: nat, m: nat, x: Seq) -> bool { + &&& holds_group(gp, base, off, stride, n, m) + &&& off % (m as int) == 0 + &&& forall|k: int| #[trigger] gp.dom().contains(k) ==> gp[k].value() == x[k] +} + +/// What the closure returns: the same permissions, with the values `target`. +pub open spec fn group_post(res: Map>, gp: Map>, target: Seq) -> bool { + &&& res.dom() == gp.dom() + &&& forall|k: int| + #[trigger] gp.dom().contains(k) ==> res[k].ptr() == gp[k].ptr() && res[k].is_init() && res[k].value() == target[k] +} + +/// A layer on rows given lane by lane is the layer on the buffer of the rows. +pub proof fn lemma_rows_layer(a: Seq>, b: Seq>, m: nat, h: nat, tw: spec_fn(int) -> u64) + requires + m > 0, + h > 0, + a.len() % (2 * h) == 0, + layer_rows(a, b, m, h, tw), + ensures + layer_map(rows_seq(a, m), m, h, tw, false) == rows_seq(b, m), +{ + let mi = m as int; + let n = a.len() as int; + let x = rows_seq(a, m); + assert forall|p: int| 0 <= p < n * mi implies layer_map(x, m, h, tw, false)[p] == rows_seq(b, m)[p] by { + lemma_split_word(p, mi); + lemma_gather_rows(p, mi, n, h as int); + let i = p / mi; + assert(row_of(p, m) == i); + if i % (2 * h as int) < h { + assert(x[p + h * m] == a[i + h][p % mi]); + } else { + assert(x[p - h * m] == a[i - h][p % mi]); + } + } + assert(layer_map(x, m, h, tw, false) =~= rows_seq(b, m)); +} + +/// The three layers of a block, gathered on one row group, are the three layers of the group. +pub proof fn lemma_gather_layer3(x: Seq, m: nat, e: nat, t: Seq, r: int) + requires + m > 0, + e > 0, + 0 <= r < e, + x.len() == 8 * e * m, + ensures + gather(layer3(x, m, e, t), m, 0, r, e as int, 8) == layer3(gather(x, m, 0, r, e as int, 8), m, 1, t), +{ + let t0 = |b: int| t[0].0; + let t1 = |b: int| t[1 + b].0; + let t2 = |b: int| t[3 + b].0; + let x1 = layer_map(x, m, 4 * e, t0, false); + let x2 = layer_map(x1, m, 2 * e, t1, false); + let g = gather(x, m, 0, r, e as int, 8); + assert((0 + 8 * e) * m <= x.len()) by (nonlinear_arith) + requires + x.len() == 8 * e * m, + ; + assert(8nat % (2 * 4nat) == 0 && 8nat % (2 * 2nat) == 0 && 8nat % (2 * 1nat) == 0) by (compute); + lemma_gather_layer(x, m, 4 * e, t0, t0, false, 0, r, e as int, 8, 4); + lemma_gather_layer(x1, m, 2 * e, t1, t1, false, 0, r, e as int, 8, 2); + lemma_gather_layer(x2, m, e, t2, t2, false, 0, r, e as int, 8, 1); +} + +/// The two layers of a block, gathered on one row group, are the two layers of the group. +pub proof fn lemma_gather_layer2(x: Seq, m: nat, e: nat, t_outer: u64, t_inner_a: u64, t_inner_b: u64, r: int) + requires + m > 0, + e > 0, + 0 <= r < e, + x.len() == 4 * e * m, + ensures + gather(layer2(x, m, e, t_outer, t_inner_a, t_inner_b), m, 0, r, e as int, 4) == layer2( + gather(x, m, 0, r, e as int, 4), + m, + 1, + t_outer, + t_inner_a, + t_inner_b, + ), +{ + let t0 = |b: int| t_outer; + let t1 = pick2(t_inner_a, t_inner_b); + let x1 = layer_map(x, m, 2 * e, t0, false); + assert((0 + 4 * e) * m <= x.len()) by (nonlinear_arith) + requires + x.len() == 4 * e * m, + ; + assert(4nat % (2 * 2nat) == 0 && 4nat % (2 * 1nat) == 0) by (compute); + lemma_gather_layer(x, m, 2 * e, t0, t0, false, 0, r, e as int, 4, 2); + lemma_gather_layer(x1, m, e, t1, t1, false, 0, r, e as int, 4, 1); +} + +/// Word `off + i * stride + lane` of a group is word `q = i * m + lane` of its gather. +proof fn lemma_group_word(q: int, m: int, e: int, r: int) + requires + m > 0, + q >= 0, + ensures + gidx(m as nat, 0, r, e, q) == r * m + (q / m) * (e * m) + q % m, +{ + assert((0 + r + (q / m) * e) * m == r * m + (q / m) * (e * m)) by (nonlinear_arith); +} + +/// The key `k` of a group is `off + i * stride + lane`, word `i * m + lane` of the group's buffer. +proof fn lemma_group_key(k: int, off: int, stride: int, n: nat, m: nat) + requires + m > 0, + 0 <= off, + off + m <= stride, + 0 <= k < n * stride, + off <= k % stride < off + m, + ensures + ({ + let i = k / stride; + let lane = k % stride - off; + &&& 0 <= i < n + &&& 0 <= lane < m + &&& k == off + i * stride + lane + &&& 0 <= i * m + lane < n * m + &&& (i * m + lane) / (m as int) == i + &&& (i * m + lane) % (m as int) == lane + }), +{ + let i = k / stride; + let lane = k % stride - off; + lemma_split_word(k, stride); + assert(i < n) by (nonlinear_arith) + requires + k == i * stride + k % stride, + k % stride >= 0, + k < n * stride, + stride > 0, + ; + lemma_word(i, lane, m as int); + assert(i * m + lane < n * m) by (nonlinear_arith) + requires + i + 1 <= n, + 0 <= lane < m, + ; + assert(i * m + lane >= 0) by (nonlinear_arith) + requires + i >= 0, + lane >= 0, + ; +} + +/// A group's buffer is the gather of its rows of the block. +proof fn lemma_group_seq(gp: Map>, base: *mut F64, x: Seq, off: int, n: nat, m: nat, e: nat) + requires + m > 0, + e > 0, + off % (m as int) == 0, + x.len() == n * (e * m), + group_pre(gp, base, off, (e * m) as int, n, m, x), + ensures + group_seq(gp, off, (e * m) as int, n, m) == gather(x, m, 0, off / (m as int), e as int, n), +{ + let mi = m as int; + let stride = (e * m) as int; + let r = off / mi; + lemma_fundamental_div_mod(off, mi); + assert forall|q: int| 0 <= q < n * m implies group_seq(gp, off, stride, n, m)[q] == gather(x, m, 0, r, e as int, n)[q] by { + lemma_split_word(q, mi); + lemma_group_word(q, mi, e as int, r); + let i = q / mi; + let lane = q % mi; + let k = off + i * stride + lane; + assert(r * mi == off) by { + lemma_mul_is_commutative(r, mi); + } + assert(i < n) by (nonlinear_arith) + requires + q == i * mi + lane, + lane >= 0, + q < n * mi, + mi > 0, + ; + assert(k >= 0) by (nonlinear_arith) + requires + k == off + i * stride + lane, + off >= 0, + i >= 0, + stride > 0, + lane >= 0, + ; + assert(k < n * stride) by (nonlinear_arith) + requires + k == off + i * stride + lane, + i + 1 <= n, + off + lane < stride, + stride > 0, + ; + lemma_fundamental_div_mod_converse(k, stride, i, off + lane); + assert(gp.dom().contains(k)); + } + assert(group_seq(gp, off, stride, n, m) =~= gather(x, m, 0, r, e as int, n)); +} + +/// Back from a group's buffer to its keys: the group's words are the target's. +proof fn lemma_group_back(gp: Map>, res: Map>, base: *mut F64, target: Seq, off: int, n: nat, m: nat, e: nat) + requires + m > 0, + e > 0, + off % (m as int) == 0, + target.len() == n * (e * m), + holds_group(gp, base, off, (e * m) as int, n, m), + res.dom() == gp.dom(), + group_seq(res, off, (e * m) as int, n, m) == gather(target, m, 0, off / (m as int), e as int, n), + ensures + forall|k: int| #[trigger] gp.dom().contains(k) ==> res[k].value() == target[k], +{ + let mi = m as int; + let stride = (e * m) as int; + let r = off / mi; + lemma_fundamental_div_mod(off, mi); + assert forall|k: int| #[trigger] gp.dom().contains(k) implies res[k].value() == target[k] by { + lemma_group_key(k, off, stride, n, m); + let i = k / stride; + let lane = k % stride - off; + let q = i * mi + lane; + lemma_group_word(q, mi, e as int, r); + assert(r * mi == off) by { + lemma_mul_is_commutative(r, mi); + } + assert(group_seq(res, off, stride, n, m)[q] == res[k].value()); + assert(gather(target, m, 0, r, e as int, n)[q] == target[k]); + } +} + +/// Row `i` of a group: its words are in the group, with their permissions, and end inside the block. +proof fn lemma_row_in_group(g: Map>, base: *mut F64, off: int, stride: int, n: nat, m: nat, i: int) + requires + m > 0, + 0 <= i < n, + holds_group(g, base, off, stride, n, m), + ensures + Set::range(i * stride + off, i * stride + off + m) <= g.dom(), + i * stride + off + m <= n * stride, + i * stride >= 0, + base@.addr + (i * stride + off + m) * size_of::() <= usize::MAX, + base@.addr + (i * stride + off) * size_of::() <= usize::MAX, +{ + assert(i * stride + off + m <= n * stride) by (nonlinear_arith) + requires + i + 1 <= n, + off + m <= stride, + 0 <= off, + ; + assert(i * stride >= 0) by (nonlinear_arith) + requires + i >= 0, + stride >= 0, + ; + lemma_mul_le(i * stride + off + m, n * stride, size_of::() as int); + lemma_mul_le(i * stride + off, i * stride + off + m, size_of::() as int); + assert forall|k: int| Set::range(i * stride + off, i * stride + off + m).contains(k) implies g.dom().contains(k) by { + lemma_fundamental_div_mod_converse(k, stride, i, k - i * stride); + } +} + +/// A group's buffer, given row by row. +proof fn lemma_rows_group_seq(g: Map>, off: int, stride: int, n: nat, m: nat, rows: Seq>) + requires + m > 0, + rows.len() == n, + forall|i: int| 0 <= i < n ==> (#[trigger] rows[i]).len() == m, + forall|i: int, lane: int| 0 <= i < n && 0 <= lane < m ==> (#[trigger] rows[i][lane]) == g[i * stride + off + lane].value(), + ensures + group_seq(g, off, stride, n, m) == rows_seq(rows, m), +{ + let mi = m as int; + assert forall|q: int| 0 <= q < n * m implies group_seq(g, off, stride, n, m)[q] == rows_seq(rows, m)[q] by { + lemma_split_word(q, mi); + assert(q / mi < n) by (nonlinear_arith) + requires + q == (q / mi) * mi + q % mi, + q % mi >= 0, + q < n * mi, + mi > 0, + ; + assert(off + (q / mi) * stride + q % mi == (q / mi) * stride + off + q % mi); + } + assert(group_seq(g, off, stride, n, m) =~= rows_seq(rows, m)); +} + +/// The butterfly of rows `i` and `i + h` of `a`, lane by lane, gives rows `i` and `i + h` of `b`. +pub open spec fn pair_fact(a: Seq>, b: Seq>, m: nat, i: int, h: int, t: u64) -> bool { + forall|lane: int| + 0 <= lane < m ==> ((#[trigger] b[i][lane]).0, b[i + h][lane].0) == butterfly_spec(false, a[i][lane].0, a[i + h][lane].0, t) +} + +/// Rows of `m` lanes, `n` of them. +pub open spec fn rows_shape(a: Seq>, n: nat, m: nat) -> bool { + a.len() == n && forall|i: int| 0 <= i < n ==> (#[trigger] a[i]).len() == m +} + +proof fn lemma_radix8_layer_a(a: Seq>, b: Seq>, m: nat, tw: spec_fn(int) -> u64) + requires + rows_shape(a, 8, m), + rows_shape(b, 8, m), + pair_fact(a, b, m, 0, 4, tw(0)), + pair_fact(a, b, m, 1, 4, tw(0)), + pair_fact(a, b, m, 2, 4, tw(0)), + pair_fact(a, b, m, 3, 4, tw(0)), + ensures + layer_rows(a, b, m, 4, tw), +{ + assert forall|i: int, lane: int| 0 <= i < 8 && 0 <= lane < m implies if i % (2 * 4nat as int) < 4 { + (#[trigger] b[i][lane]).0 == bf_top(a[i][lane].0, a[i + 4][lane].0, tw(i / (2 * 4nat as int))) + } else { + b[i][lane].0 == bf_bot(a[i - 4][lane].0, a[i][lane].0, tw(i / (2 * 4nat as int))) + } by { + if i == 0 { + assert((b[0][lane].0, b[4][lane].0) == butterfly_spec(false, a[0][lane].0, a[4][lane].0, tw(0))); + } else if i == 1 { + assert((b[1][lane].0, b[5][lane].0) == butterfly_spec(false, a[1][lane].0, a[5][lane].0, tw(0))); + } else if i == 2 { + assert((b[2][lane].0, b[6][lane].0) == butterfly_spec(false, a[2][lane].0, a[6][lane].0, tw(0))); + } else if i == 3 { + assert((b[3][lane].0, b[7][lane].0) == butterfly_spec(false, a[3][lane].0, a[7][lane].0, tw(0))); + } else if i == 4 { + assert((b[0][lane].0, b[4][lane].0) == butterfly_spec(false, a[0][lane].0, a[4][lane].0, tw(0))); + } else if i == 5 { + assert((b[1][lane].0, b[5][lane].0) == butterfly_spec(false, a[1][lane].0, a[5][lane].0, tw(0))); + } else if i == 6 { + assert((b[2][lane].0, b[6][lane].0) == butterfly_spec(false, a[2][lane].0, a[6][lane].0, tw(0))); + } else { + assert((b[3][lane].0, b[7][lane].0) == butterfly_spec(false, a[3][lane].0, a[7][lane].0, tw(0))); + } + } +} + +proof fn lemma_radix8_layer_b(a: Seq>, b: Seq>, m: nat, tw: spec_fn(int) -> u64) + requires + rows_shape(a, 8, m), + rows_shape(b, 8, m), + pair_fact(a, b, m, 0, 2, tw(0)), + pair_fact(a, b, m, 1, 2, tw(0)), + pair_fact(a, b, m, 4, 2, tw(1)), + pair_fact(a, b, m, 5, 2, tw(1)), + ensures + layer_rows(a, b, m, 2, tw), +{ + assert forall|i: int, lane: int| 0 <= i < 8 && 0 <= lane < m implies if i % (2 * 2nat as int) < 2 { + (#[trigger] b[i][lane]).0 == bf_top(a[i][lane].0, a[i + 2][lane].0, tw(i / (2 * 2nat as int))) + } else { + b[i][lane].0 == bf_bot(a[i - 2][lane].0, a[i][lane].0, tw(i / (2 * 2nat as int))) + } by { + if i == 0 { + assert((b[0][lane].0, b[2][lane].0) == butterfly_spec(false, a[0][lane].0, a[2][lane].0, tw(0))); + } else if i == 1 { + assert((b[1][lane].0, b[3][lane].0) == butterfly_spec(false, a[1][lane].0, a[3][lane].0, tw(0))); + } else if i == 2 { + assert((b[0][lane].0, b[2][lane].0) == butterfly_spec(false, a[0][lane].0, a[2][lane].0, tw(0))); + } else if i == 3 { + assert((b[1][lane].0, b[3][lane].0) == butterfly_spec(false, a[1][lane].0, a[3][lane].0, tw(0))); + } else if i == 4 { + assert((b[4][lane].0, b[6][lane].0) == butterfly_spec(false, a[4][lane].0, a[6][lane].0, tw(1))); + } else if i == 5 { + assert((b[5][lane].0, b[7][lane].0) == butterfly_spec(false, a[5][lane].0, a[7][lane].0, tw(1))); + } else if i == 6 { + assert((b[4][lane].0, b[6][lane].0) == butterfly_spec(false, a[4][lane].0, a[6][lane].0, tw(1))); + } else { + assert((b[5][lane].0, b[7][lane].0) == butterfly_spec(false, a[5][lane].0, a[7][lane].0, tw(1))); + } + } +} + +proof fn lemma_radix8_layer_c(a: Seq>, b: Seq>, m: nat, tw: spec_fn(int) -> u64) + requires + rows_shape(a, 8, m), + rows_shape(b, 8, m), + pair_fact(a, b, m, 0, 1, tw(0)), + pair_fact(a, b, m, 2, 1, tw(1)), + pair_fact(a, b, m, 4, 1, tw(2)), + pair_fact(a, b, m, 6, 1, tw(3)), + ensures + layer_rows(a, b, m, 1, tw), +{ + assert forall|i: int, lane: int| 0 <= i < 8 && 0 <= lane < m implies if i % (2 * 1nat as int) < 1 { + (#[trigger] b[i][lane]).0 == bf_top(a[i][lane].0, a[i + 1][lane].0, tw(i / (2 * 1nat as int))) + } else { + b[i][lane].0 == bf_bot(a[i - 1][lane].0, a[i][lane].0, tw(i / (2 * 1nat as int))) + } by { + if i == 0 { + assert((b[0][lane].0, b[1][lane].0) == butterfly_spec(false, a[0][lane].0, a[1][lane].0, tw(0))); + } else if i == 1 { + assert((b[0][lane].0, b[1][lane].0) == butterfly_spec(false, a[0][lane].0, a[1][lane].0, tw(0))); + } else if i == 2 { + assert((b[2][lane].0, b[3][lane].0) == butterfly_spec(false, a[2][lane].0, a[3][lane].0, tw(1))); + } else if i == 3 { + assert((b[2][lane].0, b[3][lane].0) == butterfly_spec(false, a[2][lane].0, a[3][lane].0, tw(1))); + } else if i == 4 { + assert((b[4][lane].0, b[5][lane].0) == butterfly_spec(false, a[4][lane].0, a[5][lane].0, tw(2))); + } else if i == 5 { + assert((b[4][lane].0, b[5][lane].0) == butterfly_spec(false, a[4][lane].0, a[5][lane].0, tw(2))); + } else if i == 6 { + assert((b[6][lane].0, b[7][lane].0) == butterfly_spec(false, a[6][lane].0, a[7][lane].0, tw(3))); + } else { + assert((b[6][lane].0, b[7][lane].0) == butterfly_spec(false, a[6][lane].0, a[7][lane].0, tw(3))); + } + } +} + +proof fn lemma_radix4_layer_a(a: Seq>, b: Seq>, m: nat, tw: spec_fn(int) -> u64) + requires + rows_shape(a, 4, m), + rows_shape(b, 4, m), + pair_fact(a, b, m, 0, 2, tw(0)), + pair_fact(a, b, m, 1, 2, tw(0)), + ensures + layer_rows(a, b, m, 2, tw), +{ + assert forall|i: int, lane: int| 0 <= i < 4 && 0 <= lane < m implies if i % (2 * 2nat as int) < 2 { + (#[trigger] b[i][lane]).0 == bf_top(a[i][lane].0, a[i + 2][lane].0, tw(i / (2 * 2nat as int))) + } else { + b[i][lane].0 == bf_bot(a[i - 2][lane].0, a[i][lane].0, tw(i / (2 * 2nat as int))) + } by { + if i == 0 { + assert((b[0][lane].0, b[2][lane].0) == butterfly_spec(false, a[0][lane].0, a[2][lane].0, tw(0))); + } else if i == 1 { + assert((b[1][lane].0, b[3][lane].0) == butterfly_spec(false, a[1][lane].0, a[3][lane].0, tw(0))); + } else if i == 2 { + assert((b[0][lane].0, b[2][lane].0) == butterfly_spec(false, a[0][lane].0, a[2][lane].0, tw(0))); + } else { + assert((b[1][lane].0, b[3][lane].0) == butterfly_spec(false, a[1][lane].0, a[3][lane].0, tw(0))); + } + } +} + +proof fn lemma_radix4_layer_b(a: Seq>, b: Seq>, m: nat, tw: spec_fn(int) -> u64) + requires + rows_shape(a, 4, m), + rows_shape(b, 4, m), + pair_fact(a, b, m, 0, 1, tw(0)), + pair_fact(a, b, m, 2, 1, tw(1)), + ensures + layer_rows(a, b, m, 1, tw), +{ + assert forall|i: int, lane: int| 0 <= i < 4 && 0 <= lane < m implies if i % (2 * 1nat as int) < 1 { + (#[trigger] b[i][lane]).0 == bf_top(a[i][lane].0, a[i + 1][lane].0, tw(i / (2 * 1nat as int))) + } else { + b[i][lane].0 == bf_bot(a[i - 1][lane].0, a[i][lane].0, tw(i / (2 * 1nat as int))) + } by { + if i == 0 { + assert((b[0][lane].0, b[1][lane].0) == butterfly_spec(false, a[0][lane].0, a[1][lane].0, tw(0))); + } else if i == 1 { + assert((b[0][lane].0, b[1][lane].0) == butterfly_spec(false, a[0][lane].0, a[1][lane].0, tw(0))); + } else if i == 2 { + assert((b[2][lane].0, b[3][lane].0) == butterfly_spec(false, a[2][lane].0, a[3][lane].0, tw(1))); + } else { + assert((b[2][lane].0, b[3][lane].0) == butterfly_spec(false, a[2][lane].0, a[3][lane].0, tw(1))); + } + } +} + +/// The union of `rest` with the maps `fs[0..n]`, later ones taking precedence. +pub open spec fn union_all(rest: Map>, fs: Seq>>, n: nat) -> Map> + decreases n, +{ + if n == 0 { + rest + } else { + union_all(rest, fs, (n - 1) as nat).union_prefer_right(fs[n - 1]) + } +} + +/// A key of `fs[i]` that no later map holds keeps its permission in the union. +proof fn lemma_union_all(rest: Map>, fs: Seq>>, n: nat, i: int, k: int) + requires + 0 <= i < n, + fs[i].dom().contains(k), + forall|j: int| i < j < n ==> !(#[trigger] fs[j].dom().contains(k)), + ensures + union_all(rest, fs, n).dom().contains(k), + union_all(rest, fs, n)[k] == fs[i][k], + decreases n, +{ + if i < n - 1 { + lemma_union_all(rest, fs, (n - 1) as nat, i, k); + } +} + +/// The domain of the union. +proof fn lemma_union_all_dom(rest: Map>, fs: Seq>>, n: nat, k: int) + ensures + union_all(rest, fs, n).dom().contains(k) <==> rest.dom().contains(k) || exists|j: int| 0 <= j < n && #[trigger] fs[j].dom().contains(k), + decreases n, +{ + if n > 0 { + lemma_union_all_dom(rest, fs, (n - 1) as nat, k); + if exists|j: int| 0 <= j < n && #[trigger] fs[j].dom().contains(k) { + let j = choose|j: int| 0 <= j < n && #[trigger] fs[j].dom().contains(k); + if j < n - 1 { + assert(exists|j: int| 0 <= j < n - 1 && #[trigger] fs[j].dom().contains(k)); + } + } + } +} + +/// Take the permissions of the words `lo .. lo + m` out of a map. +proof fn take_row(tracked gp: &mut Map>, base: *mut F64, lo: int, m: nat) -> (tracked p: Map>) + requires + 0 <= lo, + base@.addr + (lo + m) * size_of::() <= usize::MAX, + Set::range(lo, lo + m) <= old(gp).dom(), + forall|k: int| #[trigger] old(gp).dom().contains(k) ==> old(gp)[k].ptr() == ptr_at(base, k) && old(gp)[k].is_init(), + ensures + owns(p, base, lo, m as int), + forall|j: int| 0 <= j < m ==> #[trigger] vals(p, lo, m as int)[j] == old(gp)[lo + j].value(), + *final(gp) == old(gp).remove_keys(Set::range(lo, lo + m)), +{ + let ghost g = *gp; + let tracked p = gp.tracked_remove_keys(Set::range(lo, lo + m)); + assert(p.dom() =~= Set::range(lo, lo + m)); + assert forall|k: int| lo <= k < lo + m implies (#[trigger] p[k]).ptr() == ptr_at(base, k) && p[k].is_init() by { + assert(g.dom().contains(k)); + } + p +} + +/// The rows of a group, taken out row by row, run, and put back: the group's permissions, with the +/// rows' new values. +proof fn lemma_group_rows_back( + g0: Map>, + rest: Map>, + fs: Seq>>, + base: *mut F64, + off: int, + stride: int, + n: nat, + m: nat, + s0: Seq>, + s3: Seq>, +) + requires + m > 0, + holds_group(g0, base, off, stride, n, m), + fs.len() == n, + rows_shape(s0, n, m), + rows_shape(s3, n, m), + forall|k: int| #[trigger] rest.dom().contains(k) ==> g0.dom().contains(k) && rest[k] == g0[k], + forall|i: int| 0 <= i < n ==> owns(#[trigger] fs[i], base, i * stride + off, m as int) && s3[i] == vals(fs[i], i * stride + off, m as int), + forall|i: int, lane: int| 0 <= i < n && 0 <= lane < m ==> (#[trigger] s0[i][lane]) == g0[i * stride + off + lane].value(), + ensures + union_all(rest, fs, n).dom() == g0.dom(), + forall|k: int| #[trigger] g0.dom().contains(k) ==> union_all(rest, fs, n)[k].ptr() == g0[k].ptr() && union_all(rest, fs, n)[k].is_init(), + group_seq(union_all(rest, fs, n), off, stride, n, m) == rows_seq(s3, m), + group_seq(g0, off, stride, n, m) == rows_seq(s0, m), +{ + let u = union_all(rest, fs, n); + // A word of row `i` of the group belongs to no other row. + assert forall|i: int, lane: int| 0 <= i < n && 0 <= lane < m implies #[trigger] u[i * stride + off + lane] == fs[i][i * stride + off + lane] + && u.dom().contains(i * stride + off + lane) by { + let k = i * stride + off + lane; + assert(fs[i].dom().contains(k)); + assert forall|j: int| i < j < n implies !(#[trigger] fs[j].dom().contains(k)) by { + assert(j * stride >= (i + 1) * stride) by (nonlinear_arith) + requires + j >= i + 1, + stride >= 0, + ; + assert((i + 1) * stride == i * stride + stride) by (nonlinear_arith); + } + lemma_union_all(rest, fs, n, i, k); + } + assert forall|k: int| #[trigger] g0.dom().contains(k) implies u.dom().contains(k) by { + lemma_group_key(k, off, stride, n, m); + let i = k / stride; + let lane = k % stride - off; + assert(u[i * stride + off + lane] == fs[i][i * stride + off + lane]); + assert(u.dom().contains(i * stride + off + lane)); + } + assert forall|k: int| #[trigger] u.dom().contains(k) implies g0.dom().contains(k) by { + lemma_union_all_dom(rest, fs, n, k); + if !rest.dom().contains(k) { + let j = choose|j: int| 0 <= j < n && #[trigger] fs[j].dom().contains(k); + lemma_row_in_group(g0, base, off, stride, n, m, j); + assert(owns(fs[j], base, j * stride + off, m as int)); + assert(Set::range(j * stride + off, j * stride + off + m).contains(k)); + } + } + assert(u.dom() =~= g0.dom()); + assert forall|k: int| #[trigger] g0.dom().contains(k) implies u[k].ptr() == g0[k].ptr() && u[k].is_init() by { + lemma_group_key(k, off, stride, n, m); + let i = k / stride; + let lane = k % stride - off; + assert(u[i * stride + off + lane] == fs[i][i * stride + off + lane]); + } + lemma_rows_group_seq(u, off, stride, n, m, s3); + lemma_rows_group_seq(g0, off, stride, n, m, s0); +} + +/// The twelve butterflies of one radix-8 row group, with its seven twiddles breadth-first. +/// +/// Rewritten: the eight rows are eight arguments (production's `rows: &mut [&mut [F64]; 8]` and +/// `let [r0, ..] = rows`); Verus does not support arrays of mutable references. The twelve calls are +/// production's. +fn radix8_butterflies( + r0: &mut [F64], + r1: &mut [F64], + r2: &mut [F64], + r3: &mut [F64], + r4: &mut [F64], + r5: &mut [F64], + r6: &mut [F64], + r7: &mut [F64], + t: &[F64; 7], +) + requires + rows_shape(seq![old(r0)@, old(r1)@, old(r2)@, old(r3)@, old(r4)@, old(r5)@, old(r6)@, old(r7)@], 8, old(r0)@.len()), + ensures + rows_shape(seq![final(r0)@, final(r1)@, final(r2)@, final(r3)@, final(r4)@, final(r5)@, final(r6)@, final(r7)@], 8, old(r0)@.len()), + rows_seq(seq![final(r0)@, final(r1)@, final(r2)@, final(r3)@, final(r4)@, final(r5)@, final(r6)@, final(r7)@], old(r0)@.len()) + == layer3( + rows_seq(seq![old(r0)@, old(r1)@, old(r2)@, old(r3)@, old(r4)@, old(r5)@, old(r6)@, old(r7)@], old(r0)@.len()), + old(r0)@.len(), + 1, + t@, + ), +{ + let ghost m = r0@.len(); + let ghost s0 = seq![r0@, r1@, r2@, r3@, r4@, r5@, r6@, r7@]; + // Layer L: rows 4 apart, one twiddle for the whole block. + butterfly_lanes(r0, r4, t[0]); + butterfly_lanes(r1, r5, t[0]); + butterfly_lanes(r2, r6, t[0]); + butterfly_lanes(r3, r7, t[0]); + let ghost s1 = seq![r0@, r1@, r2@, r3@, r4@, r5@, r6@, r7@]; + // Layer L+1: rows 2 apart, one twiddle per half. + butterfly_lanes(r0, r2, t[1]); + butterfly_lanes(r1, r3, t[1]); + butterfly_lanes(r4, r6, t[2]); + butterfly_lanes(r5, r7, t[2]); + let ghost s2 = seq![r0@, r1@, r2@, r3@, r4@, r5@, r6@, r7@]; + // Layer L+2: adjacent rows, one twiddle per quarter. + butterfly_lanes(r0, r1, t[3]); + butterfly_lanes(r2, r3, t[4]); + butterfly_lanes(r4, r5, t[5]); + butterfly_lanes(r6, r7, t[6]); + let ghost s3 = seq![r0@, r1@, r2@, r3@, r4@, r5@, r6@, r7@]; + proof { + let tw0 = |b: int| t@[0].0; + let tw1 = |b: int| t@[1 + b].0; + let tw2 = |b: int| t@[3 + b].0; + assert(rows_shape(s1, 8, m) && rows_shape(s2, 8, m) && rows_shape(s3, 8, m)); + lemma_radix8_layer_a(s0, s1, m, tw0); + lemma_radix8_layer_b(s1, s2, m, tw1); + lemma_radix8_layer_c(s2, s3, m, tw2); + if m > 0 { + assert(8nat % (2 * 4nat) == 0 && 8nat % (2 * 2nat) == 0 && 8nat % (2 * 1nat) == 0) by (compute); + lemma_rows_layer(s0, s1, m, 4, tw0); + lemma_rows_layer(s1, s2, m, 2, tw1); + lemma_rows_layer(s2, s3, m, 1, tw2); + } else { + assert(rows_seq(s3, m) =~= layer3(rows_seq(s0, m), m, 1, t@)); + } + } +} + +/// The rows of group `off` of a block of eight slabs, built as `fused_rows` builds them, through the +/// radix-8 butterflies. +/// +/// This is production's `fused_rows` loop body for `N = 8` (`std::array::from_fn` of +/// `from_raw_parts_mut(base.add(i * stride + off), num_ntts)`, then `do_one(&mut rows)`), written out per +/// row: Verus has no `from_fn` and no arrays of mutable references. +fn radix8_group( + base: *mut F64, + stride: usize, + off: usize, + num_ntts: usize, + gp: Tracked>>, + t: &[F64; 7], +) -> (res: Tracked>>) + requires + num_ntts > 0, + holds_group(gp@, base, off as int, stride as int, 8, num_ntts as nat), + ensures + res@.dom() == gp@.dom(), + forall|k: int| #[trigger] gp@.dom().contains(k) ==> res@[k].ptr() == gp@[k].ptr() && res@[k].is_init(), + group_seq(res@, off as int, stride as int, 8, num_ntts as nat) == layer3( + group_seq(gp@, off as int, stride as int, 8, num_ntts as nat), + num_ntts as nat, + 1, + t@, + ), +{ + let ghost g0 = gp@; + let ghost m = num_ntts as nat; + let tracked mut gp = gp.get(); + proof { + lemma_row_in_group(g0, base, off as int, stride as int, 8, m, 0); + lemma_row_in_group(g0, base, off as int, stride as int, 8, m, 1); + lemma_row_in_group(g0, base, off as int, stride as int, 8, m, 2); + lemma_row_in_group(g0, base, off as int, stride as int, 8, m, 3); + lemma_row_in_group(g0, base, off as int, stride as int, 8, m, 4); + lemma_row_in_group(g0, base, off as int, stride as int, 8, m, 5); + lemma_row_in_group(g0, base, off as int, stride as int, 8, m, 6); + lemma_row_in_group(g0, base, off as int, stride as int, 8, m, 7); + } + let tracked mut p0 = take_row(&mut gp, base, (0 * stride + off) as int, m); + let tracked mut p1 = take_row(&mut gp, base, (1 * stride + off) as int, m); + let tracked mut p2 = take_row(&mut gp, base, (2 * stride + off) as int, m); + let tracked mut p3 = take_row(&mut gp, base, (3 * stride + off) as int, m); + let tracked mut p4 = take_row(&mut gp, base, (4 * stride + off) as int, m); + let tracked mut p5 = take_row(&mut gp, base, (5 * stride + off) as int, m); + let tracked mut p6 = take_row(&mut gp, base, (6 * stride + off) as int, m); + let tracked mut p7 = take_row(&mut gp, base, (7 * stride + off) as int, m); + // SAFETY: + // - The groups are disjoint, as argued above. + // - The row ends inside its slab, since the row index is below the slab height. + let r0 = unsafe { from_raw_parts_mut(base.add(0 * stride + off), num_ntts, Ghost(base), Ghost((0 * stride + off) as int), Tracked(&mut p0)) }; + let r1 = unsafe { from_raw_parts_mut(base.add(1 * stride + off), num_ntts, Ghost(base), Ghost((1 * stride + off) as int), Tracked(&mut p1)) }; + let r2 = unsafe { from_raw_parts_mut(base.add(2 * stride + off), num_ntts, Ghost(base), Ghost((2 * stride + off) as int), Tracked(&mut p2)) }; + let r3 = unsafe { from_raw_parts_mut(base.add(3 * stride + off), num_ntts, Ghost(base), Ghost((3 * stride + off) as int), Tracked(&mut p3)) }; + let r4 = unsafe { from_raw_parts_mut(base.add(4 * stride + off), num_ntts, Ghost(base), Ghost((4 * stride + off) as int), Tracked(&mut p4)) }; + let r5 = unsafe { from_raw_parts_mut(base.add(5 * stride + off), num_ntts, Ghost(base), Ghost((5 * stride + off) as int), Tracked(&mut p5)) }; + let r6 = unsafe { from_raw_parts_mut(base.add(6 * stride + off), num_ntts, Ghost(base), Ghost((6 * stride + off) as int), Tracked(&mut p6)) }; + let r7 = unsafe { from_raw_parts_mut(base.add(7 * stride + off), num_ntts, Ghost(base), Ghost((7 * stride + off) as int), Tracked(&mut p7)) }; + let ghost s0 = seq![r0@, r1@, r2@, r3@, r4@, r5@, r6@, r7@]; + radix8_butterflies(r0, r1, r2, r3, r4, r5, r6, r7, t); + let ghost s3 = seq![r0@, r1@, r2@, r3@, r4@, r5@, r6@, r7@]; + proof { + let fs = seq![p0, p1, p2, p3, p4, p5, p6, p7]; + let rest = gp; + gp.tracked_union_prefer_right(p0); + gp.tracked_union_prefer_right(p1); + gp.tracked_union_prefer_right(p2); + gp.tracked_union_prefer_right(p3); + gp.tracked_union_prefer_right(p4); + gp.tracked_union_prefer_right(p5); + gp.tracked_union_prefer_right(p6); + gp.tracked_union_prefer_right(p7); + reveal_with_fuel(union_all, 9); + assert(gp == union_all(rest, fs, 8)); + assert forall|i: int| 0 <= i < 8 implies owns(#[trigger] fs[i], base, i * stride + off, m as int) && s3[i] == vals( + fs[i], + i * stride + off, + m as int, + ) by { + if i == 0 {} else if i == 1 {} else if i == 2 {} else if i == 3 {} else if i == 4 {} else if i == 5 {} else if i == 6 {} else {} + } + assert forall|i: int, lane: int| 0 <= i < 8 && 0 <= lane < m implies (#[trigger] s0[i][lane]) == g0[i * stride + off + lane].value() by { + if i == 0 {} else if i == 1 {} else if i == 2 {} else if i == 3 {} else if i == 4 {} else if i == 5 {} else if i == 6 {} else {} + } + lemma_group_rows_back(g0, rest, fs, base, off as int, stride as int, 8, m, s0, s3); + } + Tracked(gp) +} + +/// The body of the closure that `butterfly_interleaved_fused_2layer` hands `fused_rows`: layers L and +/// L+1 on one group of four rows. +/// +/// Rewritten: the four rows are four arguments (production's `let [row_a, row_b, row_c, row_d] = rows`). +fn radix4_rows(row_a: &mut [F64], row_b: &mut [F64], row_c: &mut [F64], row_d: &mut [F64], t_outer: F64, t_inner_a: F64, t_inner_b: F64) + requires + rows_shape(seq![old(row_a)@, old(row_b)@, old(row_c)@, old(row_d)@], 4, old(row_a)@.len()), + ensures + rows_shape(seq![final(row_a)@, final(row_b)@, final(row_c)@, final(row_d)@], 4, old(row_a)@.len()), + rows_seq(seq![final(row_a)@, final(row_b)@, final(row_c)@, final(row_d)@], old(row_a)@.len()) == layer2( + rows_seq(seq![old(row_a)@, old(row_b)@, old(row_c)@, old(row_d)@], old(row_a)@.len()), + old(row_a)@.len(), + 1, + t_outer.0, + t_inner_a.0, + t_inner_b.0, + ), +{ + let ghost m = row_a@.len(); + let ghost s0 = seq![row_a@, row_b@, row_c@, row_d@]; + // Layer L: rows 2 apart, one twiddle for the block. + butterfly_lanes(row_a, row_c, t_outer); + butterfly_lanes(row_b, row_d, t_outer); + let ghost s1 = seq![row_a@, row_b@, row_c@, row_d@]; + // Layer L+1: adjacent rows, one twiddle per half. + butterfly_lanes(row_a, row_b, t_inner_a); + butterfly_lanes(row_c, row_d, t_inner_b); + let ghost s2 = seq![row_a@, row_b@, row_c@, row_d@]; + proof { + let tw0 = |b: int| t_outer.0; + let tw1 = pick2(t_inner_a.0, t_inner_b.0); + assert(rows_shape(s1, 4, m) && rows_shape(s2, 4, m)); + lemma_radix4_layer_a(s0, s1, m, tw0); + lemma_radix4_layer_b(s1, s2, m, tw1); + if m > 0 { + assert(4nat % (2 * 2nat) == 0 && 4nat % (2 * 1nat) == 0) by (compute); + lemma_rows_layer(s0, s1, m, 2, tw0); + lemma_rows_layer(s1, s2, m, 1, tw1); + } else { + assert(rows_seq(s2, m) =~= layer2(rows_seq(s0, m), m, 1, t_outer.0, t_inner_a.0, t_inner_b.0)); + } + } +} + +/// The rows of group `off` of a block of four slabs, built as `fused_rows` builds them, through the +/// radix-4 butterflies: production's `fused_rows` loop body for `N = 4`, written out per row as +/// [`radix8_group`] is. +fn radix4_group( + base: *mut F64, + stride: usize, + off: usize, + num_ntts: usize, + gp: Tracked>>, + t_outer: F64, + t_inner_a: F64, + t_inner_b: F64, +) -> (res: Tracked>>) + requires + num_ntts > 0, + holds_group(gp@, base, off as int, stride as int, 4, num_ntts as nat), + ensures + res@.dom() == gp@.dom(), + forall|k: int| #[trigger] gp@.dom().contains(k) ==> res@[k].ptr() == gp@[k].ptr() && res@[k].is_init(), + group_seq(res@, off as int, stride as int, 4, num_ntts as nat) == layer2( + group_seq(gp@, off as int, stride as int, 4, num_ntts as nat), + num_ntts as nat, + 1, + t_outer.0, + t_inner_a.0, + t_inner_b.0, + ), +{ + let ghost g0 = gp@; + let ghost m = num_ntts as nat; + let tracked mut gp = gp.get(); + proof { + lemma_row_in_group(g0, base, off as int, stride as int, 4, m, 0); + lemma_row_in_group(g0, base, off as int, stride as int, 4, m, 1); + lemma_row_in_group(g0, base, off as int, stride as int, 4, m, 2); + lemma_row_in_group(g0, base, off as int, stride as int, 4, m, 3); + } + let tracked mut p0 = take_row(&mut gp, base, (0 * stride + off) as int, m); + let tracked mut p1 = take_row(&mut gp, base, (1 * stride + off) as int, m); + let tracked mut p2 = take_row(&mut gp, base, (2 * stride + off) as int, m); + let tracked mut p3 = take_row(&mut gp, base, (3 * stride + off) as int, m); + // SAFETY: + // - The groups are disjoint, as argued above. + // - The row ends inside its slab, since the row index is below the slab height. + let row_a = unsafe { from_raw_parts_mut(base.add(0 * stride + off), num_ntts, Ghost(base), Ghost((0 * stride + off) as int), Tracked(&mut p0)) }; + let row_b = unsafe { from_raw_parts_mut(base.add(1 * stride + off), num_ntts, Ghost(base), Ghost((1 * stride + off) as int), Tracked(&mut p1)) }; + let row_c = unsafe { from_raw_parts_mut(base.add(2 * stride + off), num_ntts, Ghost(base), Ghost((2 * stride + off) as int), Tracked(&mut p2)) }; + let row_d = unsafe { from_raw_parts_mut(base.add(3 * stride + off), num_ntts, Ghost(base), Ghost((3 * stride + off) as int), Tracked(&mut p3)) }; + let ghost s0 = seq![row_a@, row_b@, row_c@, row_d@]; + radix4_rows(row_a, row_b, row_c, row_d, t_outer, t_inner_a, t_inner_b); + let ghost s2 = seq![row_a@, row_b@, row_c@, row_d@]; + proof { + let fs = seq![p0, p1, p2, p3]; + let rest = gp; + gp.tracked_union_prefer_right(p0); + gp.tracked_union_prefer_right(p1); + gp.tracked_union_prefer_right(p2); + gp.tracked_union_prefer_right(p3); + reveal_with_fuel(union_all, 5); + assert(gp == union_all(rest, fs, 4)); + assert forall|i: int| 0 <= i < 4 implies owns(#[trigger] fs[i], base, i * stride + off, m as int) && s2[i] == vals( + fs[i], + i * stride + off, + m as int, + ) by { + if i == 0 {} else if i == 1 {} else if i == 2 {} else {} + } + assert forall|i: int, lane: int| 0 <= i < 4 && 0 <= lane < m implies (#[trigger] s0[i][lane]) == g0[i * stride + off + lane].value() by { + if i == 0 {} else if i == 1 {} else if i == 2 {} else {} + } + lemma_group_rows_back(g0, rest, fs, base, off as int, stride as int, 4, m, s0, s2); + } + Tracked(gp) +} + +/// Group `r` of a block of slabs `stride` words long takes row `r` of every slab: the words whose offset in +/// their slab, divided by the row width `m`, is `r`. +pub open spec fn group_owner(stride: int, m: int) -> spec_fn(int) -> int { + |k: int| (k % stride) / m +} + +/// The permissions of group `r` of a block: a group as [`holds_group`] describes it. +proof fn lemma_group_of_block(perms: Map>, base: *mut F64, x: Seq, n: nat, stride_rows: nat, m: nat, r: int) + requires + n > 0, + m > 0, + 0 <= r < stride_rows, + owns(perms, base, 0, (n * (stride_rows * m)) as int), + x.len() == n * (stride_rows * m), + n * (stride_rows * m) <= usize::MAX, + forall|k: int| + 0 <= k < x.len() && (group_owner((stride_rows * m) as int, m as int))(k) >= r ==> #[trigger] perms[k].value() == x[k], + ensures + ({ + let gp = perms.restrict(keys_of(perms, group_owner((stride_rows * m) as int, m as int), r, r + 1)); + group_pre(gp, base, r * m, (stride_rows * m) as int, n, m, x) + }), +{ + let stride = (stride_rows * m) as int; + let owner = group_owner(stride, m as int); + let gp = perms.restrict(keys_of(perms, owner, r, r + 1)); + assert(stride > 0) by (nonlinear_arith) + requires + stride == stride_rows * m, + r < stride_rows, + r >= 0, + m > 0, + ; + assert(r * m + m <= stride) by (nonlinear_arith) + requires + stride == stride_rows * m, + r + 1 <= stride_rows, + ; + assert(r * m >= 0) by (nonlinear_arith) + requires + r >= 0, + ; + assert forall|k: int| #[trigger] gp.dom().contains(k) <==> 0 <= k < n * stride && r * m <= k % stride < r * m + m by { + if 0 <= k { + lemma_mod_bound(k, stride); + lemma_chunk_keys(k % stride, r, m as int); + } + } + assert((r * m) % (m as int) == 0) by { + lemma_mod_multiples_basic(r, m as int); + } + assert forall|k: int| #[trigger] gp.dom().contains(k) implies gp[k].value() == x[k] by { + lemma_mod_bound(k, stride); + lemma_chunk_keys(k % stride, r, m as int); + } +} + +/// Visit every row group of a fused multi-layer block. +/// +/// ```text +/// the block is N slabs of equal height +/// +/// slab 0: row 0 row 1 ... row r ... +/// slab 1: row 0 row 1 ... row r ... +/// ... +/// slab N-1: row 0 row 1 ... row r ... +/// +/// group r = row r of every slab +/// ``` +/// +/// # Why the rows are disjoint +/// +/// - Distinct groups take distinct rows inside each slab. +/// - The slabs do not overlap. +/// - So the groups are pairwise disjoint, and one base pointer can stand in for N nested splits. +/// +/// Verified: group `r` receives exactly the permissions of its rows, taken from the block's; `do_one` +/// returns them with the values `target`, so the block ends as `target`. Rewritten: production builds the +/// `N` rows here with `std::array::from_fn` and passes `&mut rows`; Verus supports neither, so `do_one` +/// receives the base pointer, the stride, the row offset and the group's permissions, and builds the rows +/// itself ([`radix8_group`], [`radix4_group`]). The `debug_assert_eq!` is a proven `assert`, and the target +/// is a ghost argument. +fn fused_rows(block: &mut [F64], stride_rows: usize, num_ntts: usize, do_one: F, Ghost(target): Ghost>) +where + F: Fn(*mut F64, usize, usize, Tracked>>) -> Tracked>>, + requires + N > 0, + num_ntts > 0, + stride_rows > 0, + old(block)@.len() == N * (stride_rows * num_ntts), + old(block)@.len() <= usize::MAX, + target.len() == old(block)@.len(), + forall|base: *mut F64, off: usize, gp: Map>| + group_pre(gp, base, off as int, (stride_rows * num_ntts) as int, N as nat, num_ntts as nat, old(block)@) + ==> #[trigger] do_one.requires((base, (stride_rows * num_ntts) as usize, off, Tracked(gp))), + forall|base: *mut F64, off: usize, gp: Map>, res: Tracked>>| + group_pre(gp, base, off as int, (stride_rows * num_ntts) as int, N as nat, num_ntts as nat, old(block)@) + && #[trigger] do_one.ensures((base, (stride_rows * num_ntts) as usize, off, Tracked(gp)), res) ==> group_post( + res@, + gp, + target, + ), + ensures + final(block)@ == target, +{ + let ghost x0 = block@; + let ghost len = x0.len() as int; + // Words from one slab to the next. + proof { + assert(stride_rows * num_ntts <= N * (stride_rows * num_ntts)) by (nonlinear_arith) + requires + N > 0, + ; + } + let stride = stride_rows * num_ntts; + assert(block.len() == N * stride); + // Rewritten from `block.as_mut_ptr()`: the pointer comes with the block's permissions. + let (base, Tracked(perms)) = slice_as_mut_ptr(block); + let ghost owner = group_owner(stride as int, num_ntts as int); + for r in 0..stride_rows + invariant + N > 0, + num_ntts > 0, + stride == stride_rows * num_ntts, + len == N * stride, + x0.len() == len, + len <= usize::MAX, + target.len() == len, + owner == group_owner(stride as int, num_ntts as int), + owns(*perms, base, 0, len), + forall|k: int| 0 <= k < len && owner(k) < r ==> #[trigger] perms[k].value() == target[k], + forall|k: int| 0 <= k < len && owner(k) >= r ==> #[trigger] perms[k].value() == x0[k], + forall|base: *mut F64, off: usize, gp: Map>| + group_pre(gp, base, off as int, stride as int, N as nat, num_ntts as nat, x0) + ==> #[trigger] do_one.requires((base, stride, off, Tracked(gp))), + forall|base: *mut F64, off: usize, gp: Map>, res: Tracked>>| + group_pre(gp, base, off as int, stride as int, N as nat, num_ntts as nat, x0) + && #[trigger] do_one.ensures((base, stride, off, Tracked(gp)), res) ==> group_post(res@, gp, target), + { + proof { + assert(r * num_ntts < stride) by (nonlinear_arith) + requires + r < stride_rows, + stride == stride_rows * num_ntts, + num_ntts > 0, + ; + } + // Row `r` of each slab starts this many words into it. + let off = r * num_ntts; + let ghost cur = *perms; + proof { + lemma_group_of_block(cur, base, x0, N as nat, stride_rows as nat, num_ntts as nat, r as int); + } + let tracked gp = perms.tracked_remove_keys(keys_of(cur, owner, r as int, r + 1)); + let ghost g = gp; + let Tracked(res) = do_one(base, stride, off, Tracked(gp)); + proof { + perms.tracked_union_prefer_right(res); + assert(perms.dom() =~= cur.dom()); + assert forall|k: int| 0 <= k < len implies (#[trigger] perms[k]).ptr() == ptr_at(base, k) && perms[k].is_init() && (owner(k) < r + 1 ==> perms[k].value() == target[k]) && (owner(k) >= r + 1 ==> perms[k].value() == x0[k]) by { + if owner(k) == r { + assert(g.dom().contains(k)); + } else { + assert(!g.dom().contains(k)); + } + } + } + } + proof { + assert forall|k: int| 0 <= k < len implies perms[k].value() == target[k] by { + lemma_mod_bound(k, stride as int); + lemma_div_is_ordered(k % (stride as int), stride as int, num_ntts as int); + lemma_div_multiples_vanish(stride_rows as int, num_ntts as int); + lemma_mul_is_commutative(stride_rows as int, num_ntts as int); + if owner(k) >= stride_rows { + lemma_fundamental_div_mod(k % (stride as int), num_ntts as int); + lemma_mod_bound(k % (stride as int), num_ntts as int); + assert(false) by (nonlinear_arith) + requires + k % (stride as int) == num_ntts * owner(k) + (k % (stride as int)) % (num_ntts as int), + (k % (stride as int)) % (num_ntts as int) >= 0, + owner(k) >= stride_rows, + k % (stride as int) < stride, + stride == stride_rows * num_ntts, + num_ntts > 0, + ; + } + } + assert(vals(*perms, 0, len) =~= target); + } +} + +/// Layers L, L+1 and L+2 fused into one sweep over a layer-L block. +/// +/// ```text +/// a group is 8 rows, e = (block rows) / 8 apart: r, r + e, ..., r + 7e +/// +/// layer L pairs rows 4e apart +/// layer L+1 pairs rows 2e apart +/// layer L+2 pairs rows e apart +/// ``` +/// +/// - The eight rows stay in L1 across all twelve butterflies. +/// - The seven twiddles are breadth-first: one for layer L, two for L+1, four for L+2. +/// +/// The closure builds its group's rows from the permissions `fused_rows` hands it (see [`fused_rows`]). +fn butterfly_interleaved_fused_3layer(block: &mut [F64], t: &[F64; 7], eighth: usize, num_ntts: usize) + requires + eighth > 0, + num_ntts > 0, + old(block)@.len() == 8 * (eighth * num_ntts), + old(block)@.len() <= usize::MAX, + ensures + final(block)@ == layer3(old(block)@, num_ntts as nat, eighth as nat, t@), +{ + let ghost x0 = block@; + let ghost target = layer3(x0, num_ntts as nat, eighth as nat, t@); + proof { + lemma_layer3_len(x0, num_ntts as nat, eighth as nat, t@); + } + fused_rows::<8, _>( + block, + eighth, + num_ntts, + |base: *mut F64, stride: usize, off: usize, gp: Tracked>>| -> (res: Tracked>>) + requires + eighth > 0, + num_ntts > 0, + stride == eighth * num_ntts, + x0.len() == 8 * stride, + group_pre(gp@, base, off as int, stride as int, 8, num_ntts as nat, x0), + ensures + group_post(res@, gp@, layer3(x0, num_ntts as nat, eighth as nat, t@)), + { + let res = radix8_group(base, stride, off, num_ntts, gp, t); + proof { + let m = num_ntts as nat; + let e = eighth as nat; + let r = off / num_ntts; + lemma_group_seq(gp@, base, x0, off as int, 8, m, e); + lemma_fundamental_div_mod(off as int, num_ntts as int); + assert(r < e) by (nonlinear_arith) + requires + off as int == num_ntts * r + (off as int) % (num_ntts as int), + (off as int) % (num_ntts as int) == 0, + off + num_ntts <= stride, + stride == e * num_ntts, + num_ntts > 0, + ; + assert(x0.len() == 8 * e * m) by (nonlinear_arith) + requires + x0.len() == 8 * stride, + stride == e * m, + ; + lemma_gather_layer3(x0, m, e, t@, r as int); + lemma_layer3_len(x0, m, e, t@); + lemma_group_back(gp@, res@, base, layer3(x0, m, e, t@), off as int, 8, m, e); + } + res + }, + Ghost(target), + ); +} + +proof fn lemma_layer3_len(x: Seq, m: nat, e: nat, t: Seq) + ensures + layer3(x, m, e, t).len() == x.len(), +{ +} + +proof fn lemma_layer2_len(x: Seq, m: nat, e: nat, a: u64, b: u64, c: u64) + ensures + layer2(x, m, e, a, b, c).len() == x.len(), +{ +} + +/// Layers L and L+1 fused into one sweep over a layer-L block, four rows at a time. +/// +/// The closure builds its group's rows from the permissions `fused_rows` hands it (see [`fused_rows`]); +/// its four butterflies are [`radix4_rows`]. +fn butterfly_interleaved_fused_2layer( + block: &mut [F64], + t_outer: F64, + t_inner_a: F64, + t_inner_b: F64, + quarter: usize, + num_ntts: usize, +) + requires + quarter > 0, + num_ntts > 0, + old(block)@.len() == 4 * (quarter * num_ntts), + old(block)@.len() <= usize::MAX, + ensures + final(block)@ == layer2(old(block)@, num_ntts as nat, quarter as nat, t_outer.0, t_inner_a.0, t_inner_b.0), +{ + let ghost x0 = block@; + let ghost target = layer2(x0, num_ntts as nat, quarter as nat, t_outer.0, t_inner_a.0, t_inner_b.0); + fused_rows::<4, _>( + block, + quarter, + num_ntts, + |base: *mut F64, stride: usize, off: usize, gp: Tracked>>| -> (res: Tracked>>) + requires + quarter > 0, + num_ntts > 0, + stride == quarter * num_ntts, + x0.len() == 4 * stride, + group_pre(gp@, base, off as int, stride as int, 4, num_ntts as nat, x0), + ensures + group_post(res@, gp@, layer2(x0, num_ntts as nat, quarter as nat, t_outer.0, t_inner_a.0, t_inner_b.0)), + { + let res = radix4_group(base, stride, off, num_ntts, gp, t_outer, t_inner_a, t_inner_b); + proof { + let m = num_ntts as nat; + let e = quarter as nat; + let r = off / num_ntts; + lemma_group_seq(gp@, base, x0, off as int, 4, m, e); + lemma_fundamental_div_mod(off as int, num_ntts as int); + assert(r < e) by (nonlinear_arith) + requires + off as int == num_ntts * r + (off as int) % (num_ntts as int), + (off as int) % (num_ntts as int) == 0, + off + num_ntts <= stride, + stride == e * num_ntts, + num_ntts > 0, + ; + assert(x0.len() == 4 * e * m) by (nonlinear_arith) + requires + x0.len() == 4 * stride, + stride == e * m, + ; + lemma_gather_layer2(x0, m, e, t_outer.0, t_inner_a.0, t_inner_b.0, r as int); + lemma_layer2_len(x0, m, e, t_outer.0, t_inner_a.0, t_inner_b.0); + lemma_group_back(gp@, res@, base, layer2(x0, m, e, t_outer.0, t_inner_a.0, t_inner_b.0), off as int, 4, m, e); + } + res + }, + Ghost(target), + ); +} + +/// One layer on one block of `2 * block_size_half` rows. +/// +/// Rewritten: the `split_at_mut` halves are bound to their lengths once more for the proof (no exec change). +#[inline] +fn butterfly_interleaved_block(block: &mut [F64], twiddle: F64, block_size_half: usize, num_ntts: usize) + requires + num_ntts > 0, + block_size_half > 0, + old(block)@.len() == 2 * (block_size_half * num_ntts), + old(block)@.len() <= usize::MAX, + ensures + final(block)@ == layer_map(old(block)@, num_ntts as nat, block_size_half as nat, |b: int| twiddle.0, false), +{ + let ghost x0 = block@; + let ghost m = num_ntts as int; + let ghost h = block_size_half as int; + proof { + assert(block_size_half * num_ntts <= 2 * (block_size_half * num_ntts)) by (nonlinear_arith); + } + let half_offset = block_size_half * num_ntts; + let (top, bot) = block.split_at_mut(half_offset); + let ghost (top0, bot0) = (top@, bot@); + assert(0 * m == 0); + for r in 0..block_size_half + invariant + m == num_ntts, + h == block_size_half, + m > 0, + half_offset == h * m, + top0.len() == half_offset, + bot0.len() == half_offset, + top@.len() == half_offset, + bot@.len() == half_offset, + forall|p: int| + 0 <= p < r * m ==> (#[trigger] top@[p]).0 == bf_top(top0[p].0, bot0[p].0, twiddle.0) && bot@[p].0 == bf_bot( + top0[p].0, + bot0[p].0, + twiddle.0, + ), + forall|p: int| r * m <= p < half_offset ==> #[trigger] top@[p] == top0[p] && bot@[p] == bot0[p], + { + proof { + assert((r + 1) * m <= h * m) by (nonlinear_arith) + requires + r + 1 <= h, + m > 0, + ; + assert((r + 1) * m == r * m + m) by (nonlinear_arith); + } + let off = r * num_ntts; + let ghost (t1, b1) = (top@, bot@); + butterfly_lanes(&mut top[off..off + num_ntts], &mut bot[off..off + num_ntts], twiddle); + proof { + assert forall|p: int| 0 <= p < (r + 1) * m implies (#[trigger] top@[p]).0 == bf_top(top0[p].0, bot0[p].0, twiddle.0) + && bot@[p].0 == bf_bot(top0[p].0, bot0[p].0, twiddle.0) by { + if p >= r * m { + let j = p - off; + assert(top@[p] == top@.subrange(off as int, off + m)[j]); + assert(bot@[p] == bot@.subrange(off as int, off + m)[j]); + assert(t1.subrange(off as int, off + m)[j] == t1[p]); + assert(b1.subrange(off as int, off + m)[j] == b1[p]); + assert((top@.subrange(off as int, off + m)[j].0, bot@.subrange(off as int, off + m)[j].0) == butterfly_spec( + false, + t1.subrange(off as int, off + m)[j].0, + b1.subrange(off as int, off + m)[j].0, + twiddle.0, + )); + } else { + assert(top@[p] == top@.subrange(0, off as int)[p]); + assert(bot@[p] == bot@.subrange(0, off as int)[p]); + } + } + assert forall|p: int| (r + 1) * m <= p < half_offset implies #[trigger] top@[p] == top0[p] && bot@[p] == bot0[p] by { + assert(top@[p] == top@.subrange(off + m, half_offset as int)[p - off - m]); + assert(bot@[p] == bot@.subrange(off + m, half_offset as int)[p - off - m]); + } + } + } + proof { + let y = layer_map(x0, num_ntts as nat, block_size_half as nat, |b: int| twiddle.0, false); + assert(x0 == top0 + bot0); + let fin = top@ + bot@; + assert forall|p: int| 0 <= p < x0.len() implies fin[p] == y[p] by { + lemma_split_word(p, m); + let row = p / m; + assert(row < 2 * h) by (nonlinear_arith) + requires + p == row * m + p % m, + p % m >= 0, + p < 2 * (h * m), + m > 0, + ; + lemma_small_div(row, 2 * h); + assert(row_of(p, num_ntts as nat) == row); + assert(blk_of(p, num_ntts as nat, block_size_half as nat) == 0); + assert(r_of(p, num_ntts as nat, block_size_half as nat) == row); + if row < h { + assert(p < h * m) by (nonlinear_arith) + requires + p == row * m + p % m, + p % m < m, + row + 1 <= h, + ; + assert(x0[p + h * m] == bot0[p]); + assert(x0[p] == top0[p]); + assert(fin[p] == top@[p]); + assert(top@[p].0 == bf_top(top0[p].0, bot0[p].0, twiddle.0)); + assert(y[p] == F64(bf_top(x0[p].0, x0[p + h * m].0, twiddle.0))); + } else { + assert(p >= h * m) by (nonlinear_arith) + requires + p == row * m + p % m, + p % m >= 0, + row >= h, + m > 0, + ; + assert(x0[p - h * m] == top0[p - h * m]); + assert(x0[p] == bot0[p - h * m]); + assert(fin[p] == bot@[p - h * m]); + assert(top@[p - h * m].0 == bf_top(top0[p - h * m].0, bot0[p - h * m].0, twiddle.0)); + assert(bot@[p - h * m].0 == bf_bot(top0[p - h * m].0, bot0[p - h * m].0, twiddle.0)); + assert(y[p] == F64(bf_bot(x0[p - h * m].0, x0[p].0, twiddle.0))); + } + } + assert(fin =~= y); + } +} + +/// A row below the block size is in block 0, at itself. +proof fn lemma_small_div(row: int, n: int) + requires + 0 <= row < n, + ensures + row / n == 0, + row % n == row, +{ + lemma_fundamental_div_mod_converse(row, n, 0, row); +} + +// --------------------------------------------------------------------------------------------- +// Layers on one sub-block +// --------------------------------------------------------------------------------------------- +/// `buf` is `prev` with its first `n_done` words replaced by those of `target`. +pub open spec fn swept_to(buf: Seq, prev: Seq, target: Seq, n_done: int) -> bool { + &&& buf.len() == prev.len() + &&& target.len() == prev.len() + &&& forall|p: int| 0 <= p < n_done ==> #[trigger] buf[p] == target[p] + &&& forall|p: int| n_done <= p < prev.len() ==> #[trigger] buf[p] == prev[p] +} + +/// Layers with twiddles that agree on the blocks of the buffer are equal. +pub proof fn lemma_layer_map_tw_n(x: Seq, m: nat, h: nat, tw: spec_fn(int) -> u64, tw2: spec_fn(int) -> u64, nb: nat) + requires + m > 0, + h > 0, + x.len() == nb * (2 * h * m), + forall|b: int| 0 <= b < nb ==> #[trigger] tw(b) == tw2(b), + ensures + layer_map(x, m, h, tw, false) == layer_map(x, m, h, tw2, false), +{ + assert forall|p: int| 0 <= p < x.len() implies layer_map(x, m, h, tw, false)[p] == layer_map(x, m, h, tw2, false)[p] by { + let mi = m as int; + lemma_split_word(p, mi); + let row = p / mi; + assert(row < nb * (2 * h)) by (nonlinear_arith) + requires + p == row * mi + p % mi, + p % mi >= 0, + p < nb * (2 * h * m), + mi == m, + m > 0, + ; + lemma_div_pos_is_pos(row, 2 * h as int); + lemma_div_is_ordered(row, nb * (2 * h) - 1, 2 * h as int); + assert(nb * (2 * h) - 1 == (nb - 1) * (2 * h) + (2 * h - 1)) by (nonlinear_arith) + requires + nb >= 1, + ; + lemma_div_multiples_vanish_fancy(nb - 1, 2 * h - 1, 2 * h as int); + assert((nb - 1) * (2 * h) == (2 * h) * (nb - 1)) by (nonlinear_arith); + } + assert(layer_map(x, m, h, tw, false) =~= layer_map(x, m, h, tw2, false)); +} + +/// Block `b` of a layer of a sub-block: the target layers, restricted to it, are the layers of the block +/// as sub-block `s * 2^(layer - o) + b` of `2^layer`. +proof fn lemma_block_target(tab: Seq>, prev: Seq, m: nat, log_d: nat, o: nat, s: int, layer: nat, c: nat, b: int) + requires + m > 0, + o <= layer, + layer + c <= log_d, + s >= 0, + 0 <= b < pow2((layer - o) as nat), + prev.len() == m * pow2((log_d - o) as nat), + ensures + ({ + let be = pow2((log_d - layer) as nat) * m; + let s2 = s * pow2((layer - o) as nat) + b; + &&& (b + 1) * be <= prev.len() + &&& b * be + be == (b + 1) * be + &&& b * be >= 0 + &&& be > 0 + &&& sub_layers(tab, prev, m, log_d, o, s, layer, layer + c).subrange(b * be, b * be + be) == sub_layers( + tab, + prev.subrange(b * be, b * be + be), + m, + log_d, + layer, + s2, + layer, + layer + c, + ) + }), +{ + let bs = pow2((log_d - layer) as nat) as int; + let be = bs * m; + let po = pow2((layer - o) as nat) as int; + let s2 = s * po + b; + let target = sub_layers(tab, prev, m, log_d, o, s, layer, layer + c); + lemma_sub_layers_len(tab, prev, m, log_d, o, s, layer, layer + c); + lemma_pow2_pos((log_d - layer) as nat); + lemma_pow2_adds((layer - o) as nat, (log_d - layer) as nat); + assert((layer - o) as nat + (log_d - layer) as nat == (log_d - o) as nat); + assert(pow2(0) == 1) by { + lemma2_to64(); + } + assert(s * po <= s2 < (s + 1) * po) by (nonlinear_arith) + requires + s2 == s * po + b, + 0 <= b < po, + ; + lemma_gather_sub_layers(tab, prev, m, log_d, o, s, log_d, layer, s2, 0, layer, layer + c); + assert((b + 1) * bs * m <= prev.len()) by (nonlinear_arith) + requires + b + 1 <= po, + prev.len() == m * (po * bs), + bs > 0, + ; + assert(b * bs >= 0) by (nonlinear_arith) + requires + b >= 0, + bs > 0, + ; + assert((b * bs + bs) * m == (b + 1) * bs * m) by (nonlinear_arith); + lemma_gather_contiguous(target, m, b * bs, bs as nat); + lemma_gather_contiguous(prev, m, b * bs, bs as nat); + assert(b * bs * m == b * be && (b * bs + bs) * m == b * be + be && (b + 1) * be == b * be + be) by (nonlinear_arith) + requires + be == bs * m, + ; + assert(be > 0) by (nonlinear_arith) + requires + be == bs * m, + bs > 0, + m > 0, + ; + assert(b * be >= 0) by (nonlinear_arith) + requires + b >= 0, + be > 0, + ; + assert(s2 - s * po == b); +} + +/// One block done: the sweep moves past it. +proof fn lemma_block_swept(before: Seq, after: Seq, prev: Seq, target: Seq, start: int, be: int) + requires + 0 <= start, + 0 <= be, + start + be <= prev.len(), + swept_to(before, prev, target, start), + after == before.subrange(0, start) + target.subrange(start, start + be) + before.subrange(start + be, before.len() as int), + ensures + swept_to(after, prev, target, start + be), +{ + assert forall|p: int| 0 <= p < start + be implies #[trigger] after[p] == target[p] by { + if p < start { + assert(after[p] == before.subrange(0, start)[p]); + } else { + assert(after[p] == target.subrange(start, start + be)[p - start]); + } + } + assert forall|p: int| start + be <= p < prev.len() implies #[trigger] after[p] == prev[p] by { + assert(after[p] == before.subrange(start + be, before.len() as int)[p - start - be]); + } +} + +/// Three layers on one block, as the fused sweep runs them with `twiddles_radix8`'s twiddles. +proof fn lemma_sub3(tab: Seq>, blk: Seq, m: nat, log_d: nat, layer: nat, s2: int, t: Seq) + requires + m > 0, + layer + 3 <= log_d, + blk.len() == m * pow2((log_d - layer) as nat), + t.len() == 7, + t[0].0 == twiddle_spec(tab, layer, s2 as usize), + t[1].0 == twiddle_spec(tab, layer + 1, (2 * s2) as usize), + t[2].0 == twiddle_spec(tab, layer + 1, (2 * s2 + 1) as usize), + t[3].0 == twiddle_spec(tab, layer + 2, (4 * s2) as usize), + t[4].0 == twiddle_spec(tab, layer + 2, (4 * s2 + 1) as usize), + t[5].0 == twiddle_spec(tab, layer + 2, (4 * s2 + 2) as usize), + t[6].0 == twiddle_spec(tab, layer + 2, (4 * s2 + 3) as usize), + ensures + sub_layers(tab, blk, m, log_d, layer, s2, layer, layer + 3) == layer3(blk, m, pow2((log_d - layer - 3) as nat), t), +{ + let e = pow2((log_d - layer - 3) as nat); + lemma_pow2_unfold((log_d - layer) as nat); + lemma_pow2_unfold((log_d - layer - 1) as nat); + lemma_pow2_unfold((log_d - layer - 2) as nat); + lemma_pow2_pos((log_d - layer - 3) as nat); + lemma2_to64(); + assert(layer_half(log_d, layer) == 4 * e); + assert(layer_half(log_d, layer + 1) == 2 * e); + assert(layer_half(log_d, layer + 2) == e); + let x0 = blk; + let x1 = layer_map(x0, m, 4 * e, sub_twiddles(tab, layer, layer, s2), false); + let y1 = layer_map(x0, m, 4 * e, |b: int| t[0].0, false); + assert(blk.len() == 1 * (2 * (4 * e) * m)) by (nonlinear_arith) + requires + blk.len() == m * (2 * (2 * (2 * e))), + ; + lemma_layer_map_tw_n(x0, m, 4 * e, sub_twiddles(tab, layer, layer, s2), |b: int| t[0].0, 1); + assert(x1.len() == 2 * (2 * (2 * e) * m)) by (nonlinear_arith) + requires + x1.len() == 1 * (2 * (4 * e) * m), + ; + let x2 = layer_map(x1, m, 2 * e, sub_twiddles(tab, layer + 1, layer, s2), false); + assert forall|b: int| 0 <= b < 2 implies #[trigger] (sub_twiddles(tab, layer + 1, layer, s2))(b) == (|b: int| t[1 + b].0)(b) by { + assert(pow2(((layer + 1) - layer) as nat) == 2); + if b == 0 {} else {} + } + lemma_layer_map_tw_n(x1, m, 2 * e, sub_twiddles(tab, layer + 1, layer, s2), |b: int| t[1 + b].0, 2); + assert(x2.len() == 4 * (2 * e * m)) by (nonlinear_arith) + requires + x2.len() == 2 * (2 * (2 * e) * m), + ; + assert forall|b: int| 0 <= b < 4 implies #[trigger] (sub_twiddles(tab, layer + 2, layer, s2))(b) == (|b: int| t[3 + b].0)(b) by { + assert(pow2(((layer + 2) - layer) as nat) == 4); + if b == 0 {} else if b == 1 {} else if b == 2 {} else {} + } + lemma_layer_map_tw_n(x2, m, e, sub_twiddles(tab, layer + 2, layer, s2), |b: int| t[3 + b].0, 4); + reveal_with_fuel(sub_layers, 4); + assert(sub_layers(tab, blk, m, log_d, layer, s2, layer, layer + 3) == layer_map( + x2, + m, + e, + sub_twiddles(tab, layer + 2, layer, s2), + false, + )); +} + +/// Two layers on one block, as the fused sweep runs them with the block's three twiddles. +proof fn lemma_sub2(tab: Seq>, blk: Seq, m: nat, log_d: nat, layer: nat, s2: int, t_outer: u64, t_inner_a: u64, t_inner_b: u64) + requires + m > 0, + layer + 2 <= log_d, + blk.len() == m * pow2((log_d - layer) as nat), + t_outer == twiddle_spec(tab, layer, s2 as usize), + t_inner_a == twiddle_spec(tab, layer + 1, (2 * s2) as usize), + t_inner_b == twiddle_spec(tab, layer + 1, (2 * s2 + 1) as usize), + ensures + sub_layers(tab, blk, m, log_d, layer, s2, layer, layer + 2) == layer2( + blk, + m, + pow2((log_d - layer - 2) as nat), + t_outer, + t_inner_a, + t_inner_b, + ), +{ + let e = pow2((log_d - layer - 2) as nat); + lemma_pow2_unfold((log_d - layer) as nat); + lemma_pow2_unfold((log_d - layer - 1) as nat); + lemma_pow2_pos((log_d - layer - 2) as nat); + lemma2_to64(); + assert(layer_half(log_d, layer) == 2 * e); + assert(layer_half(log_d, layer + 1) == e); + let x0 = blk; + let x1 = layer_map(x0, m, 2 * e, sub_twiddles(tab, layer, layer, s2), false); + assert(blk.len() == 1 * (2 * (2 * e) * m)) by (nonlinear_arith) + requires + blk.len() == m * (2 * (2 * e)), + ; + assert(pow2((layer - layer) as nat) == 1); + assert(s2 * pow2((layer - layer) as nat) == s2) by { + lemma_mul_basics(s2); + } + assert((sub_twiddles(tab, layer, layer, s2))(0) == t_outer); + lemma_layer_map_tw_n(x0, m, 2 * e, sub_twiddles(tab, layer, layer, s2), |b: int| t_outer, 1); + assert(x1.len() == 2 * (2 * e * m)) by (nonlinear_arith) + requires + x1.len() == 1 * (2 * (2 * e) * m), + ; + assert forall|b: int| 0 <= b < 2 implies #[trigger] (sub_twiddles(tab, layer + 1, layer, s2))(b) == (pick2(t_inner_a, t_inner_b))(b) by { + assert(pow2(((layer + 1) - layer) as nat) == 2); + if b == 0 {} else {} + } + lemma_layer_map_tw_n(x1, m, e, sub_twiddles(tab, layer + 1, layer, s2), pick2(t_inner_a, t_inner_b), 2); + reveal_with_fuel(sub_layers, 3); + assert(sub_layers(tab, blk, m, log_d, layer, s2, layer, layer + 2) == layer_map( + x1, + m, + e, + sub_twiddles(tab, layer + 1, layer, s2), + false, + )); +} + +/// One layer on one block. +proof fn lemma_sub1(tab: Seq>, blk: Seq, m: nat, log_d: nat, layer: nat, s2: int, t: u64) + requires + m > 0, + layer + 1 <= log_d, + blk.len() == m * pow2((log_d - layer) as nat), + t == twiddle_spec(tab, layer, s2 as usize), + ensures + sub_layers(tab, blk, m, log_d, layer, s2, layer, layer + 1) == layer_map( + blk, + m, + pow2((log_d - layer - 1) as nat), + |b: int| t, + false, + ), +{ + let h = pow2((log_d - layer - 1) as nat); + lemma_pow2_unfold((log_d - layer) as nat); + lemma_pow2_pos((log_d - layer - 1) as nat); + lemma2_to64(); + assert(blk.len() == 1 * (2 * h * m)) by (nonlinear_arith) + requires + blk.len() == m * (2 * h), + ; + assert(pow2((layer - layer) as nat) == 1); + lemma_layer_map_tw_n(blk, m, h, sub_twiddles(tab, layer, layer, s2), |b: int| t, 1); + reveal_with_fuel(sub_layers, 2); +} + +/// The table rows a layer's twiddles read. +proof fn lemma_table_rows(ntt: &AdditiveNttF64, layer: nat) + requires + ntt.well_formed(), + layer < ntt.table().len(), + ensures + ntt.table()[ntt.table().len() - layer - 1].len() == layer + 1, + ntt.table().len() <= 63, +{ +} + +/// The global index of block `b` of sub-block `s` is below the layer's block count. +proof fn lemma_global_block(s: int, o: nat, layer: nat, b: int) + requires + o <= layer, + 0 <= s < pow2(o), + 0 <= b < pow2((layer - o) as nat), + ensures + 0 <= s * pow2((layer - o) as nat) + b < pow2(layer), +{ + lemma_pow2_adds(o, (layer - o) as nat); + assert(o + (layer - o) as nat == layer); + assert(0 <= s * pow2((layer - o) as nat) + b < pow2(o) * pow2((layer - o) as nat)) by (nonlinear_arith) + requires + 0 <= s < pow2(o), + 0 <= b < pow2((layer - o) as nat), + ; +} + +impl AdditiveNttF64 { + /// Run a range of layers in place over one sub-block of the domain. + /// + /// - The domain has `2^d` rows and splits into `2^o` equal sub-blocks. + /// - The buffer is one of them; its index fixes the global block index, and so the twiddle, of each block. + /// - Three layers fuse into one radix-8 sweep where blocks are wide enough, then two, then one. + /// + /// Verified: the buffer ends as [`sub_layers`]. Rewritten: the `global` closure carries its + /// specification (a ghost addition). + #[allow(clippy::too_many_arguments)] + pub fn run_layers( + &self, + buf: &mut [F64], + log_d: usize, + num_ntts: usize, + first_layer: usize, + end_layer: usize, + outer_log: usize, + sub_idx: usize, + ) + requires + self.well_formed(), + num_ntts > 0, + outer_log <= first_layer <= end_layer <= log_d <= self.table().len(), + sub_idx < pow2(outer_log as nat), + old(buf)@.len() == num_ntts * pow2((log_d - outer_log) as nat), + old(buf)@.len() <= usize::MAX, + ensures + final(buf)@ == sub_layers( + self.table(), + old(buf)@, + num_ntts as nat, + log_d as nat, + outer_log as nat, + sub_idx as int, + first_layer as nat, + end_layer as nat, + ), + { + let ghost tab = self.table(); + let ghost b0 = buf@; + let ghost m = num_ntts as nat; + let ghost len = b0.len() as int; + let mut layer = first_layer; + while layer < end_layer + invariant + self.well_formed(), + tab == self.table(), + m == num_ntts, + num_ntts > 0, + outer_log <= first_layer <= layer <= end_layer <= log_d <= tab.len(), + sub_idx < pow2(outer_log as nat), + len == b0.len(), + len == num_ntts * pow2((log_d - outer_log) as nat), + len <= usize::MAX, + buf@ == sub_layers(tab, b0, m, log_d as nat, outer_log as nat, sub_idx as int, first_layer as nat, layer as nat), + decreases end_layer - layer, + { + let ghost prev = buf@; + proof { + lemma_sub_layers_len(tab, b0, m, log_d as nat, outer_log as nat, sub_idx as int, first_layer as nat, layer as nat); + lemma_table_rows(self, layer as nat); + lemma_usize_pow2_no_overflow((layer - outer_log) as nat); + lemma_usize_pow2_no_overflow((log_d - layer) as nat); + lemma_usize_shl_is_mul(1, (layer - outer_log) as usize); + lemma_usize_shl_is_mul(1, (log_d - layer) as usize); + lemma_pow2_adds((layer - outer_log) as nat, (log_d - layer) as nat); + assert((layer - outer_log) as nat + (log_d - layer) as nat == (log_d - outer_log) as nat); + lemma_pow2_pos((layer - outer_log) as nat); + lemma_pow2_pos((log_d - layer) as nat); + assert(pow2((log_d - layer) as nat) * num_ntts <= len) by (nonlinear_arith) + requires + len == num_ntts * (pow2((layer - outer_log) as nat) * pow2((log_d - layer) as nat)), + pow2((layer - outer_log) as nat) >= 1, + ; + } + // This layer's blocks inside the buffer: how many, and their size in rows and words. + let num_blocks_in_buf = 1usize << (layer - outer_log); + let block_size = 1usize << (log_d - layer); + let block_elems = block_size * num_ntts; + // A block's index in the whole domain, which picks its twiddle. + let global = |block_in_buf: usize| -> (r: usize) + requires + block_in_buf < num_blocks_in_buf, + num_blocks_in_buf == pow2((layer - outer_log) as nat), + sub_idx < pow2(outer_log as nat), + outer_log <= layer < 64, + ensures + r == sub_idx * num_blocks_in_buf + block_in_buf, + r < pow2(layer as nat), + { + proof { + lemma_global_block(sub_idx as int, outer_log as nat, layer as nat, block_in_buf as int); + lemma_usize_pow2_no_overflow(layer as nat); + } + sub_idx * num_blocks_in_buf + block_in_buf + }; + let ghost tw_layers: nat = if layer + 2 < end_layer && block_size >= 8 { + 3 + } else if layer + 1 < end_layer && block_size >= 4 { + 2 + } else { + 1 + }; + let ghost target = sub_layers(tab, prev, m, log_d as nat, outer_log as nat, sub_idx as int, layer as nat, (layer + tw_layers) as nat); + proof { + lemma_sub_layers_len(tab, prev, m, log_d as nat, outer_log as nat, sub_idx as int, layer as nat, (layer + tw_layers) as nat); + } + if layer + 2 < end_layer && block_size >= 8 { + // Three layers to go and blocks of at least 8 rows: one radix-8 sweep. + let eighth = block_size >> 3; + proof { + lemma_usize_shr_is_div(block_size, 3); + lemma_pow2_unfold((log_d - layer) as nat); + lemma_pow2_unfold((log_d - layer - 1) as nat); + lemma_pow2_unfold((log_d - layer - 2) as nat); + lemma2_to64(); + lemma_div_multiples_vanish(pow2((log_d - layer - 3) as nat) as int, 8); + assert(eighth == pow2((log_d - layer - 3) as nat)); + lemma_table_rows(self, (layer + 1) as nat); + lemma_table_rows(self, (layer + 2) as nat); + assert(0 * block_elems == 0); + } + for block_in_buf in 0..num_blocks_in_buf + invariant + self.well_formed(), + tab == self.table(), + num_ntts > 0, + m == num_ntts, + outer_log <= layer, + layer + 3 <= end_layer <= log_d <= tab.len() <= 63, + sub_idx < pow2(outer_log as nat), + num_blocks_in_buf == pow2((layer - outer_log) as nat), + block_size == pow2((log_d - layer) as nat), + block_elems == block_size * num_ntts, + eighth == pow2((log_d - layer - 3) as nat), + block_size == 8 * eighth, + prev.len() == len, + len == num_ntts * pow2((log_d - outer_log) as nat), + len <= usize::MAX, + target == sub_layers(tab, prev, m, log_d as nat, outer_log as nat, sub_idx as int, layer as nat, (layer + 3) as nat), + swept_to(buf@, prev, target, block_in_buf * block_elems), + forall|b: usize| b < num_blocks_in_buf ==> #[trigger] global.requires((b,)), + forall|b: usize, r: usize| + #[trigger] global.ensures((b,), r) ==> r == sub_idx * num_blocks_in_buf + b && r < pow2(layer as nat), + { + proof { + lemma_block_target(tab, prev, m, log_d as nat, outer_log as nat, sub_idx as int, layer as nat, 3, block_in_buf as int); + assert(block_elems == m * pow2((log_d - layer) as nat)) by (nonlinear_arith) + requires + block_elems == block_size * num_ntts, + block_size == pow2((log_d - layer) as nat), + m == num_ntts, + ; + } + let t = self.twiddles_radix8(layer, global(block_in_buf)); + let start = block_in_buf * block_elems; + let ghost before = buf@; + proof { + let s2 = sub_idx * pow2((layer - outer_log) as nat) + block_in_buf; + lemma_sub3(tab, prev.subrange(start as int, start + block_elems), m, log_d as nat, layer as nat, s2, t@); + assert(before.subrange(start as int, start + block_elems) =~= prev.subrange(start as int, start + block_elems)); + assert(block_elems == 8 * (eighth * num_ntts)) by (nonlinear_arith) + requires + block_elems == block_size * num_ntts, + block_size == 8 * eighth, + ; + } + butterfly_interleaved_fused_3layer(&mut buf[start..start + block_elems], &t, eighth, num_ntts); + proof { + lemma_block_swept(before, buf@, prev, target, start as int, block_elems as int); + assert((block_in_buf + 1) * block_elems == start + block_elems) by (nonlinear_arith) + requires + start == block_in_buf * block_elems, + ; + } + } + layer += 3; + } else if layer + 1 < end_layer && block_size >= 4 { + // Two layers to go: one radix-4 sweep. + let quarter = block_size >> 2; + proof { + lemma_usize_shr_is_div(block_size, 2); + lemma_pow2_unfold((log_d - layer) as nat); + lemma_pow2_unfold((log_d - layer - 1) as nat); + lemma2_to64(); + lemma_div_multiples_vanish(pow2((log_d - layer - 2) as nat) as int, 4); + assert(quarter == pow2((log_d - layer - 2) as nat)); + lemma_table_rows(self, (layer + 1) as nat); + assert(0 * block_elems == 0); + } + for block_in_buf in 0..num_blocks_in_buf + invariant + self.well_formed(), + tab == self.table(), + num_ntts > 0, + m == num_ntts, + outer_log <= layer, + layer + 2 <= end_layer <= log_d <= tab.len() <= 63, + sub_idx < pow2(outer_log as nat), + num_blocks_in_buf == pow2((layer - outer_log) as nat), + block_size == pow2((log_d - layer) as nat), + block_elems == block_size * num_ntts, + quarter == pow2((log_d - layer - 2) as nat), + block_size == 4 * quarter, + prev.len() == len, + len == num_ntts * pow2((log_d - outer_log) as nat), + len <= usize::MAX, + target == sub_layers(tab, prev, m, log_d as nat, outer_log as nat, sub_idx as int, layer as nat, (layer + 2) as nat), + swept_to(buf@, prev, target, block_in_buf * block_elems), + forall|b: usize| b < num_blocks_in_buf ==> #[trigger] global.requires((b,)), + forall|b: usize, r: usize| + #[trigger] global.ensures((b,), r) ==> r == sub_idx * num_blocks_in_buf + b && r < pow2(layer as nat), + { + proof { + lemma_block_target(tab, prev, m, log_d as nat, outer_log as nat, sub_idx as int, layer as nat, 2, block_in_buf as int); + assert(block_elems == m * pow2((log_d - layer) as nat)) by (nonlinear_arith) + requires + block_elems == block_size * num_ntts, + block_size == pow2((log_d - layer) as nat), + m == num_ntts, + ; + } + let global_block = global(block_in_buf); + proof { + lemma_pow2_unfold((layer + 1) as nat); + lemma_usize_pow2_no_overflow((layer + 1) as nat); + } + let t_outer = self.twiddle(layer, global_block); + let t_inner_a = self.twiddle(layer + 1, 2 * global_block); + let t_inner_b = self.twiddle(layer + 1, 2 * global_block + 1); + let start = block_in_buf * block_elems; + let ghost before = buf@; + proof { + let s2 = sub_idx * pow2((layer - outer_log) as nat) + block_in_buf; + lemma_sub2(tab, prev.subrange(start as int, start + block_elems), m, log_d as nat, layer as nat, s2, t_outer.0, t_inner_a.0, t_inner_b.0); + assert(before.subrange(start as int, start + block_elems) =~= prev.subrange(start as int, start + block_elems)); + assert(block_elems == 4 * (quarter * num_ntts)) by (nonlinear_arith) + requires + block_elems == block_size * num_ntts, + block_size == 4 * quarter, + ; + } + butterfly_interleaved_fused_2layer( + &mut buf[start..start + block_elems], + t_outer, + t_inner_a, + t_inner_b, + quarter, + num_ntts, + ); + proof { + lemma_block_swept(before, buf@, prev, target, start as int, block_elems as int); + assert((block_in_buf + 1) * block_elems == start + block_elems) by (nonlinear_arith) + requires + start == block_in_buf * block_elems, + ; + } + } + layer += 2; + } else { + // One layer: a plain butterfly sweep. + let block_size_half = block_size >> 1; + proof { + lemma_usize_shr_is_div(block_size, 1); + lemma_pow2_unfold((log_d - layer) as nat); + lemma2_to64(); + lemma_div_multiples_vanish(pow2((log_d - layer - 1) as nat) as int, 2); + assert(block_size_half == pow2((log_d - layer - 1) as nat)); + lemma_pow2_pos((log_d - layer - 1) as nat); + assert(0 * block_elems == 0); + } + for block_in_buf in 0..num_blocks_in_buf + invariant + self.well_formed(), + tab == self.table(), + num_ntts > 0, + m == num_ntts, + outer_log <= layer, + layer + 1 <= end_layer <= log_d <= tab.len() <= 63, + sub_idx < pow2(outer_log as nat), + num_blocks_in_buf == pow2((layer - outer_log) as nat), + block_size == pow2((log_d - layer) as nat), + block_elems == block_size * num_ntts, + block_size_half == pow2((log_d - layer - 1) as nat), + block_size == 2 * block_size_half, + block_size_half > 0, + prev.len() == len, + len == num_ntts * pow2((log_d - outer_log) as nat), + len <= usize::MAX, + target == sub_layers(tab, prev, m, log_d as nat, outer_log as nat, sub_idx as int, layer as nat, (layer + 1) as nat), + swept_to(buf@, prev, target, block_in_buf * block_elems), + forall|b: usize| b < num_blocks_in_buf ==> #[trigger] global.requires((b,)), + forall|b: usize, r: usize| + #[trigger] global.ensures((b,), r) ==> r == sub_idx * num_blocks_in_buf + b && r < pow2(layer as nat), + { + proof { + lemma_block_target(tab, prev, m, log_d as nat, outer_log as nat, sub_idx as int, layer as nat, 1, block_in_buf as int); + assert(block_elems == m * pow2((log_d - layer) as nat)) by (nonlinear_arith) + requires + block_elems == block_size * num_ntts, + block_size == pow2((log_d - layer) as nat), + m == num_ntts, + ; + } + let twiddle = self.twiddle(layer, global(block_in_buf)); + let start = block_in_buf * block_elems; + let ghost before = buf@; + proof { + let s2 = sub_idx * pow2((layer - outer_log) as nat) + block_in_buf; + lemma_sub1(tab, prev.subrange(start as int, start + block_elems), m, log_d as nat, layer as nat, s2, twiddle.0); + assert(before.subrange(start as int, start + block_elems) =~= prev.subrange(start as int, start + block_elems)); + assert(block_elems == 2 * (block_size_half * num_ntts)) by (nonlinear_arith) + requires + block_elems == block_size * num_ntts, + block_size == 2 * block_size_half, + ; + } + butterfly_interleaved_block(&mut buf[start..start + block_elems], twiddle, block_size_half, num_ntts); + proof { + lemma_block_swept(before, buf@, prev, target, start as int, block_elems as int); + assert((block_in_buf + 1) * block_elems == start + block_elems) by (nonlinear_arith) + requires + start == block_in_buf * block_elems, + ; + } + } + layer += 1; + } + proof { + assert(num_blocks_in_buf * block_elems == len) by (nonlinear_arith) + requires + num_blocks_in_buf == pow2((layer - tw_layers - outer_log) as nat), + block_elems == pow2((log_d - (layer - tw_layers)) as nat) * num_ntts, + len == num_ntts * (pow2((layer - tw_layers - outer_log) as nat) * pow2((log_d - (layer - tw_layers)) as nat)), + ; + assert(buf@ =~= target); + lemma_sub_layers_split(tab, b0, m, log_d as nat, outer_log as nat, sub_idx as int, first_layer as nat, (layer - tw_layers) as nat, layer as nat); + } + } + } +} + +// --------------------------------------------------------------------------------------------- +// The gathered pass +// --------------------------------------------------------------------------------------------- +/// Word `k` is in the gather of the rows `base_row + r + i * step`, `i < cnt`. +pub open spec fn in_gather(k: int, m: nat, base_row: int, r: int, step: int, cnt: nat) -> bool { + &&& 0 <= k + &&& base_row <= k / (m as int) < base_row + cnt * step + &&& (k / (m as int) - base_row) % step == r +} + +/// Row `i` of a gather: its word offset, and the gather words it holds. +proof fn lemma_gather_row(m: nat, base_row: int, r: int, step: int, cnt: nat, i: int) + requires + m > 0, + base_row >= 0, + 0 <= r < step, + 0 <= i < cnt, + ensures + (base_row + r + i * step) * m >= 0, + (base_row + r + i * step) * m + m <= (base_row + cnt * step) * m, + forall|lane: int| 0 <= lane < m ==> #[trigger] gidx(m, base_row, r, step, i * m + lane) == (base_row + r + i * step) * m + lane, +{ + let mi = m as int; + assert(base_row + r + i * step >= 0) by (nonlinear_arith) + requires + base_row >= 0, + r >= 0, + i >= 0, + step > 0, + ; + assert(base_row + r + i * step + 1 <= base_row + cnt * step) by (nonlinear_arith) + requires + i + 1 <= cnt, + 0 <= r < step, + ; + assert((base_row + r + i * step) * mi >= 0) by (nonlinear_arith) + requires + base_row + r + i * step >= 0, + mi > 0, + ; + assert((base_row + r + i * step) * mi + mi <= (base_row + cnt * step) * mi) by (nonlinear_arith) + requires + base_row + r + i * step + 1 <= base_row + cnt * step, + mi > 0, + ; + assert forall|lane: int| 0 <= lane < m implies #[trigger] gidx(m, base_row, r, step, i * m + lane) == (base_row + r + i * step) * m + lane by { + lemma_word(i, lane, mi); + } +} + +/// The words of row `i` of a gather are in it. +proof fn lemma_row_in_gather(k: int, m: nat, base_row: int, r: int, step: int, cnt: nat, i: int) + requires + m > 0, + base_row >= 0, + 0 <= r < step, + 0 <= i < cnt, + (base_row + r + i * step) * m <= k < (base_row + r + i * step) * m + m, + ensures + in_gather(k, m, base_row, r, step, cnt), +{ + let mi = m as int; + lemma_gather_row(m, base_row, r, step, cnt, i); + lemma_word(base_row + r + i * step, k - (base_row + r + i * step) * mi, mi); + assert(base_row + r + i * step + 1 <= base_row + cnt * step) by (nonlinear_arith) + requires + i + 1 <= cnt, + 0 <= r < step, + ; + assert(i * step >= 0) by (nonlinear_arith) + requires + i >= 0, + step > 0, + ; + lemma_fundamental_div_mod_converse(r + i * step, step, i, r); +} + +/// Every word of a gather is word `q` of row `q / m`. +proof fn lemma_in_gather_index(k: int, m: nat, base_row: int, r: int, step: int, cnt: nat) + requires + m > 0, + base_row >= 0, + 0 <= r < step, + in_gather(k, m, base_row, r, step, cnt), + ensures + ({ + let q = ((k / (m as int) - base_row) / step) * m + k % (m as int); + &&& 0 <= q < cnt * m + &&& gidx(m, base_row, r, step, q) == k + }), +{ + let mi = m as int; + let row = k / mi; + let i = (row - base_row) / step; + let lane = k % mi; + lemma_split_word(k, mi); + lemma_fundamental_div_mod(row - base_row, step); + lemma_div_pos_is_pos(row - base_row, step); + assert(row - base_row == i * step + r) by { + lemma_mul_is_commutative(step, i); + } + assert(i < cnt) by (nonlinear_arith) + requires + row - base_row == i * step + r, + r >= 0, + row < base_row + cnt * step, + step > 0, + ; + lemma_gather_row(m, base_row, r, step, cnt, i); + lemma_word(i, lane, mi); + assert(i * mi + lane < cnt * mi) by (nonlinear_arith) + requires + i + 1 <= cnt, + lane < mi, + mi > 0, + ; + assert(i * mi + lane >= 0) by (nonlinear_arith) + requires + i >= 0, + lane >= 0, + mi > 0, + ; +} + +/// Lend this thread's scratch buffer, grown to the requested length. +/// +/// - It is cache-line aligned, so a full-width load never splits a line. +/// - It lives as long as the thread, so no pass allocates in its hot loop. +/// +/// Trusted (`external_body`): Verus has no model of `thread_local!` and `RefCell`. Specification: `f` +/// runs once, on a slice of exactly `len` words. Rewritten: `f` also takes a `state` argument, which +/// `with_scratch` hands it untouched; Verus does not support closures that capture a mutable +/// reference, which the deep pass's closure does (its task's slice). +#[verifier::external_body] +fn with_scratch R>(len: usize, state: S, f: F) -> (r: R) + requires + forall|s: &mut [F64]| s@.len() == len ==> #[trigger] f.requires((s, state)), + ensures + exists|s: &mut [F64]| s@.len() == len && #[trigger] f.ensures((s, state), r), +{ + // One cache line of words. + #[derive(Clone, Copy)] + #[repr(C, align(64))] + struct Line([F64; 8]); + thread_local! { + static SCRATCH: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; + } + SCRATCH.with_borrow_mut(|lines| { + // Grow only; a later, smaller request reuses the same lines. + if lines.len() * 8 < len { + lines.resize(len.div_ceil(8), Line([F64(0); 8])); + } + // SAFETY: + // - A line is exactly eight contiguous words, with no padding. + // - The buffer holds at least the requested words, all initialized. + f(unsafe { std::slice::from_raw_parts_mut(lines.as_mut_ptr().cast::(), len) }, state) + }) +} + +/// What a group reads: the gathered input `src`, from the codeword (no message) or from the message +/// (`in_place`: block 0 of the codeword; otherwise its own buffer). +pub open spec fn group_src( + own: Map>, + mperms: Map>, + base: *mut F64, + msg: Option>, + in_place: bool, + m: nat, + block_row: int, + r: int, + step: int, + rows: nat, + src: Seq, +) -> bool { + &&& src.len() == rows * m + &&& base@.addr + (block_row + rows * step) * m * size_of::() <= usize::MAX + &&& match msg { + None => forall|q: int| + 0 <= q < rows * m ==> #[trigger] own[gidx(m, block_row, r, step, q)].value() == src[q], + Some(mp) => if in_place { + &&& mp.0 == base + &&& forall|q: int| + 0 <= q < rows * m ==> own.dom().contains(#[trigger] gidx(m, 0, r, step, q)) && own[gidx(m, 0, r, step, q)].ptr() == ptr_at( + base, + gidx(m, 0, r, step, q), + ) && own[gidx(m, 0, r, step, q)].is_init() && own[gidx(m, 0, r, step, q)].value() == src[q] + } else { + &&& covers(mperms, mp.0, 0, rows * step * m) + &&& forall|q: int| 0 <= q < rows * m ==> #[trigger] mperms[gidx(m, 0, r, step, q)].value() == src[q] + }, + } +} + +/// The words of row `i` of a gather, all held by a map that holds the whole gather. +proof fn lemma_row_keys_in(own: Map>, m: nat, base_row: int, r: int, step: int, cnt: nat, i: int) + requires + m > 0, + base_row >= 0, + 0 <= r < step, + 0 <= i < cnt, + forall|q: int| 0 <= q < cnt * m ==> own.dom().contains(#[trigger] gidx(m, base_row, r, step, q)), + ensures + Set::range((base_row + r + i * step) * m, (base_row + r + i * step) * m + m) <= own.dom(), +{ + let mi = m as int; + lemma_gather_row(m, base_row, r, step, cnt, i); + assert forall|k: int| Set::range((base_row + r + i * step) * m, (base_row + r + i * step) * m + m).contains(k) implies own.dom().contains(k) by { + let lane = k - (base_row + r + i * step) * mi; + assert(gidx(m, base_row, r, step, i * m + lane) == k); + assert(i * mi + lane < cnt * mi) by (nonlinear_arith) + requires + i + 1 <= cnt, + 0 <= lane < mi, + ; + assert(i * mi + lane >= 0) by (nonlinear_arith) + requires + i >= 0, + lane >= 0, + mi > 0, + ; + } +} + +/// `a` holds the permissions of `b`, with the same values. +pub open spec fn same_vals(a: Map>, b: Map>) -> bool { + &&& a.dom() == b.dom() + &&& forall|k: int| + #[trigger] b.dom().contains(k) ==> a[k].ptr() == b[k].ptr() && a[k].is_init() && a[k].value() == b[k].value() +} + +impl AdditiveNttF64 { + /// One row group of a gathered pass: gather, transform, scatter. + /// + /// This is production's `group` closure inside `gathered_pass`, as a function: Verus does not support + /// closures that capture mutable state (the scratch buffer). Its body is the closure's, with + /// `chunks_exact(_mut)(num_ntts).enumerate()` as index loops and the permissions of each row taken + /// out for its slice and put back (ghost). Streaming stores are not modeled: `Stream::copy` copies + /// `src` over `dst` as `copy_from_slice` does (`primitives::stream`), so both arms are + /// `copy_from_slice` here. + #[allow(clippy::too_many_arguments)] + fn group( + &self, + scratch: &mut [F64], + base: SendPtr, + msg: Option>, + log_d: usize, + num_ntts: usize, + layer: usize, + g: usize, + log_step: usize, + block: usize, + r: usize, + Tracked(own): Tracked<&mut Map>>, + Tracked(mperms): Tracked<&Map>>, + Ghost(in_place): Ghost, + Ghost(src): Ghost>, + ) + requires + self.well_formed(), + num_ntts > 0, + 0 < g, + layer + g <= log_d <= self.table().len(), + log_step == log_d - layer - g, + block < pow2(layer as nat), + r < pow2(log_step as nat), + old(scratch)@.len() == pow2(g as nat) * num_ntts, + num_ntts * pow2(log_d as nat) <= usize::MAX, + base.0@.addr + num_ntts * pow2(log_d as nat) * size_of::() <= usize::MAX, + forall|q: int| + 0 <= q < pow2(g as nat) * num_ntts ==> old(own).dom().contains( + #[trigger] gidx(num_ntts as nat, block * pow2((log_d - layer) as nat), r as int, pow2(log_step as nat) as int, q), + ), + forall|k: int| #[trigger] old(own).dom().contains(k) ==> old(own)[k].ptr() == ptr_at(base.0, k) && old(own)[k].is_init(), + group_src( + *old(own), + *mperms, + base.0, + msg, + in_place, + num_ntts as nat, + block * pow2((log_d - layer) as nat), + r as int, + pow2(log_step as nat) as int, + pow2(g as nat), + src, + ), + ensures + final(own).dom() == old(own).dom(), + forall|k: int| #[trigger] old(own).dom().contains(k) ==> final(own)[k].ptr() == old(own)[k].ptr() && final(own)[k].is_init(), + forall|q: int| + 0 <= q < pow2(g as nat) * num_ntts ==> #[trigger] final(own)[gidx( + num_ntts as nat, + block * pow2((log_d - layer) as nat), + r as int, + pow2(log_step as nat) as int, + q, + )].value() == sub_layers( + self.table(), + src, + num_ntts as nat, + (layer + g) as nat, + layer as nat, + block as int, + layer as nat, + (layer + g) as nat, + )[q], + forall|k: int| + #[trigger] old(own).dom().contains(k) && !in_gather( + k, + num_ntts as nat, + block * pow2((log_d - layer) as nat), + r as int, + pow2(log_step as nat) as int, + pow2(g as nat), + ) ==> final(own)[k] == old(own)[k], + { + let ghost m = num_ntts as nat; + let ghost stp = pow2(log_step as nat) as int; + let ghost rws = pow2(g as nat); + let ghost bs = pow2((log_d - layer) as nat) as int; + let ghost brow = block * bs; + let ghost own0 = *own; + proof { + lemma_pow2_adds(g as nat, log_step as nat); + assert(g as nat + log_step as nat == (log_d - layer) as nat); + lemma_pow2_adds(layer as nat, (log_d - layer) as nat); + assert(layer as nat + (log_d - layer) as nat == log_d as nat); + lemma_pow2_pos(log_step as nat); + lemma_pow2_pos(g as nat); + lemma_usize_pow2_no_overflow(log_step as nat); + lemma_usize_pow2_no_overflow((log_d - layer) as nat); + lemma_usize_shl_is_mul(1, log_step); + assert(brow + bs <= pow2(log_d as nat)) by (nonlinear_arith) + requires + block + 1 <= pow2(layer as nat), + brow == block * bs, + pow2(log_d as nat) == pow2(layer as nat) * bs, + ; + assert(brow >= 0) by (nonlinear_arith) + requires + brow == block * bs, + block >= 0, + bs > 0, + ; + assert((brow + bs) * m <= num_ntts * pow2(log_d as nat)) by (nonlinear_arith) + requires + brow + bs <= pow2(log_d as nat), + m == num_ntts, + ; + assert(bs == rws * stp); + assert(pow2(log_d as nat) <= num_ntts * pow2(log_d as nat)) by (nonlinear_arith) + requires + num_ntts > 0, + ; + lemma_usize_shl_is_mul(block, (log_d - layer) as usize); + assert(brow * m <= (brow + bs) * m) by (nonlinear_arith) + requires + bs > 0, + ; + assert(rws * m <= (brow + bs) * m) by (nonlinear_arith) + requires + bs == rws * stp, + stp >= 1, + rws > 0, + brow >= 0, + ; + } + // Row `i` of the group, as a word offset inside its block. + let row = |i: usize| -> (o: usize) + requires + i < rws, + r < stp, + stp == 1usize << log_step, + (r + i * stp) * m <= usize::MAX, + log_step < 64, + num_ntts > 0, + m == num_ntts, + ensures + o == (r + i * stp) * m, + { + proof { + assert(i * stp >= 0) by (nonlinear_arith) + requires + stp > 0, + ; + assert(r + i * stp <= (r + i * stp) * m) by (nonlinear_arith) + requires + m >= 1, + r >= 0, + i * stp >= 0, + ; + lemma_usize_shl_is_mul(i, log_step); + } + (r + (i << log_step)) * num_ntts + }; + let block_off = (block << (log_d - layer)) * num_ntts; + proof { + assert(0 * m == 0); + assert forall|i: usize| i < rws implies #[trigger] row.requires((i,)) by { + lemma_gather_row(m, 0, r as int, stp, rws, i as int); + assert((r + i * stp) * m + m <= bs * m) by (nonlinear_arith) + requires + (0 + r + i * stp) * m + m <= (0 + rws * stp) * m, + bs == rws * stp, + ; + assert(bs * m <= (brow + bs) * m) by (nonlinear_arith) + requires + brow >= 0, + m > 0, + ; + } + } + // Gather: the scattered rows become one contiguous 2^g-row buffer. + // + // Rewritten from `for (i, dst) in scratch.chunks_exact_mut(num_ntts).enumerate()`. + let rows = 1usize << g; + proof { + lemma_usize_pow2_no_overflow(g as nat); + lemma_usize_shl_is_mul(1, g); + } + for i in 0..rows + invariant + // ... + rows == rws, + m == num_ntts, + num_ntts > 0, + stp > 0, + 0 <= r < stp, + bs == rws * stp, + brow == block * bs, + brow >= 0, + block_off == brow * m, + (brow + bs) * m <= num_ntts * pow2(log_d as nat), + num_ntts * pow2(log_d as nat) <= usize::MAX, + base.0@.addr + num_ntts * pow2(log_d as nat) * size_of::() <= usize::MAX, + scratch@.len() == rws * m, + rws * m <= usize::MAX, + same_vals(*own, own0), + forall|k: int| #[trigger] own0.dom().contains(k) && !in_gather(k, m, brow, r as int, stp, rws) ==> own[k] == own0[k], + forall|q: int| 0 <= q < rws * num_ntts ==> own0.dom().contains(#[trigger] gidx(m, brow, r as int, stp, q)), + forall|k: int| #[trigger] own0.dom().contains(k) ==> own0[k].ptr() == ptr_at(base.0, k) && own0[k].is_init(), + group_src(own0, *mperms, base.0, msg, in_place, m, brow, r as int, stp, rws, src), + forall|q: int| 0 <= q < i * m ==> #[trigger] scratch@[q] == src[q], + forall|i: usize| i < rws ==> #[trigger] row.requires((i,)), + forall|i: usize, o: usize| #[trigger] row.ensures((i,), o) ==> o == (r + i * stp) * m, + { + proof { + lemma_gather_row(m, brow, r as int, stp, rws, i as int); + lemma_gather_row(m, 0, r as int, stp, rws, i as int); + assert((i + 1) * m <= rws * m) by (nonlinear_arith) + requires + i + 1 <= rws, + ; + assert((i + 1) * m == i * m + m) by (nonlinear_arith); + assert((r + i * stp) * m + m <= bs * m) by (nonlinear_arith) + requires + (0 + r + i * stp) * m + m <= (0 + rws * stp) * m, + bs == rws * stp, + ; + assert(bs * m <= (brow + bs) * m) by (nonlinear_arith) + requires + brow >= 0, + m > 0, + ; + assert(block_off + (r + i * stp) * m == (brow + r + i * stp) * m) by (nonlinear_arith) + requires + block_off == brow * m, + ; + } + let dst = &mut scratch[i * num_ntts..(i + 1) * num_ntts]; + let ghost lo = (brow + r + i * stp) * m; + proof { + lemma_mul_le(lo + m, num_ntts * pow2(log_d as nat), size_of::() as int); + lemma_row_keys_in(own0, m, brow, r as int, stp, rws, i as int); + } + let ghost lo0 = (r + i * stp) * m; + let ghost cur = *own; + proof { + lemma_gather_row(m, 0, r as int, stp, rws, i as int); + lemma_mul_le(lo0 + m, num_ntts * pow2(log_d as nat), size_of::() as int); + lemma_mul_le(lo0, num_ntts * pow2(log_d as nat), size_of::() as int); + assert(lo0 + m <= bs * m) by (nonlinear_arith) + requires + (0 + r + i * stp) * m + m <= (0 + rws * stp) * m, + lo0 == (r + i * stp) * m, + bs == rws * stp, + ; + if let Some(mp) = msg { + assert forall|k: int| lo0 <= k < lo0 + m implies { + if in_place { + &&& #[trigger] own.dom().contains(k) + &&& own[k].ptr() == ptr_at(mp.0, k) + &&& own[k].is_init() + &&& own[k].value() == src[i * m + (k - lo0)] + } else { + mperms[k].value() == src[i * m + (k - lo0)] + } + } by { + let lane = k - lo0; + assert(gidx(m, 0, r as int, stp, i * m + lane) == k); + assert(i * m + lane < rws * m) by (nonlinear_arith) + requires + i + 1 <= rws, + 0 <= lane < m, + ; + assert(i * m + lane >= 0) by (nonlinear_arith) + requires + lane >= 0, + ; + if in_place { + assert(own0.dom().contains(k)); + } + } + assert(lo0 >= 0) by (nonlinear_arith) + requires r >= 0, i >= 0, stp > 0, m > 0, lo0 == (r + i * stp) * m; + if !in_place { + lemma_mul_le(lo0 + m, rws * stp * m, size_of::() as int); + lemma_mul_le(lo0, rws * stp * m, size_of::() as int); + assert(covers(*mperms, mp.0, lo0, m as int)); + } else { + assert(mp.0 == base.0); + assert forall|k: int| lo0 <= k < lo0 + m implies + #[trigger] own.dom().contains(k) && own[k].ptr() == ptr_at(mp.0, k) && own[k].is_init() by { + let lane = k - lo0; + let q = i * m + lane; + lemma_word(i as int, lane, m as int); + assert(0 <= q < rws * m) by (nonlinear_arith) + requires q == i * m + lane, 0 <= lane < m, 0 <= i < rws, m > 0; + assert(gidx(m, 0, r as int, stp, q) == k); + assert(own0.dom().contains(gidx(m, 0, r as int, stp, q))); + } + assert(m as int >= 0); + lemma_mul_le(lo0 + m, num_ntts * pow2(log_d as nat), size_of::() as int); + assert(mp.0@.addr + (lo0 + m) * size_of::() <= usize::MAX); + assert(covers(*own, mp.0, lo0, m as int)); + } + } + } + let tracked mut rowp = Map::>::tracked_empty(); + proof { + if msg is None { + rowp = take_row(own, base.0, lo, m); + } + } + // SAFETY: + // - The message and the codeword both cover every row addressed here. + // - Tasks own disjoint residues, so no other task writes these rows. + // - Block 0 is written last, after every read of the message in it. + let src_row = unsafe { + match msg { + Some(mp) => parallel::from_raw_parts( + mp.add(row(i)), + num_ntts, + Ghost(mp.0), + Ghost(lo0), + Tracked( + if in_place { + &*own + } else { + mperms + }, + ), + ), + None => base.slice(block_off + row(i), num_ntts, Tracked(&mut rowp)), + } + }; + dst.copy_from_slice(src_row); + proof { + if msg is None { + let rp = rowp; + own.tracked_union_prefer_right(rowp); + assert(own.dom() =~= own0.dom()); + assert forall|k: int| #[trigger] own0.dom().contains(k) implies own[k].ptr() == own0[k].ptr() && own[k].is_init() + && own[k].value() == own0[k].value() by { + if lo <= k < lo + m { + assert(own[k] == rp[k]); + assert(vals(rp, lo, m as int)[k - lo] == rp[k].value()); + } + } + } + assert(same_vals(*own, own0)); + assert forall|k: int| #[trigger] own0.dom().contains(k) && !in_gather(k, m, brow, r as int, stp, rws) implies own[k] == own0[k] by { + if lo <= k < lo + m { + lemma_row_in_gather(k, m, brow, r as int, stp, rws, i as int); + } + } + assert forall|q: int| 0 <= q < (i + 1) * m implies #[trigger] scratch@[q] == src[q] by { + if q >= i * m { + lemma_word(i as int, q - i * m, m as int); + assert(gidx(m, brow, r as int, stp, i * m + (q - i * m)) == lo + (q - i * m)); + assert(gidx(m, 0, r as int, stp, i * m + (q - i * m)) == lo0 + (q - i * m)); + } + } + } + } + // Transform: a (layer + g)-layer domain whose sub-block index is the global block. + proof { + assert(scratch@ =~= src); + lemma_pow2_adds(layer as nat, g as nat); + assert(pow2(((layer + g) - layer) as nat) == rws); + } + self.run_layers(scratch, layer + g, num_ntts, layer, layer + g, layer, block); + let ghost out = scratch@; + proof { + lemma_sub_layers_len(self.table(), src, m, (layer + g) as nat, layer as nat, block as int, layer as nat, (layer + g) as nat); + } + // Scatter: every row returns to its place. + // + // Rewritten from `for (i, src) in scratch.chunks_exact(num_ntts).enumerate()`; both arms of the + // `stream` match are `copy_from_slice` (see above). + for i in 0..rows + invariant + rows == rws, + m == num_ntts, + num_ntts > 0, + stp > 0, + 0 <= r < stp, + bs == rws * stp, + brow == block * bs, + brow >= 0, + block_off == brow * m, + (brow + bs) * m <= num_ntts * pow2(log_d as nat), + num_ntts * pow2(log_d as nat) <= usize::MAX, + base.0@.addr + num_ntts * pow2(log_d as nat) * size_of::() <= usize::MAX, + scratch@ == out, + out.len() == rws * m, + rws * m <= usize::MAX, + own.dom() == own0.dom(), + forall|q: int| 0 <= q < rws * num_ntts ==> own0.dom().contains(#[trigger] gidx(m, brow, r as int, stp, q)), + forall|k: int| #[trigger] own0.dom().contains(k) ==> own0[k].ptr() == ptr_at(base.0, k) && own0[k].is_init(), + forall|k: int| #[trigger] own0.dom().contains(k) ==> own[k].ptr() == ptr_at(base.0, k) && own[k].is_init(), + forall|q: int| 0 <= q < i * m ==> #[trigger] own[gidx(m, brow, r as int, stp, q)].value() == out[q], + forall|k: int| #[trigger] own0.dom().contains(k) && !in_gather(k, m, brow, r as int, stp, rws) ==> own[k] == own0[k], + forall|i: usize| i < rws ==> #[trigger] row.requires((i,)), + forall|i: usize, o: usize| #[trigger] row.ensures((i,), o) ==> o == (r + i * stp) * m, + { + proof { + lemma_gather_row(m, brow, r as int, stp, rws, i as int); + lemma_gather_row(m, 0, r as int, stp, rws, i as int); + assert((i + 1) * m <= rws * m) by (nonlinear_arith) + requires + i + 1 <= rws, + ; + assert((i + 1) * m == i * m + m) by (nonlinear_arith); + assert((r + i * stp) * m + m <= bs * m) by (nonlinear_arith) + requires + (0 + r + i * stp) * m + m <= (0 + rws * stp) * m, + bs == rws * stp, + ; + assert(bs * m <= (brow + bs) * m) by (nonlinear_arith) + requires + brow >= 0, + m > 0, + ; + assert(block_off + (r + i * stp) * m == (brow + r + i * stp) * m) by (nonlinear_arith) + requires + block_off == brow * m, + ; + } + let src_row = &scratch[i * num_ntts..(i + 1) * num_ntts]; + let ghost lo = (brow + r + i * stp) * m; + proof { + lemma_mul_le(lo + m, num_ntts * pow2(log_d as nat), size_of::() as int); + lemma_row_keys_in(own0, m, brow, r as int, stp, rws, i as int); + } + let ghost cur = *own; + let tracked mut rowp = take_row(own, base.0, lo, m); + // SAFETY: this group alone owns these rows of the codeword. + let dst = unsafe { base.slice(block_off + row(i), num_ntts, Tracked(&mut rowp)) }; + dst.copy_from_slice(src_row); + proof { + own.tracked_union_prefer_right(rowp); + assert(own.dom() =~= own0.dom()); + let rp = rowp; + assert forall|q: int| 0 <= q < (i + 1) * m implies #[trigger] own[gidx(m, brow, r as int, stp, q)].value() == out[q] by { + if q >= i * m { + let lane = q - i * m; + lemma_word(i as int, lane, m as int); + assert(gidx(m, brow, r as int, stp, i * m + lane) == lo + lane); + assert(own[lo + lane] == rp[lo + lane]); + assert(vals(rp, lo, m as int)[lane] == rp[lo + lane].value()); + assert(out.subrange(i * m, (i + 1) * m)[lane] == out[q]); + } else { + lemma_split_word(q, m as int); + let iq = q / (m as int); + assert(iq < i) by (nonlinear_arith) + requires + q == iq * m + q % (m as int), + q % (m as int) >= 0, + q < i * m, + m > 0, + ; + lemma_gather_row(m, brow, r as int, stp, rws, iq); + lemma_word(iq, q % (m as int), m as int); + assert(gidx(m, brow, r as int, stp, q) < lo) by (nonlinear_arith) + requires + gidx(m, brow, r as int, stp, q) == (brow + r + iq * stp) * m + q % (m as int), + q % (m as int) < m, + lo == (brow + r + i * stp) * m, + iq + 1 <= i, + stp > 0, + ; + } + } + assert forall|k: int| #[trigger] own0.dom().contains(k) && !in_gather(k, m, brow, r as int, stp, rws) implies own[k] == own0[k] by { + if lo <= k < lo + m { + lemma_row_in_gather(k, m, brow, r as int, stp, rws, i as int); + } + } + } + } + proof { + let s_out = sub_layers(self.table(), src, m, (layer + g) as nat, layer as nat, block as int, layer as nat, (layer + g) as nat); + assert(out == s_out); + assert forall|k: int| + #[trigger] own0.dom().contains(k) && !in_gather(k, m, brow, r as int, stp, rws) implies own[k] == own0[k] by {} + assert(rws * num_ntts == rws * m); + } + } +} + +/// A gathered task owns one residue in one block. +pub open spec fn gather_owner(m: int, bs: int, step: int, k: int) -> int { + (k / m / bs) * step + (k / m) % step +} + +proof fn lemma_gather_partition(m: int, bs: int, step: int, rows: int, blocks: int, k: int) + requires m > 0, step > 0, rows > 0, blocks > 0, bs == rows * step, + 0 <= k < blocks * bs * m, + ensures + 0 <= k / m / bs < blocks, + 0 <= (k / m) % step < step, + 0 <= gather_owner(m, bs, step, k) < blocks * step, + in_gather(k, m as nat, (k / m / bs) * bs, (k / m) % step, step, rows as nat), +{ + assert(bs > 0) by (nonlinear_arith) requires bs == rows * step, rows > 0, step > 0; + lemma_split_word(k, m); + let row = k / m; + lemma_fundamental_div_mod(row, bs); + lemma_mod_bound(row, bs); + lemma_div_pos_is_pos(row, bs); + lemma_mod_bound(row, step); + assert(0 <= row < blocks * bs) by (nonlinear_arith) + requires k == row * m + k % m, 0 <= k % m < m, + 0 <= k < blocks * bs * m, m > 0; + assert(row / bs < blocks) by (nonlinear_arith) + requires row == bs * (row / bs) + row % bs, 0 <= row % bs, + row < blocks * bs, bs > 0; + lemma_mod_multiples_basic((row / bs) * rows, step); + assert((row / bs) * bs == ((row / bs) * rows) * step) by (nonlinear_arith) + requires bs == rows * step; + lemma_mod_multiples_vanish(-((row / bs) * rows), row, step); + assert(step * -((row / bs) * rows) + row == row - (row / bs) * bs) by (nonlinear_arith) + requires bs == rows * step; + assert(0 <= gather_owner(m, bs, step, k) < blocks * step) by (nonlinear_arith) + requires 0 <= row / bs < blocks, 0 <= row % step < step, step > 0, + gather_owner(m, bs, step, k) == (row / bs) * step + row % step; +} + +proof fn lemma_gather_owner(m: nat, bs: int, step: int, rows: nat, block: int, r: int, q: int) + requires m > 0, step > 0, rows > 0, bs == rows * step, + block >= 0, 0 <= r < step, 0 <= q < rows * m, + ensures + 0 <= gidx(m, block * bs, r, step, q) < (block + 1) * bs * m, + gather_owner(m as int, bs, step, gidx(m, block * bs, r, step, q)) == block * step + r, +{ + lemma_split_word(q, m as int); + let i = q / (m as int); + let lane = q % (m as int); + assert(i < rows) by (nonlinear_arith) + requires q == i * m + lane, lane >= 0, q < rows * m, m > 0; + lemma_gather_row(m, block * bs, r, step, rows, i); + assert(0 <= r + i * step < bs) by (nonlinear_arith) + requires 0 <= i < rows, 0 <= r < step, bs == rows * step, step > 0; + lemma_word(block * bs + r + i * step, lane, m as int); + lemma_fundamental_div_mod_converse(block * bs + r + i * step, bs, block, r + i * step); + assert(block * bs + r + i * step == (block * rows + i) * step + r) by (nonlinear_arith) + requires bs == rows * step; + lemma_fundamental_div_mod_converse(block * bs + r + i * step, step, block * rows + i, r); + assert((block * bs + rows * step) * m == (block + 1) * bs * m) by (nonlinear_arith) + requires bs == rows * step; +} + +impl AdditiveNttF64 { + /// Gather, run and scatter a band of layers after its input replicas exist. + /// + /// Executable refinement of production's no-message pass: `for_each` replaces + /// the chunk loop and borrows scratch once per group rather than once per claimed + /// range. The dispatcher still runs the real pool. Streaming stores are replaced + /// by ordinary copies. These changes do not verify production's cache policy. + pub fn gathered_pass( + &self, data: &mut [F64], log_d: usize, num_ntts: usize, layer: usize, g: usize, + ) + requires self.well_formed(), num_ntts > 0, + 0 < g, layer + g <= log_d <= self.table().len(), + old(data)@.len() == num_ntts * pow2(log_d as nat), + old(data)@.len() <= usize::MAX, + ensures final(data)@ == sub_layers(self.table(), old(data)@, num_ntts as nat, + log_d as nat, 0, 0, layer as nat, (layer + g) as nat), + { + let ghost x = data@; + let ghost m = num_ntts as nat; + let ghost target = sub_layers(self.table(), x, m, log_d as nat, 0, 0, layer as nat, (layer + g) as nat); + let log_step = log_d - layer - g; + proof { + lemma_usize_pow2_no_overflow(g as nat); + lemma_usize_pow2_no_overflow(log_step as nat); + lemma_usize_pow2_no_overflow(layer as nat); + lemma_usize_pow2_no_overflow((log_d - layer) as nat); + lemma_usize_shl_is_mul(1, g); + lemma_usize_shl_is_mul(1, log_step); + lemma_usize_shl_is_mul(1, layer); + lemma_usize_shl_is_mul(1, (log_d - layer) as usize); + lemma_pow2_adds(g as nat, log_step as nat); + lemma_pow2_adds(layer as nat, (log_d - layer) as nat); + lemma_pow2_pos(g as nat); + lemma_pow2_pos(log_step as nat); + lemma_pow2_pos(layer as nat); + lemma_sub_layers_len(self.table(), x, m, log_d as nat, 0, 0, layer as nat, (layer + g) as nat); + } + let rows = 1usize << g; + let step = 1usize << log_step; + let blocks = 1usize << layer; + let bs = 1usize << (log_d - layer); + proof { + assert(bs == rows * step); + assert(x.len() == blocks * bs * m) by (nonlinear_arith) + requires x.len() == m * pow2(log_d as nat), + pow2(log_d as nat) == blocks * bs; + assert(blocks * step <= x.len() && rows * m <= x.len()) by (nonlinear_arith) + requires x.len() == blocks * bs * m, bs == rows * step, + blocks > 0, rows > 0, step > 0, m > 0; + } + let n_tasks = blocks * step; + let scratch_len = rows * num_ntts; + let (ptr, Tracked(perms)) = slice_as_mut_ptr(data); + let base = SendPtr(ptr); + let ghost p0 = *perms; + let ghost owner = |k: int| gather_owner(m as int, bs as int, step as int, k); + let ghost post = |t: int, p: Map>| group_post(p, claimed(p0, owner, t, t + 1), target); + proof { + assert forall|k: int| #[trigger] p0.dom().contains(k) implies 0 <= owner(k) < n_tasks by { + lemma_gather_partition(m as int, bs as int, step as int, rows as int, blocks as int, k); + } + } + let tracked all = perms.tracked_remove_keys(perms.dom()); + proof { assert(all =~= p0); } + let Tracked(out) = parallel::for_each(n_tasks, + |t: usize, tp: Tracked>>| -> (res: Tracked>>) + requires t < n_tasks, tp@ == claimed(p0, owner, t as int, t + 1), + ensures res@.dom() == keys_of(p0, owner, t as int, t + 1), post(t as int, res@), + { + let block = t / step; + let r = t % step; + let ghost src = gather(x, m, block * bs, r as int, step as int, rows as nat); + let tracked mut own = tp.get(); + let ghost before = own; + proof { + lemma_split_word(t as int, step as int); + assert(block < blocks) by (nonlinear_arith) + requires t == block * step + r, r >= 0, t < blocks * step, step > 0; + assert forall|q: int| 0 <= q < rows * m implies own.dom().contains( + #[trigger] gidx(m, block * bs, r as int, step as int, q)) by { + lemma_gather_owner(m, bs as int, step as int, rows as nat, block as int, r as int, q); + assert((block + 1) * bs * m <= x.len()) by (nonlinear_arith) + requires block + 1 <= blocks, x.len() == blocks * bs * m, bs > 0, m > 0; + } + assert((block * bs + rows * step) * m <= x.len()) by (nonlinear_arith) + requires block + 1 <= blocks, x.len() == blocks * bs * m, + bs == rows * step, bs > 0, m > 0; + lemma_mul_le((block * bs + rows * step) * m, x.len() as int, size_of::() as int); + assert forall|q: int| 0 <= q < rows * m implies + #[trigger] own[gidx(m, block * bs, r as int, step as int, q)].value() == src[q] by { + let k = gidx(m, block * bs, r as int, step as int, q); + assert(own.dom().contains(k)); + assert(p0[k].value() == vals(p0, 0, x.len() as int)[k]); + } + assert(group_src(own, Map::empty(), base.0, None, false, m, block * bs, + r as int, step as int, rows as nat, src)); + } + let Tracked(result) = with_scratch(scratch_len, Tracked(own), + |scratch: &mut [F64], tp: Tracked>>| -> (res: Tracked>>) + requires scratch@.len() == scratch_len, tp@ == before, + ensures res@.dom() == before.dom(), post(t as int, res@), + { + let tracked mut p = tp.get(); + let tracked empty = Map::tracked_empty(); + self.group(scratch, base, None, log_d, num_ntts, layer, g, log_step, + block, r, Tracked(&mut p), Tracked(&empty), Ghost(false), Ghost(src)); + proof { + lemma_gather_sub_layers(self.table(), x, m, log_d as nat, 0, 0, + (layer + g) as nat, layer as nat, block as int, r as int, + layer as nat, (layer + g) as nat); + assert forall|k: int| #[trigger] before.dom().contains(k) implies + p[k].ptr() == before[k].ptr() && p[k].is_init() && p[k].value() == target[k] by { + lemma_gather_partition(m as int, bs as int, step as int, rows as int, blocks as int, k); + let b = k / (m as int) / (bs as int); + let rr = (k / (m as int)) % (step as int); + lemma_fundamental_div_mod_converse(t as int, step as int, b, rr); + assert(b == block && rr == r); + lemma_in_gather_index(k, m, block * bs, r as int, step as int, rows as nat); + let q = ((k / (m as int) - block * bs) / (step as int)) * m + k % (m as int); + assert(gather(target, m, block * bs, r as int, step as int, rows as nat)[q] == target[k]); + } + } + Tracked(p) + }); + proof { + assert(post(t as int, result)); + assert(result.dom() =~= keys_of(p0, owner, t as int, t + 1)); + } + Tracked(result) + }, Tracked(all), Ghost(owner), Ghost(post)); + proof { + perms.tracked_union_prefer_right(out); + assert(perms.dom() =~= p0.dom()); + assert forall|k: int| 0 <= k < x.len() implies + (#[trigger] perms[k]).ptr() == ptr_at(ptr, k) && perms[k].is_init() && perms[k].value() == target[k] by { + lemma_gather_partition(m as int, bs as int, step as int, rows as int, blocks as int, k); + let t = owner(k); + let p = out.restrict(keys_of(p0, owner, t, t + 1)); + assert(post(t, p)); + let orig = claimed(p0, owner, t, t + 1); + assert(orig.dom().contains(k)); + assert(orig[k] == p0[k]); + assert(p0[k].ptr() == ptr_at(ptr, k)); + assert(perms[k] == out[k]); + assert(p[k] == out[k]); + } + assert(owns(*perms, ptr, 0, x.len() as int)); + assert(vals(*perms, 0, x.len() as int) =~= target); + } + } + /// Contiguous deep pass, one pool item per sub-block. Production batches + /// adjacent sub-blocks; here the pool's claim coalescing is the only batching. + pub fn deep_pass(&self, data: &mut [F64], log_d: usize, num_ntts: usize, first: usize) + requires self.well_formed(), num_ntts > 0, first <= log_d <= self.table().len(), + old(data)@.len() == num_ntts * pow2(log_d as nat), old(data)@.len() <= usize::MAX, + ensures final(data)@ == sub_layers(self.table(), old(data)@, num_ntts as nat, + log_d as nat, 0, 0, first as nat, log_d as nat), + { + let ghost x = data@; + let ghost m = num_ntts as nat; + let ghost target = sub_layers(self.table(), x, m, log_d as nat, 0, 0, first as nat, log_d as nat); + proof { + lemma_usize_pow2_no_overflow((log_d - first) as nat); + lemma_usize_shl_is_mul(1, (log_d - first) as usize); + lemma_pow2_pos((log_d - first) as nat); + lemma_pow2_pos(first as nat); + lemma_pow2_adds(first as nat, (log_d - first) as nat); + lemma_sub_layers_len(self.table(), x, m, log_d as nat, 0, 0, first as nat, log_d as nat); + assert(pow2((log_d - first) as nat) * m <= x.len()) by (nonlinear_arith) + requires x.len() == m * (pow2(first as nat) * pow2((log_d - first) as nat)), + pow2(first as nat) >= 1, m > 0, pow2((log_d - first) as nat) > 0; + } + let chunk = (1usize << (log_d - first)) * num_ntts; + let ghost count = pow2(first as nat) as int; + proof { + assert(chunk > 0) by (nonlinear_arith) + requires chunk == pow2((log_d - first) as nat) * m, + pow2((log_d - first) as nat) > 0, m > 0; + assert(x.len() == count * chunk) by (nonlinear_arith) + requires x.len() == m * (pow2(first as nat) * pow2((log_d - first) as nat)), + chunk == pow2((log_d - first) as nat) * m, count == pow2(first as nat); + lemma_div_multiples_vanish_fancy(count, chunk - 1, chunk as int); + assert((x.len() + chunk - 1) / (chunk as int) == count); + assert forall|i: int| 0 <= i < count implies + #[trigger] chunk_len(x.len(), chunk as nat, i) == chunk by { + assert((i + 1) * chunk <= x.len()) by (nonlinear_arith) + requires i + 1 <= count, x.len() == count * chunk, chunk > 0; + assert(x.len() - i * chunk >= chunk) by (nonlinear_arith) + requires (i + 1) * chunk <= x.len(); + assert(i * chunk + chunk == (i + 1) * chunk) by (nonlinear_arith); + } + } + let ghost post = |i: int, before: Seq, after: Seq| + after == target.subrange(i * chunk, (i + 1) * chunk); + parallel::chunks_mut(data, chunk, + |i: usize, sub: &mut [F64]| + requires i < count, sub@ == x.subrange(i * chunk, i * chunk + chunk_len(x.len(), chunk as nat, i as int)), + ensures final(sub)@.len() == old(sub)@.len(), post(i as int, old(sub)@, final(sub)@), + { + let ghost before = sub@; + proof { + assert(chunk_len(x.len(), chunk as nat, i as int) == chunk); + assert(i * chunk + chunk == (i + 1) * chunk) by (nonlinear_arith); + lemma_block_target(self.table(), x, m, log_d as nat, 0, 0, first as nat, + (log_d - first) as nat, i as int); + } + self.run_layers(sub, log_d, num_ntts, first, log_d, first, i); + proof { + lemma_sub_layers_len(self.table(), before, m, log_d as nat, first as nat, + i as int, first as nat, log_d as nat); + } + }, Ghost(post)); + proof { + assert(data@ =~= target) by { + assert forall|k: int| 0 <= k < x.len() implies data@[k] == target[k] by { + lemma_split_word(k, chunk as int); + let i = k / (chunk as int); + assert(i < count) by (nonlinear_arith) + requires k == i * chunk + k % (chunk as int), k % (chunk as int) >= 0, + k < x.len(), x.len() == count * chunk, chunk > 0; + assert(chunk_len(x.len(), chunk as nat, i) == chunk); + assert(i * chunk + chunk == (i + 1) * chunk) by (nonlinear_arith); + assert(post(i, x.subrange(i * chunk, (i + 1) * chunk), + data@.subrange(i * chunk, (i + 1) * chunk))); + assert(data@.subrange(i * chunk, (i + 1) * chunk)[k - i * chunk] == data@[k]); + } + } + } + } + + /// Execute any valid gathered/deep plan on populated replicas. The cache + /// planner is an explicit input. Message replication, row sinks and streaming + /// store fences are not performed by this entry point. + pub fn transform(&self, data: &mut [F64], log_d: usize, num_ntts: usize, + start: usize, deep_start: usize, gathered_width: usize) + requires self.well_formed(), num_ntts > 0, gathered_width > 0, + start <= deep_start <= log_d <= self.table().len(), + old(data)@.len() == num_ntts * pow2(log_d as nat), old(data)@.len() <= usize::MAX, + ensures final(data)@ == forward_layers(self.table(), old(data)@, num_ntts as nat, + log_d as nat, start as nat, log_d as nat), + { + let ghost x = data@; + let ghost m = num_ntts as nat; + let mut layer = start; + while layer < deep_start + invariant self.well_formed(), num_ntts > 0, gathered_width > 0, + start <= layer <= deep_start <= log_d <= self.table().len(), + x.len() == num_ntts * pow2(log_d as nat), x.len() <= usize::MAX, + m == num_ntts, + data@ == sub_layers(self.table(), x, m, log_d as nat, 0, 0, start as nat, layer as nat), + decreases deep_start - layer, + { + proof { lemma_sub_layers_len(self.table(), x, m, log_d as nat, 0, 0, start as nat, layer as nat); } + let g = if deep_start - layer < gathered_width { deep_start - layer } else { gathered_width }; + self.gathered_pass(data, log_d, num_ntts, layer, g); + proof { + lemma_sub_layers_split(self.table(), x, m, log_d as nat, 0, 0, + start as nat, layer as nat, (layer + g) as nat); + } + layer += g; + } + proof { lemma_sub_layers_len(self.table(), x, m, log_d as nat, 0, 0, start as nat, deep_start as nat); } + self.deep_pass(data, log_d, num_ntts, deep_start); + proof { + lemma_sub_layers_split(self.table(), x, m, log_d as nat, 0, 0, + start as nat, deep_start as nat, log_d as nat); + lemma_forward_is_sub(self.table(), x, m, log_d as nat, start as nat, log_d as nat); + } + } +} + +} // verus! diff --git a/verification/verus/src/ntt_lanes.rs b/verification/verus/src/ntt_lanes.rs new file mode 100644 index 0000000000..39cf305ce9 --- /dev/null +++ b/verification/verus/src/ntt_lanes.rs @@ -0,0 +1,323 @@ +//! The lane-interleaved additive NTT transforms every lane independently. +//! +//! A buffer of `m` interleaved lanes holds lane `l`'s word `v` at word `m v + l` ([`lane`]). The layers of +//! `crate::ntt` act on such a buffer row by row, the same butterfly on every lane of a row pair; this module +//! proves that the lanes do not mix, so the single-lane theorems of `crate::ntt` hold lane by lane. +//! +//! Main results: [`lemma_lane_layer`] (one layer commutes with taking a lane), [`lemma_lane_forward_layers`] +//! and [`lemma_lane_inverse_layers`] (so do the forward and inverse transforms), and for +//! `AdditiveNttF64::standard(dim)` the evaluation theorems [`lemma_standard_lanes_forward_evaluates`] +//! (output word `m v + l` is lane `l`'s polynomial at the point `v`) and +//! [`lemma_standard_lanes_encode_evaluates`] (the encoder at rate `2^-r` Reed-Solomon encodes every lane). +//! No executable code: the copy these are about is `crate::ntt::forward_scalar_from_layer`, whose +//! postcondition is [`forward_layers`] for any lane count. +use crate::gf2_64::*; +use crate::ntt::*; +use vstd::arithmetic::div_mod::*; +use vstd::arithmetic::mul::*; +use vstd::arithmetic::power2::*; +use vstd::prelude::*; + +verus! { + +/// Lane `l` of a buffer of `m` interleaved lanes: word `v` of the lane is word `m v + l` of the buffer. +pub open spec fn lane(data: Seq, m: nat, l: nat) -> Seq { + Seq::new(data.len() / m, |v: int| data[(m as int) * v + (l as int)]) +} + +/// A lane of a buffer of `n` rows has `n` words. +pub proof fn lemma_lane_len(data: Seq, m: nat, l: nat, n: nat) + requires + m > 0, + data.len() == n * m, + ensures + lane(data, m, l).len() == n, +{ + lemma_div_by_multiple(n as int, m as int); +} + +/// Word `v` of a lane, `v = 2h b + r`, is word `(2h b + r) m + l` of the buffer, in block `b` and row `r` +/// of that block, both as a lane (`m = 1`) and in the buffer. +proof fn lemma_lane_word(v: int, m: nat, l: nat, h: nat) + requires + 0 <= v, + l < m, + h > 0, + ensures + ({ + let (b, r) = (v / ((2 * h) as int), v % ((2 * h) as int)); + let p = (m as int) * v + (l as int); + &&& 0 <= b + &&& 0 <= r < 2 * h + &&& v == (b * (2 * h) + r) * 1 + 0 + &&& p == (b * (2 * h) + r) * m + l + &&& blk_of(v, 1, h) == b && r_of(v, 1, h) == r + &&& blk_of(p, m, h) == b && r_of(p, m, h) == r + }), +{ + let (b, r) = (v / ((2 * h) as int), v % ((2 * h) as int)); + lemma_fundamental_div_mod(v, (2 * h) as int); + lemma_mod_pos_bound(v, (2 * h) as int); + lemma_div_pos_is_pos(v, (2 * h) as int); + assert(v == (b * (2 * h) + r) * 1 + 0) by (nonlinear_arith) + requires + v == (2 * h) * b + r, + ; + assert((m as int) * v + (l as int) == (b * (2 * h) + r) * m + l) by (nonlinear_arith) + requires + v == (2 * h) * b + r, + ; + lemma_compose(b, r, 0, 1, h); + lemma_compose(b, r, l as int, m, h); +} + +/// One layer of butterflies commutes with taking a lane: lane `l` of the layer's output is the same layer +/// applied to lane `l` alone, for either direction and any twiddles. +pub proof fn lemma_lane_layer(data: Seq, m: nat, l: nat, h: nat, nb: nat, tw: spec_fn(int) -> u64, inverse: bool) + requires + l < m, + h > 0, + data.len() == nb * (2 * h) * m, + ensures + lane(layer_map(data, m, h, tw, inverse), m, l) == layer_map(lane(data, m, l), 1, h, tw, inverse), +{ + let x = lane(data, m, l); + let lhs = lane(layer_map(data, m, h, tw, inverse), m, l); + let rhs = layer_map(x, 1, h, tw, inverse); + lemma_lane_len(data, m, l, nb * (2 * h)); + lemma_lane_len(layer_map(data, m, h, tw, inverse), m, l, nb * (2 * h)); + assert forall|v: int| 0 <= v < x.len() implies lhs[v] == rhs[v] by { + lemma_lane_word(v, m, l, h); + let (b, r) = (v / ((2 * h) as int), v % ((2 * h) as int)); + let p = (m as int) * v + (l as int); + assert(v < nb * (2 * h) * 1) by (nonlinear_arith) + requires + v < nb * (2 * h), + ; + lemma_blk_bound(v, 1, h, nb); + lemma_bound(b, r, l as int, m, h, nb); + assert(nb * (2 * h) * m == (nb * (2 * h)) * m) by (nonlinear_arith); + if r < h { + lemma_bound(b, r + h, l as int, m, h, nb); + lemma_bound(b, r + h, 0, 1, h, nb); + assert(p + h * m == (m as int) * (v + h) + (l as int)) by (nonlinear_arith) + requires + p == (m as int) * v + (l as int), + ; + assert((b * (2 * h) + (r + h)) * m + l == p + h * m) by (nonlinear_arith) + requires + p == (b * (2 * h) + r) * m + l, + ; + assert((b * (2 * h) + (r + h)) * 1 + 0 == v + h * 1) by (nonlinear_arith) + requires + v == (b * (2 * h) + r) * 1 + 0, + ; + assert(x[v + h * 1] == data[p + h * m]); + } else { + assert(p - h * m == (m as int) * (v - h) + (l as int)) by (nonlinear_arith) + requires + p == (m as int) * v + (l as int), + ; + assert(0 <= v - h) by (nonlinear_arith) + requires + v == (b * (2 * h) + r) * 1 + 0, + r >= h, + b >= 0, + ; + assert(x[v - h * 1] == data[p - h * m]); + } + } + assert(lhs =~= rhs); +} + +/// The forward layers `start..end` commute with taking a lane: lane `l` of the output is the single-lane +/// transform of lane `l` of the input. +pub proof fn lemma_lane_forward_layers( + tab: Seq>, + data: Seq, + m: nat, + log_d: nat, + start: nat, + end: nat, + l: nat, +) + requires + l < m, + data.len() == m * pow2(log_d), + end <= log_d, + ensures + lane(forward_layers(tab, data, m, log_d, start, end), m, l) == forward_layers( + tab, + lane(data, m, l), + 1, + log_d, + start, + end, + ), + decreases end, +{ + if end > start { + let layer = (end - 1) as nat; + lemma_lane_forward_layers(tab, data, m, log_d, start, layer, l); + let y = forward_layers(tab, data, m, log_d, start, layer); + lemma_forward_layers_len(tab, data, m, log_d, start, layer); + lemma_layer_shape(log_d, layer, m); + lemma_lane_layer(y, m, l, layer_half(log_d, layer), pow2(layer), layer_twiddles(tab, layer), false); + } +} + +/// The inverse layers commute with taking a lane too. +pub proof fn lemma_lane_inverse_layers(tab: Seq>, data: Seq, m: nat, log_d: nat, lo: nat, l: nat) + requires + l < m, + data.len() == m * pow2(log_d), + ensures + lane(inverse_layers(tab, data, m, log_d, lo), m, l) == inverse_layers(tab, lane(data, m, l), 1, log_d, lo), + decreases log_d - lo, +{ + if lo < log_d { + lemma_lane_inverse_layers(tab, data, m, log_d, lo + 1, l); + let y = inverse_layers(tab, data, m, log_d, lo + 1); + lemma_inverse_layers_len(tab, data, m, log_d, lo + 1); + lemma_layer_shape(log_d, lo, m); + lemma_lane_layer(y, m, l, layer_half(log_d, lo), pow2(lo), layer_twiddles(tab, lo), true); + } +} + +/// Word `m v + l` of a buffer of `n` rows is word `v` of lane `l`. +proof fn lemma_lane_index(data: Seq, m: nat, l: nat, n: nat, v: int) + requires + l < m, + data.len() == n * m, + 0 <= v < n, + ensures + 0 <= (m as int) * v + (l as int) < data.len(), + lane(data, m, l)[v] == data[(m as int) * v + (l as int)], +{ + lemma_lane_len(data, m, l, n); + assert((m as int) * v + (l as int) < n * m) by (nonlinear_arith) + requires + 0 <= v < n, + l < m, + ; + lemma_mul_nonnegative(m as int, v); +} + +/// The lane-interleaved forward transform of `AdditiveNttF64::standard(dim)` evaluates every lane's +/// novel-basis polynomial on the domain `{0, .., 2^dim - 1}`: on `m` lanes of `2^dim` rows, output word +/// `m v + l` is `P_l(v) = Σ_{j < 2^dim} a_(m j + l) X_j(v)` ([`novel_sum`] of lane `l` of the input). +pub proof fn lemma_standard_lanes_forward_evaluates(tab: Seq>, dim: nat, m: nat, a: Seq) + requires + AdditiveNttF64::is_table_of(tab, standard_basis(dim)), + 1 <= dim <= 63, + m > 0, + a.len() == m * pow2(dim), + ensures + forall|v: int, l: int| + 0 <= v < pow2(dim) && 0 <= l < m ==> (#[trigger] forward_layers(tab, a, m, dim, 0, dim)[(m as int) * v + + l]).0 == novel_sum(standard_basis(dim), lane(a, m, l as nat), dim, v as u64), +{ + let y = forward_layers(tab, a, m, dim, 0, dim); + lemma_forward_layers_len(tab, a, m, dim, 0, dim); + assert(a.len() == pow2(dim) * m) by (nonlinear_arith) + requires + a.len() == m * pow2(dim), + ; + assert forall|v: int, l: int| 0 <= v < pow2(dim) && 0 <= l < m implies (#[trigger] y[(m as int) * v + l]).0 + == novel_sum(standard_basis(dim), lane(a, m, l as nat), dim, v as u64) by { + let x = lane(a, m, l as nat); + lemma_lane_len(a, m, l as nat, pow2(dim)); + lemma_lane_forward_layers(tab, a, m, dim, 0, dim, l as nat); + lemma_lane_index(y, m, l as nat, pow2(dim), v); + lemma_standard_forward_evaluates(tab, dim, x); + assert(forward_layers(tab, x, 1, dim, 0, dim)[v] == y[(m as int) * v + l]); + } +} + +/// Lane `l` of `n m` words of copies of an `m`-lane message is `n` words of copies of the message's lane `l`. +proof fn lemma_lane_replicate(msg: Seq, m: nat, l: nat, k: nat, n: nat) + requires + l < m, + k > 0, + msg.len() == m * k, + ensures + lane(replicate(msg, n * m), m, l) == replicate(lane(msg, m, l), n), +{ + let x = replicate(msg, n * m); + let lm = lane(msg, m, l); + lemma_lane_len(x, m, l, n); + assert(msg.len() == k * m) by (nonlinear_arith) + requires + msg.len() == m * k, + ; + lemma_lane_len(msg, m, l, k); + assert forall|v: int| 0 <= v < n implies #[trigger] lane(x, m, l)[v] == replicate(lm, n)[v] by { + let (q, s) = (v / (k as int), v % (k as int)); + lemma_fundamental_div_mod(v, k as int); + lemma_mod_pos_bound(v, k as int); + lemma_div_pos_is_pos(v, k as int); + let p = (m as int) * v + (l as int); + assert(p == q * (m * k) + ((m as int) * s + l)) by (nonlinear_arith) + requires + v == k * q + s, + p == (m as int) * v + (l as int), + ; + assert(0 <= (m as int) * s + l < m * k) by (nonlinear_arith) + requires + 0 <= s < k, + l < m, + ; + lemma_fundamental_div_mod_converse(p, (m * k) as int, q, (m as int) * s + l); + lemma_lane_index(x, m, l, n, v); + lemma_lane_index(msg, m, l, k, s); + } + assert(lane(x, m, l) =~= replicate(lm, n)); +} + +/// The lane-interleaved encoder of `AdditiveNttF64::standard(dim)` at rate `2^-r` Reed-Solomon encodes every +/// lane: on `2^r` copies of an `m`-lane message of `2^(dim - r)` rows, the layers `r..dim` give, at word +/// `m v + l`, `P_l(v)`, `P_l` the novel-basis polynomial whose coefficients are lane `l` of the message, +/// zero-padded (degree below `2^(dim - r)`). +pub proof fn lemma_standard_lanes_encode_evaluates(tab: Seq>, dim: nat, m: nat, msg: Seq, r: nat) + requires + AdditiveNttF64::is_table_of(tab, standard_basis(dim)), + 1 <= dim <= 63, + r <= dim, + m > 0, + msg.len() == m * pow2((dim - r) as nat), + ensures + ({ + let codeword = forward_layers(tab, replicate(msg, m * pow2(dim)), m, dim, r, dim); + &&& codeword.len() == m * pow2(dim) + &&& forall|v: int, l: int| + 0 <= v < pow2(dim) && 0 <= l < m ==> (#[trigger] codeword[(m as int) * v + l]).0 == novel_sum( + standard_basis(dim), + zero_pad(lane(msg, m, l as nat), pow2(dim)), + dim, + v as u64, + ) + }), +{ + let rep = replicate(msg, m * pow2(dim)); + let codeword = forward_layers(tab, rep, m, dim, r, dim); + lemma_forward_layers_len(tab, rep, m, dim, r, dim); + lemma_pow2_pos((dim - r) as nat); + assert(m * pow2(dim) == pow2(dim) * m) by (nonlinear_arith); + assert forall|v: int, l: int| 0 <= v < pow2(dim) && 0 <= l < m implies (#[trigger] codeword[(m as int) * v + + l]).0 == novel_sum(standard_basis(dim), zero_pad(lane(msg, m, l as nat), pow2(dim)), dim, v as u64) by { + let lm = lane(msg, m, l as nat); + assert(msg.len() == pow2((dim - r) as nat) * m) by (nonlinear_arith) + requires + msg.len() == m * pow2((dim - r) as nat), + ; + lemma_lane_len(msg, m, l as nat, pow2((dim - r) as nat)); + lemma_lane_replicate(msg, m, l as nat, pow2((dim - r) as nat), pow2(dim)); + lemma_lane_forward_layers(tab, rep, m, dim, r, dim, l as nat); + lemma_lane_index(codeword, m, l as nat, pow2(dim), v); + lemma_standard_encode_evaluates(tab, dim, lm, r); + let single = forward_layers(tab, replicate(lm, pow2(dim)), 1, dim, r, dim); + assert(single[v] == codeword[(m as int) * v + l]); + } +} + +} // verus! diff --git a/verification/verus/src/ntt_simd.rs b/verification/verus/src/ntt_simd.rs new file mode 100644 index 0000000000..b7d44bc66f --- /dev/null +++ b/verification/verus/src/ntt_simd.rs @@ -0,0 +1,1062 @@ +//! The SIMD butterfly kernels of the additive NTT, `crates/pcs/src/ntt/additive_ntt_f64.rs` +//! (`butterfly_lanes_avx512`, `butterfly_lanes_avx2`, `butterfly_lanes_neon_8`, `butterfly_lane_pair_neon`), +//! copied with production's bodies and `cfg` gates; `ntt::lane_butterflies` dispatches to them as production +//! does. +//! +//! Each kernel is proven to leave its words of the two rows as the portable path leaves them: word `j` of +//! `top`/`bot` through [`butterfly_spec`] with the twiddle (forward `u' = u + t v, v' = v + u'`, transposed +//! `u' = u + v, v' = v + t u'`, products by `k_mul`), and every other word unchanged ([`butterflied`]). This rests +//! on the intrinsic specifications of `crate::intrinsics`. +//! +//! One rewrite is common to all kernels: production takes `top: *mut F64, bot: *mut F64` and loads and stores +//! through them. Verus cannot tie a pointer to the memory it addresses unless the caller holds a permission for +//! it, and `lane_butterflies` holds `&mut [F64]` borrows, from which no permission can be obtained. So each copy +//! takes the two rows and the offset `at` its pointers point to (`top.as_mut_ptr().add(at)` in production), and +//! loads and stores through the helpers of `intrinsics::x86_nttsimd` / `intrinsics::aarch64_nttsimd`, whose +//! bodies are production's load or store at that offset. +#[cfg(verus_keep_ghost)] +use crate::clmul::clmul; +use crate::gf2_64::F64; +#[cfg(verus_keep_ghost)] +use crate::gf2_64::{k_mod, k_mul, lemma_k_mod, reduce_formula}; +#[cfg(verus_keep_ghost)] +use crate::ntt::butterfly_spec; +use vstd::prelude::*; + +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +use crate::gf2_64::aarch64::reduce_pair_pmull4; +#[cfg(all(target_arch = "aarch64", verus_keep_ghost))] +use crate::gf2_64::aarch64::u64x2_u128; +#[cfg(all(target_arch = "x86_64", verus_keep_ghost))] +use crate::gf2_64::x86_64::lemma_i64_round_trip; +#[cfg(all(target_arch = "aarch64", verus_keep_ghost))] +use crate::intrinsics::aarch64::*; +#[cfg(target_arch = "aarch64")] +use crate::intrinsics::aarch64_nttsimd::*; +#[cfg(verus_keep_ghost)] +use crate::intrinsics::transmuted; +#[cfg(all(target_arch = "x86_64", verus_keep_ghost))] +use crate::intrinsics::x86::*; +#[cfg(target_arch = "x86_64")] +use crate::intrinsics::x86_nttsimd::*; +#[cfg(target_arch = "aarch64")] +use core::arch::aarch64::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// Words `lo .. hi` of the rows `(top, bot)`, within both, are those of `(top0, bot0)` through the butterfly with +/// twiddle `t`, and every other word is unchanged. +pub open spec fn butterflied( + transposed: bool, + top0: Seq, + bot0: Seq, + top: Seq, + bot: Seq, + lo: int, + hi: int, + t: u64, +) -> bool { + &&& 0 <= lo <= hi + &&& hi <= top0.len() + &&& hi <= bot0.len() + &&& top.len() == top0.len() + &&& bot.len() == bot0.len() + &&& forall|j: int| + lo <= j < hi ==> ((#[trigger] top[j]).0, bot[j].0) == butterfly_spec(transposed, top0[j].0, bot0[j].0, t) + &&& forall|j: int| 0 <= j < top0.len() && !(lo <= j < hi) ==> #[trigger] top[j] == top0[j] + &&& forall|j: int| 0 <= j < bot0.len() && !(lo <= j < hi) ==> #[trigger] bot[j] == bot0[j] +} + +/// Butterflies on consecutive ranges compose: `lo .. mid` then `mid .. hi` is `lo .. hi`. +pub proof fn lemma_butterflied_extend( + transposed: bool, + top0: Seq, + bot0: Seq, + top1: Seq, + bot1: Seq, + top2: Seq, + bot2: Seq, + lo: int, + mid: int, + hi: int, + t: u64, +) + requires + lo <= mid <= hi, + butterflied(transposed, top0, bot0, top1, bot1, lo, mid, t), + butterflied(transposed, top1, bot1, top2, bot2, mid, hi, t), + ensures + butterflied(transposed, top0, bot0, top2, bot2, lo, hi, t), +{ + assert forall|j: int| lo <= j < hi implies ((#[trigger] top2[j]).0, bot2[j].0) == butterfly_spec( + transposed, + top0[j].0, + bot0[j].0, + t, + ) by { + if j < mid { + assert(top2[j] == top1[j] && bot2[j] == bot1[j]); + } else { + assert(top1[j] == top0[j] && bot1[j] == bot0[j]); + } + } + assert forall|j: int| 0 <= j < top0.len() && !(lo <= j < hi) implies #[trigger] top2[j] == top0[j] by { + assert(top2[j] == top1[j]); + } + assert forall|j: int| 0 <= j < bot0.len() && !(lo <= j < hi) implies #[trigger] bot2[j] == bot0[j] by { + assert(bot2[j] == bot1[j]); + } +} + +/// The shape every kernel computes a lane in: the row multiplied by the twiddle, `m`, and the row the product +/// is added to, `acc`, give `sum = acc + m t`; the new rows are `(m, sum)` transposed, `(sum, v + sum)` forward. +pub proof fn lemma_butterfly_lane( + transposed: bool, + u: u64, + v: u64, + t: u64, + m: u64, + acc: u64, + sum: u64, + nu: u64, + nv: u64, +) + requires + m == if transposed { + u ^ v + } else { + v + }, + acc == if transposed { + v + } else { + u + }, + sum == acc ^ k_mul(m, t), + nu == if transposed { + m + } else { + sum + }, + nv == if transposed { + sum + } else { + v ^ sum + }, + ensures + (nu, nv) == butterfly_spec(transposed, u, v, t), +{ +} + +/// The shift-and-XOR reduction of the x86 kernels: with `lo`, `hi` the words of `p`, the spill +/// `hi>>63 ^ hi>>61 ^ hi>>60`, `x = hi ^ spill` and `g(x) = x ^ x<<1 ^ x<<3 ^ x<<4`, `lo ^ g(x)` is `k_mod(p)`. +pub proof fn lemma_shift_reduction(p: u128, lo: u64, hi: u64, x: u64) + requires + lo == p as u64, + hi == (p >> 64u128) as u64, + x == hi ^ ((hi >> 63u64) ^ (hi >> 61u64) ^ (hi >> 60u64)), + ensures + lo ^ (x ^ (x << 1u64) ^ (x << 3u64) ^ (x << 4u64)) == k_mod(p), +{ + lemma_k_mod(p); + assert(x == hi ^ (hi >> 63u64) ^ (hi >> 61u64) ^ (hi >> 60u64)) by (bit_vector) + requires + x == hi ^ ((hi >> 63u64) ^ (hi >> 61u64) ^ (hi >> 60u64)), + ; + assert(lo ^ (x ^ (x << 1u64) ^ (x << 3u64) ^ (x << 4u64)) == lo ^ x ^ (x << 1u64) ^ (x << 3u64) ^ (x << 4u64)) + by (bit_vector); +} + +// --------------------------------------------------------------------------------------------- +// x86-64: the products in lane order, and the three-way XOR +// --------------------------------------------------------------------------------------------- +/// The products of `_mm*_clmulepi64_epi128::<0x00>` and `::<0x11>` of `m` and a broadcast twiddle, unpacked +/// low with high, are word `i`'s product: `lo[i]` and `hi[i]` are the low and high words of `m[i] * t`. +#[cfg(target_arch = "x86_64")] +pub proof fn lemma_products_in_lane_order( + m: Seq, + tw: Seq, + even: Seq, + odd: Seq, + lo: Seq, + hi: Seq, + t: u64, + i: int, +) + requires + 0 <= i < m.len(), + m.len() % 2 == 0, + tw.len() == m.len(), + forall|k: int| 0 <= k < m.len() ==> #[trigger] tw[k] == t, + forall|k: int| 0 <= k < m.len() ==> #[trigger] even[k] == clmul_lane(m, tw, 0x00i32 as u8, k), + forall|k: int| 0 <= k < m.len() ==> #[trigger] odd[k] == clmul_lane(m, tw, 0x11i32 as u8, k), + forall|k: int| 0 <= k < m.len() ==> #[trigger] lo[k] == unpacklo_lane(even, odd, k), + forall|k: int| 0 <= k < m.len() ==> #[trigger] hi[k] == unpackhi_lane(even, odd, k), + ensures + lo[i] == clmul(m[i], t as u128) as u64, + hi[i] == (clmul(m[i], t as u128) >> 64u128) as u64, +{ + lemma_clmul_sel(); + if i % 2 == 0 { + assert(lo[i] == even[i]); + assert(hi[i] == even[i + 1]); + assert(tw[i] == t); + } else { + assert(lo[i] == odd[i - 1]); + assert(hi[i] == odd[i]); + assert(tw[i] == t); + } +} + +/// `vpternlogq` with immediate `0x96` is the three-way XOR. +#[cfg(target_arch = "x86_64")] +pub proof fn lemma_ternlog_xor3(a: u64, b: u64, c: u64) + ensures + ternlog(0x96i32 as u8, a, b, c) == a ^ b ^ c, +{ + let m = 0x96u8; + assert(0x96i32 as u8 == m); + assert((m >> 0u8) & 1 != 1 && (m >> 1u8) & 1 == 1 && (m >> 2u8) & 1 == 1 && (m >> 3u8) & 1 != 1 && (m >> 4u8) & 1 + == 1 && (m >> 5u8) & 1 != 1 && (m >> 6u8) & 1 != 1 && (m >> 7u8) & 1 == 1) by (bit_vector) + requires + m == 0x96u8, + ; + assert(ternlog(m, a, b, c) == (0u64 & !a & !b & !c) | (!0u64 & !a & !b & c) | (!0u64 & !a & b & !c) | (0u64 & !a + & b & c) | (!0u64 & a & !b & !c) | (0u64 & a & !b & c) | (0u64 & a & b & !c) | (!0u64 & a & b & c)); + assert((0u64 & !a & !b & !c) | (!0u64 & !a & !b & c) | (!0u64 & !a & b & !c) | (0u64 & !a & b & c) | (!0u64 & a + & !b & !c) | (0u64 & a & !b & c) | (0u64 & a & b & !c) | (!0u64 & a & b & c) == a ^ b ^ c) by (bit_vector); +} + +// --------------------------------------------------------------------------------------------- +// AVX-512 +// --------------------------------------------------------------------------------------------- +/// The `vpternlogq` immediate of the three-way XOR. +/// +/// Moved out of `butterfly_lanes_avx512`, where production declares it: Verus takes no items inside a function. +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] +const XOR3: i32 = 0x96; + +/// Eight F64 butterflies with a shared twiddle, one per 64-bit lane of an AVX-512 register. +/// +/// Rewritten for Verus: the rows are `top[at..at + 8]` and `bot[at..at + 8]` instead of two pointers, loaded +/// and stored by [`loadu512_at`] and [`storeu512_at`] (see the module documentation); `XOR3` is a module +/// constant. +/// +/// # Safety +/// +/// - Requires VPCLMULQDQ and AVX-512F. +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx512f", enable = "avx2")] +pub unsafe fn butterfly_lanes_avx512(top: &mut [F64], bot: &mut [F64], at: usize, twiddle: u64) + requires + at + 8 <= old(top).len(), + at + 8 <= old(bot).len(), + ensures + butterflied(TRANSPOSED, old(top)@, old(bot)@, final(top)@, final(bot)@, at as int, at + 8, twiddle), +{ + // The lane lemmas unfold these; here they only need to match the intrinsics' postconditions. + hide(clmul_lane); + hide(clmul_words); + hide(unpacklo_lane); + hide(unpackhi_lane); + hide(ternlog); + hide(butterfly_spec); + let ghost (top0, bot0) = (top@, bot@); + // SAFETY: + // - The caller supplies two valid eight-word rows. + // - This function's target features cover every intrinsic below. + unsafe { + // Load both rows and broadcast the twiddle to every lane. + let u = loadu512_at(top, at); + let v = loadu512_at(bot, at); + let tw = _mm512_set1_epi64(twiddle as i64); + // The row multiplied by the twiddle, and the row the product is added to. + let (m, acc) = if TRANSPOSED { + (_mm512_xor_si512(u, v), v) + } else { + (v, u) + }; + + // Products m * t: even lanes, then odd lanes, one 128-bit product per 128-bit lane. + let even = _mm512_clmulepi64_epi128::<0x00>(m, tw); + let odd = _mm512_clmulepi64_epi128::<0x11>(m, tw); + // Back to lane order: qword i of lo / hi is the low / high half of lane i's product. + let lo = _mm512_unpacklo_epi64(even, odd); + let hi = _mm512_unpackhi_epi64(even, odd); + + // Reduce modulo x^64 + x^4 + x^3 + x + 1 with shifts and three-way XORs. + // The bits of hi * (x^4 + x^3 + x + 1) that land past x^63. + let spill = _mm512_ternarylogic_epi64::( + _mm512_srli_epi64::<63>(hi), + _mm512_srli_epi64::<61>(hi), + _mm512_srli_epi64::<60>(hi), + ); + // Both hi and spill are multiplied by the same constant, so fold them first. + let x = _mm512_xor_si512(hi, spill); + // g(x) = x ^ x<<1 ^ x<<3 ^ x<<4, split across two three-way XORs. + let fx = _mm512_ternarylogic_epi64::(x, _mm512_slli_epi64::<1>(x), _mm512_slli_epi64::<3>(x)); + // acc + m * t, with the product's lo and g(x) folded in one step. + let sum = _mm512_ternarylogic_epi64::(acc, lo, _mm512_xor_si512(fx, _mm512_slli_epi64::<4>(x))); + // Forward: u' = u + v * t, then v' = v + u'. Transposed: u' = u + v, then v' = v + u' * t. + let (new_u, new_v) = if TRANSPOSED { + (m, sum) + } else { + (sum, _mm512_xor_si512(v, sum)) + }; + proof { + lemma_i64_round_trip(twiddle); + // The products, in lane order. + assert forall|i: int| 0 <= i < 8 implies #[trigger] m512(lo)[i] == clmul(m512(m)[i], twiddle as u128) as u64 + && m512(hi)[i] == (clmul(m512(m)[i], twiddle as u128) >> 64u128) as u64 by { + lemma_products_in_lane_order( + m512(m)@, + m512(tw)@, + m512(even)@, + m512(odd)@, + m512(lo)@, + m512(hi)@, + twiddle, + i, + ); + } + // Their reduction: sum = acc + m t. + assert forall|i: int| 0 <= i < 8 implies #[trigger] m512(sum)[i] == m512(acc)[i] ^ k_mul(m512(m)[i], twiddle) by { + let p = clmul(m512(m)[i], twiddle as u128); + let h = m512(hi)[i]; + lemma_ternlog_xor3(h >> 63u64, h >> 61u64, h >> 60u64); + let xi = m512(x)[i]; + assert(xi == h ^ ((h >> 63u64) ^ (h >> 61u64) ^ (h >> 60u64))); + lemma_ternlog_xor3(xi, xi << 1u64, xi << 3u64); + let fi = m512(fx)[i]; + let ai = m512(acc)[i]; + let li = m512(lo)[i]; + lemma_ternlog_xor3(ai, li, fi ^ (xi << 4u64)); + lemma_shift_reduction(p, li, h, xi); + assert(ai ^ li ^ (fi ^ (xi << 4u64)) == ai ^ (li ^ (xi ^ (xi << 1u64) ^ (xi << 3u64) ^ (xi << 4u64)))) + by (bit_vector) + requires + fi == xi ^ (xi << 1u64) ^ (xi << 3u64), + ; + } + // The new rows. + assert forall|i: int| 0 <= i < 8 implies #[trigger] m512(new_u)[i] == butterfly_spec( + TRANSPOSED, + top0[at + i].0, + bot0[at + i].0, + twiddle, + ).0 && m512(new_v)[i] == butterfly_spec(TRANSPOSED, top0[at + i].0, bot0[at + i].0, twiddle).1 by { + lemma_butterfly_lane( + TRANSPOSED, + top0[at + i].0, + bot0[at + i].0, + twiddle, + m512(m)[i], + m512(acc)[i], + m512(sum)[i], + m512(new_u)[i], + m512(new_v)[i], + ); + } + } + storeu512_at(top, at, new_u); + storeu512_at(bot, at, new_v); + proof { + assert forall|j: int| at <= j < at + 8 implies ((#[trigger] top@[j]).0, bot@[j].0) == butterfly_spec( + TRANSPOSED, + top0[j].0, + bot0[j].0, + twiddle, + ) by { + assert(m512(new_u)[j - at] == butterfly_spec(TRANSPOSED, top0[j].0, bot0[j].0, twiddle).0); + } + } + } +} + +// --------------------------------------------------------------------------------------------- +// AVX2 with VPCLMULQDQ +// --------------------------------------------------------------------------------------------- +/// The spill of every top nibble `n`: `n ^ n>>1 ^ n>>3`. +/// +/// Production computes this table in a `const` block local to `butterfly_lanes_avx2`, with a `while` loop over +/// the 16 nibbles. Verus takes no items inside a function, and evaluates no loops in constants, so the copy +/// lists the 16 values; [`lemma_spill_table`] proves they are the formula's, and the equivalence tests run the +/// kernel on every nibble. +#[cfg(all( + target_arch = "x86_64", + target_feature = "vpclmulqdq", + target_feature = "avx2", + not(target_feature = "avx512f") +))] +pub const SPILL: [u8; 16] = [0, 1, 3, 2, 6, 7, 5, 4, 13, 12, 14, 15, 11, 10, 8, 9]; + +/// Entry `n` of [`SPILL`] is `n ^ n>>1 ^ n>>3`. +#[cfg(all( + target_arch = "x86_64", + target_feature = "vpclmulqdq", + target_feature = "avx2", + not(target_feature = "avx512f") +))] +pub proof fn lemma_spill_table(n: u64) + requires + n < 16, + ensures + SPILL[n as int] as u64 == n ^ (n >> 1u64) ^ (n >> 3u64), +{ + assert(SPILL[0] == 0u8 && SPILL[1] == 1u8 && SPILL[2] == 3u8 && SPILL[3] == 2u8); + assert(SPILL[4] == 6u8 && SPILL[5] == 7u8 && SPILL[6] == 5u8 && SPILL[7] == 4u8); + assert(SPILL[8] == 13u8 && SPILL[9] == 12u8 && SPILL[10] == 14u8 && SPILL[11] == 15u8); + assert(SPILL[12] == 11u8 && SPILL[13] == 10u8 && SPILL[14] == 8u8 && SPILL[15] == 9u8); + let s = SPILL[n as int] as u64; + assert(s == n ^ (n >> 1u64) ^ (n >> 3u64)) by (bit_vector) + requires + n < 16, + n == 0 ==> s == 0, + n == 1 ==> s == 1, + n == 2 ==> s == 3, + n == 3 ==> s == 2, + n == 4 ==> s == 6, + n == 5 ==> s == 7, + n == 6 ==> s == 5, + n == 7 ==> s == 4, + n == 8 ==> s == 13, + n == 9 ==> s == 12, + n == 10 ==> s == 14, + n == 11 ==> s == 15, + n == 12 ==> s == 11, + n == 13 ==> s == 10, + n == 14 ==> s == 8, + n == 15 ==> s == 9, + ; +} + +/// A word whose bytes 1 to 7 are zero is its byte 0. +#[cfg(target_arch = "x86_64")] +pub proof fn lemma_word_of_low_byte(w: u64, b: u8) + requires + (w >> 0u64) as u8 == b, + (w >> 8u64) as u8 == 0, + (w >> 16u64) as u8 == 0, + (w >> 24u64) as u8 == 0, + (w >> 32u64) as u8 == 0, + (w >> 40u64) as u8 == 0, + (w >> 48u64) as u8 == 0, + (w >> 56u64) as u8 == 0, + ensures + w == b as u64, +{ + assert(w == b as u64) by (bit_vector) + requires + (w >> 0u64) as u8 == b, + (w >> 8u64) as u8 == 0, + (w >> 16u64) as u8 == 0, + (w >> 24u64) as u8 == 0, + (w >> 32u64) as u8 == 0, + (w >> 40u64) as u8 == 0, + (w >> 48u64) as u8 == 0, + (w >> 56u64) as u8 == 0, + ; +} + +/// The byte shuffle of `butterfly_lanes_avx2`: with `table` the 16-byte [`SPILL`] in both 128-bit lanes and `s` +/// the top nibbles of `hi`, word `i` of the shuffle is the spill `hi>>63 ^ hi>>61 ^ hi>>60` of `hi[i]`. +#[cfg(all( + target_arch = "x86_64", + target_feature = "vpclmulqdq", + target_feature = "avx2", + not(target_feature = "avx512f") +))] +pub proof fn lemma_spill_shuffle(hi: __m256i, s: __m256i, t128: __m128i, table: __m256i, spill: __m256i) + requires + m128_bytes(t128) == SPILL, + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(table)[i] == m128(t128)[i % 2], + forall|i: int| 0 <= i < 4 ==> #[trigger] m256(s)[i] == shr_imm(m256(hi)[i], 60), + forall|k: int| + 0 <= k < 32 ==> #[trigger] m256_bytes(spill)[k] == shuffle_epi8_lane( + m256_bytes(table)@, + m256_bytes(s)@, + k, + ), + ensures + forall|i: int| + 0 <= i < 4 ==> #[trigger] m256(spill)[i] == (m256(hi)[i] >> 63u64) ^ (m256(hi)[i] >> 61u64) ^ (m256( + hi, + )[i] >> 60u64), +{ + broadcast use axiom_m128_bytes, axiom_m256_bytes; + + // Byte k of the table is byte k % 16 of SPILL. + assert forall|k: int| 0 <= k < 32 implies #[trigger] m256_bytes(table)[k] == SPILL[k % 16] by { + assert(m256_bytes(table)[k] == word_byte(m256(table)@, k)); + assert(SPILL[k % 16] == m128_bytes(t128)[k % 16]); + assert(m128_bytes(t128)[k % 16] == word_byte(m128(t128)@, k % 16)); + assert((k / 8) % 2 == (k % 16) / 8 && k % 8 == (k % 16) % 8); + assert(m256(table)[k / 8] == m128(t128)[(k / 8) % 2]); + } + assert forall|i: int| 0 <= i < 4 implies #[trigger] m256(spill)[i] == (m256(hi)[i] >> 63u64) ^ (m256(hi)[i] + >> 61u64) ^ (m256(hi)[i] >> 60u64) by { + let h = m256(hi)[i]; + let n = h >> 60u64; + assert(m256(s)[i] == n); + assert(n < 16) by (bit_vector) + requires + n == h >> 60u64, + ; + // Byte j of word i of the shuffle: SPILL[n] for j = 0, else SPILL[0] = 0. + assert forall|j: int| 0 <= j < 8 implies #[trigger] m256_bytes(spill)[8 * i + j] == if j == 0 { + SPILL[n as int] + } else { + 0u8 + } by { + let k = 8 * i + j; + assert(k / 8 == i && k % 8 == j); + assert(m256_bytes(s)[k] == word_byte(m256(s)@, k)); + let sh = (8 * j) as u64; + let b = (n >> sh) as u8; + assert(m256_bytes(s)[k] == b); + assert(b & 0x80 == 0 && b & 15 == b && b < 16 && (sh == 0 ==> b as u64 == n) && (sh != 0 ==> b == 0)) by ( + bit_vector) + requires + n < 16, + sh == 0 || 8 <= sh < 64, + b == (n >> sh) as u8, + ; + assert((16 * (k / 16) + b as int) % 16 == b as int); + assert(SPILL[0] == 0u8); + } + let w = m256(spill)[i]; + assert forall|j: int| 0 <= j < 8 implies #[trigger] m256_bytes(spill)[8 * i + j] == (w >> ((8 * j) as u64)) + as u8 by { + let k = 8 * i + j; + assert(k / 8 == i && k % 8 == j); + assert(m256_bytes(spill)[k] == word_byte(m256(spill)@, k)); + } + assert(m256_bytes(spill)[8 * i + 0] == (w >> 0u64) as u8); + assert(m256_bytes(spill)[8 * i + 1] == (w >> 8u64) as u8); + assert(m256_bytes(spill)[8 * i + 2] == (w >> 16u64) as u8); + assert(m256_bytes(spill)[8 * i + 3] == (w >> 24u64) as u8); + assert(m256_bytes(spill)[8 * i + 4] == (w >> 32u64) as u8); + assert(m256_bytes(spill)[8 * i + 5] == (w >> 40u64) as u8); + assert(m256_bytes(spill)[8 * i + 6] == (w >> 48u64) as u8); + assert(m256_bytes(spill)[8 * i + 7] == (w >> 56u64) as u8); + lemma_word_of_low_byte(w, SPILL[n as int]); + lemma_spill_table(n); + assert((n ^ (n >> 1u64) ^ (n >> 3u64)) == (h >> 63u64) ^ (h >> 61u64) ^ (h >> 60u64)) by (bit_vector) + requires + n == h >> 60u64, + ; + } +} + +/// One lane of the AVX2 reduction: with `lo`, `hi` the words of `p`, `x = hi ^ spill`, and `x2`, `x8`, `x16` +/// doublings by addition (`x + x` is `x << 1`), `lo ^ (x ^ x2) ^ (x8 ^ x16)` is `k_mod(p)`. +#[cfg(target_arch = "x86_64")] +pub proof fn lemma_avx2_lane(p: u128, acc: u64, lo: u64, hi: u64, x: u64, x2: u64, x4: u64, x8: u64, x16: u64, sum: u64) + requires + lo == p as u64, + hi == (p >> 64u128) as u64, + x == hi ^ ((hi >> 63u64) ^ (hi >> 61u64) ^ (hi >> 60u64)), + x2 == (x + x) as u64, + x4 == (x2 + x2) as u64, + x8 == (x4 + x4) as u64, + x16 == (x8 + x8) as u64, + sum == acc ^ (lo ^ ((x ^ x2) ^ (x8 ^ x16))), + ensures + sum == acc ^ k_mod(p), +{ + assert(x2 == x << 1u64 && x8 == x << 3u64 && x16 == x << 4u64) by (bit_vector) + requires + x2 == (x + x) as u64, + x4 == (x2 + x2) as u64, + x8 == (x4 + x4) as u64, + x16 == (x8 + x8) as u64, + ; + lemma_shift_reduction(p, lo, hi, x); + assert(acc ^ (lo ^ ((x ^ x2) ^ (x8 ^ x16))) == acc ^ (lo ^ (x ^ (x << 1u64) ^ (x << 3u64) ^ (x << 4u64)))) + by (bit_vector) + requires + x2 == x << 1u64, + x8 == x << 3u64, + x16 == x << 4u64, + ; +} + +/// Four F64 butterflies with a shared twiddle, for a machine with VPCLMULQDQ but no AVX-512. +/// +/// Rewritten for Verus: the rows are `top[at..at + 4]` and `bot[at..at + 4]` instead of two pointers, loaded and +/// stored by [`loadu256_at`] and [`storeu256_at`] (see the module documentation); the nibble table is the module +/// constant [`SPILL`], loaded by [`loadu128_bytes`] (production: `_mm_loadu_si128(SPILL.as_ptr().cast())`). +/// +/// # Safety +/// +/// - Requires VPCLMULQDQ and AVX2. +#[cfg(all( + target_arch = "x86_64", + target_feature = "vpclmulqdq", + target_feature = "avx2", + not(target_feature = "avx512f") +))] +#[inline] +#[target_feature(enable = "vpclmulqdq", enable = "avx2")] +pub unsafe fn butterfly_lanes_avx2(top: &mut [F64], bot: &mut [F64], at: usize, twiddle: u64) + requires + at + 4 <= old(top).len(), + at + 4 <= old(bot).len(), + ensures + butterflied(TRANSPOSED, old(top)@, old(bot)@, final(top)@, final(bot)@, at as int, at + 4, twiddle), +{ + // The lane lemmas unfold these; here they only need to match the intrinsics' postconditions. + hide(clmul_lane); + hide(clmul_words); + hide(unpacklo_lane); + hide(unpackhi_lane); + hide(shuffle_epi8_lane); + hide(butterfly_spec); + let ghost (top0, bot0) = (top@, bot@); + // SAFETY: + // - The caller supplies two valid four-word rows. + // - This function's target features cover every intrinsic below. + unsafe { + let u = loadu256_at(top, at); + let v = loadu256_at(bot, at); + let tw = _mm256_set1_epi64x(twiddle as i64); + + // The row multiplied by the twiddle, and the row the product is added to. + let (m, acc) = if TRANSPOSED { + (_mm256_xor_si256(u, v), v) + } else { + (v, u) + }; + + // Products m * t, one 128-bit product per 128-bit lane, then back to lane order. + let even = _mm256_clmulepi64_epi128::<0x00>(m, tw); + let odd = _mm256_clmulepi64_epi128::<0x11>(m, tw); + let lo = _mm256_unpacklo_epi64(even, odd); + let hi = _mm256_unpackhi_epi64(even, odd); + + // The spill of every top nibble. + let t128 = loadu128_bytes(&SPILL); + let table = _mm256_broadcastsi128_si256(t128); + let nibbles = _mm256_srli_epi64::<60>(hi); + let spill = _mm256_shuffle_epi8(table, nibbles); + // Both hi and spill are multiplied by the same constant, so fold them first. + let x = _mm256_xor_si256(hi, spill); + let x2 = _mm256_add_epi64(x, x); + let x8 = { + let x4 = _mm256_add_epi64(x2, x2); + _mm256_add_epi64(x4, x4) + }; + let x16 = _mm256_add_epi64(x8, x8); + let gx = _mm256_xor_si256(_mm256_xor_si256(x, x2), _mm256_xor_si256(x8, x16)); + let product = _mm256_xor_si256(lo, gx); + + let sum = _mm256_xor_si256(acc, product); + let (new_u, new_v) = if TRANSPOSED { + (m, sum) + } else { + (sum, _mm256_xor_si256(v, sum)) + }; + proof { + lemma_i64_round_trip(twiddle); + lemma_spill_shuffle(hi, nibbles, t128, table, spill); + // The products, in lane order. + assert forall|i: int| 0 <= i < 4 implies #[trigger] m256(lo)[i] == clmul(m256(m)[i], twiddle as u128) as u64 + && m256(hi)[i] == (clmul(m256(m)[i], twiddle as u128) >> 64u128) as u64 by { + lemma_products_in_lane_order( + m256(m)@, + m256(tw)@, + m256(even)@, + m256(odd)@, + m256(lo)@, + m256(hi)@, + twiddle, + i, + ); + } + // Their reduction: sum = acc + m t. + assert forall|i: int| 0 <= i < 4 implies #[trigger] m256(sum)[i] == m256(acc)[i] ^ k_mul(m256(m)[i], twiddle) by { + let x2i = m256(x2)[i]; + lemma_avx2_lane( + clmul(m256(m)[i], twiddle as u128), + m256(acc)[i], + m256(lo)[i], + m256(hi)[i], + m256(x)[i], + x2i, + (x2i + x2i) as u64, + m256(x8)[i], + m256(x16)[i], + m256(sum)[i], + ); + } + // The new rows. + assert forall|i: int| 0 <= i < 4 implies #[trigger] m256(new_u)[i] == butterfly_spec( + TRANSPOSED, + top0[at + i].0, + bot0[at + i].0, + twiddle, + ).0 && m256(new_v)[i] == butterfly_spec(TRANSPOSED, top0[at + i].0, bot0[at + i].0, twiddle).1 by { + lemma_butterfly_lane( + TRANSPOSED, + top0[at + i].0, + bot0[at + i].0, + twiddle, + m256(m)[i], + m256(acc)[i], + m256(sum)[i], + m256(new_u)[i], + m256(new_v)[i], + ); + } + } + storeu256_at(top, at, new_u); + storeu256_at(bot, at, new_v); + proof { + assert forall|j: int| at <= j < at + 4 implies ((#[trigger] top@[j]).0, bot@[j].0) == butterfly_spec( + TRANSPOSED, + top0[j].0, + bot0[j].0, + twiddle, + ) by { + assert(m256(new_u)[j - at] == butterfly_spec(TRANSPOSED, top0[j].0, bot0[j].0, twiddle).0); + } + } + } +} + +// --------------------------------------------------------------------------------------------- +// NEON with PMULL +// --------------------------------------------------------------------------------------------- +/// The two products of a register pair by the twiddle, read back as `uint64x2_t`: their lanes as one `u128` are +/// the carry-less products. +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +pub proof fn lemma_pmull_products(m: uint64x2_t, t: u64, x0: u128, x1: u128) + requires + x0 == clmul(u64x2(m)[0], t as u128), + x1 == clmul(u64x2(m)[1], t as u128), + ensures + u64x2_u128(transmuted::(x0)) == x0, + u64x2_u128(transmuted::(x1)) == x1, +{ + broadcast use axiom_u128_as_u64x2; + + assert(((x0 as u64) as u128) | ((((x0 >> 64u128) as u64) as u128) << 64u128) == x0) by (bit_vector); + assert(((x1 as u64) as u128) | ((((x1 >> 64u128) as u64) as u128) << 64u128) == x1) by (bit_vector); +} + +/// One register pair of a NEON kernel: lane `l` of `(nu, nv)` is lane `l` of `(u, v)` through the butterfly. +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +pub proof fn lemma_neon_pair( + transposed: bool, + t: u64, + u: uint64x2_t, + v: uint64x2_t, + m: uint64x2_t, + acc: uint64x2_t, + prod: uint64x2_t, + sum: uint64x2_t, + nu: uint64x2_t, + nv: uint64x2_t, +) + requires + forall|l: int| + 0 <= l < 2 ==> #[trigger] u64x2(m)[l] == if transposed { + u64x2(u)[l] ^ u64x2(v)[l] + } else { + u64x2(v)[l] + }, + acc == if transposed { + v + } else { + u + }, + u64x2(prod)[0] == k_mul(u64x2(m)[0], t), + u64x2(prod)[1] == k_mul(u64x2(m)[1], t), + u64x2(sum)[0] == u64x2(acc)[0] ^ u64x2(prod)[0], + u64x2(sum)[1] == u64x2(acc)[1] ^ u64x2(prod)[1], + nu == if transposed { + m + } else { + sum + }, + forall|l: int| + 0 <= l < 2 ==> #[trigger] u64x2(nv)[l] == if transposed { + u64x2(sum)[l] + } else { + u64x2(v)[l] ^ u64x2(sum)[l] + }, + ensures + forall|l: int| + 0 <= l < 2 ==> (#[trigger] u64x2(nu)[l], u64x2(nv)[l]) == butterfly_spec( + transposed, + u64x2(u)[l], + u64x2(v)[l], + t, + ), +{ + assert forall|l: int| 0 <= l < 2 implies (#[trigger] u64x2(nu)[l], u64x2(nv)[l]) == butterfly_spec( + transposed, + u64x2(u)[l], + u64x2(v)[l], + t, + ) by { + assert(u64x2(m)[l] == if transposed { + u64x2(u)[l] ^ u64x2(v)[l] + } else { + u64x2(v)[l] + }); + assert(u64x2(nv)[l] == if transposed { + u64x2(sum)[l] + } else { + u64x2(v)[l] ^ u64x2(sum)[l] + }); + if l == 0 { + lemma_butterfly_lane( + transposed, + u64x2(u)[0], + u64x2(v)[0], + t, + u64x2(m)[0], + u64x2(acc)[0], + u64x2(sum)[0], + u64x2(nu)[0], + u64x2(nv)[0], + ); + } else { + lemma_butterfly_lane( + transposed, + u64x2(u)[1], + u64x2(v)[1], + t, + u64x2(m)[1], + u64x2(acc)[1], + u64x2(sum)[1], + u64x2(nu)[1], + u64x2(nv)[1], + ); + } + } +} + +/// The reduced products of a register by the twiddle, as [`reduce_pair_pmull4`] returns them from the two PMULLs. +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +pub proof fn lemma_reduced_products(m: uint64x2_t, t: u64, p0: uint64x2_t, p1: uint64x2_t, prod: uint64x2_t) + requires + p0 == transmuted::(clmul(u64x2(m)[0], t as u128)), + p1 == transmuted::(clmul(u64x2(m)[1], t as u128)), + u64x2(prod)[0] == k_mod(u64x2_u128(p0)), + u64x2(prod)[1] == k_mod(u64x2_u128(p1)), + ensures + u64x2(prod)[0] == k_mul(u64x2(m)[0], t), + u64x2(prod)[1] == k_mul(u64x2(m)[1], t), +{ + lemma_pmull_products(m, t, clmul(u64x2(m)[0], t as u128), clmul(u64x2(m)[1], t as u128)); +} + +/// Eight F64 butterflies as four independent NEON lane-pair reductions. +/// Loading all four bottom vectors before reducing them gives the out-of-order +/// core four independent PMULL chains to schedule, while one call amortizes +/// loop control and the duplicated twiddle/reduction constants over 8 lanes. +/// +/// Rewritten for Verus: the rows are `top[at..at + 8]` and `bot[at..at + 8]` instead of two pointers, loaded and +/// stored by [`vld1q_u64_at`] and [`vst1q_u64_at`] (see the module documentation). +/// +/// # Safety +/// Requires the `aes` target feature. +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +#[inline] +#[target_feature(enable = "aes")] +pub unsafe fn butterfly_lanes_neon_8(top: &mut [F64], bot: &mut [F64], at: usize, twiddle: u64) + requires + at + 8 <= old(top).len(), + at + 8 <= old(bot).len(), + ensures + butterflied(TRANSPOSED, old(top)@, old(bot)@, final(top)@, final(bot)@, at as int, at + 8, twiddle), +{ + let ghost (top0, bot0) = (top@, bot@); + // SAFETY: caller guarantees the two eight-element regions; F64 is + // repr(transparent) over u64 and this function carries the aes feature. + unsafe { + let v0 = vld1q_u64_at(bot, at); + let v1 = vld1q_u64_at(bot, at + 2); + let v2 = vld1q_u64_at(bot, at + 4); + let v3 = vld1q_u64_at(bot, at + 6); + let u0 = vld1q_u64_at(top, at); + let u1 = vld1q_u64_at(top, at + 2); + let u2 = vld1q_u64_at(top, at + 4); + let u3 = vld1q_u64_at(top, at + 6); + let tw = vdupq_n_u64(twiddle); + // The rows multiplied by the twiddle, and the rows the products are added to. + let ((m0, a0), (m1, a1), (m2, a2), (m3, a3)) = if TRANSPOSED { + ( + (veorq_u64(u0, v0), v0), + (veorq_u64(u1, v1), v1), + (veorq_u64(u2, v2), v2), + (veorq_u64(u3, v3), v3), + ) + } else { + ((v0, u0), (v1, u1), (v2, u2), (v3, u3)) + }; + + proof { + broadcast use axiom_u64x2_as_p64x2; + } + let p00: uint64x2_t = core::mem::transmute(vmull_p64(vgetq_lane_u64::<0>(m0), twiddle)); + let p01: uint64x2_t = core::mem::transmute(vmull_high_p64( + core::mem::transmute::(m0), + core::mem::transmute::(tw), + )); + let p10: uint64x2_t = core::mem::transmute(vmull_p64(vgetq_lane_u64::<0>(m1), twiddle)); + let p11: uint64x2_t = core::mem::transmute(vmull_high_p64( + core::mem::transmute::(m1), + core::mem::transmute::(tw), + )); + let p20: uint64x2_t = core::mem::transmute(vmull_p64(vgetq_lane_u64::<0>(m2), twiddle)); + let p21: uint64x2_t = core::mem::transmute(vmull_high_p64( + core::mem::transmute::(m2), + core::mem::transmute::(tw), + )); + let p30: uint64x2_t = core::mem::transmute(vmull_p64(vgetq_lane_u64::<0>(m3), twiddle)); + let p31: uint64x2_t = core::mem::transmute(vmull_high_p64( + core::mem::transmute::(m3), + core::mem::transmute::(tw), + )); + + let r0 = reduce_pair_pmull4(p00, p01); + let r1 = reduce_pair_pmull4(p10, p11); + let r2 = reduce_pair_pmull4(p20, p21); + let r3 = reduce_pair_pmull4(p30, p31); + proof { + lemma_reduced_products(m0, twiddle, p00, p01, r0); + lemma_reduced_products(m1, twiddle, p10, p11, r1); + lemma_reduced_products(m2, twiddle, p20, p21, r2); + lemma_reduced_products(m3, twiddle, p30, p31, r3); + } + let sum0 = veorq_u64(a0, r0); + let sum1 = veorq_u64(a1, r1); + let sum2 = veorq_u64(a2, r2); + let sum3 = veorq_u64(a3, r3); + // Forward: u' = u + v * t, then v' = v + u'. Transposed: u' = u + v, then v' = v + u' * t. + let ((new_u0, new_v0), (new_u1, new_v1), (new_u2, new_v2), (new_u3, new_v3)) = if TRANSPOSED { + ((m0, sum0), (m1, sum1), (m2, sum2), (m3, sum3)) + } else { + ( + (sum0, veorq_u64(v0, sum0)), + (sum1, veorq_u64(v1, sum1)), + (sum2, veorq_u64(v2, sum2)), + (sum3, veorq_u64(v3, sum3)), + ) + }; + proof { + lemma_neon_pair(TRANSPOSED, twiddle, u0, v0, m0, a0, r0, sum0, new_u0, new_v0); + lemma_neon_pair(TRANSPOSED, twiddle, u1, v1, m1, a1, r1, sum1, new_u1, new_v1); + lemma_neon_pair(TRANSPOSED, twiddle, u2, v2, m2, a2, r2, sum2, new_u2, new_v2); + lemma_neon_pair(TRANSPOSED, twiddle, u3, v3, m3, a3, r3, sum3, new_u3, new_v3); + } + + vst1q_u64_at(top, at, new_u0); + vst1q_u64_at(top, at + 2, new_u1); + vst1q_u64_at(top, at + 4, new_u2); + vst1q_u64_at(top, at + 6, new_u3); + vst1q_u64_at(bot, at, new_v0); + vst1q_u64_at(bot, at + 2, new_v1); + vst1q_u64_at(bot, at + 4, new_v2); + vst1q_u64_at(bot, at + 6, new_v3); + proof { + assert forall|j: int| at <= j < at + 8 implies ((#[trigger] top@[j]).0, bot@[j].0) == butterfly_spec( + TRANSPOSED, + top0[j].0, + bot0[j].0, + twiddle, + ) by { + let l = (j - at) % 2; + if j < at + 2 { + assert((u64x2(new_u0)[l], u64x2(new_v0)[l]) == butterfly_spec(TRANSPOSED, u64x2(u0)[l], u64x2(v0)[l], twiddle)); + } else if j < at + 4 { + assert((u64x2(new_u1)[l], u64x2(new_v1)[l]) == butterfly_spec(TRANSPOSED, u64x2(u1)[l], u64x2(v1)[l], twiddle)); + } else if j < at + 6 { + assert((u64x2(new_u2)[l], u64x2(new_v2)[l]) == butterfly_spec(TRANSPOSED, u64x2(u2)[l], u64x2(v2)[l], twiddle)); + } else { + assert((u64x2(new_u3)[l], u64x2(new_v3)[l]) == butterfly_spec(TRANSPOSED, u64x2(u3)[l], u64x2(v3)[l], twiddle)); + } + } + } + } +} + +/// Two F64 butterflies with a shared twiddle, NEON-resident end to end. +/// The two products issue as PMULL/PMULL2 on the loaded row (no lane +/// extraction) and reduce through the all-PMULL lane-pair fold +/// ([`reduce_pair_pmull4`]), replacing the +/// old 10-op shift-XOR fold chain. +/// +/// Rewritten for Verus: the rows are `top[at..at + 2]` and `bot[at..at + 2]` instead of two pointers, loaded and +/// stored by [`vld1q_u64_at`] and [`vst1q_u64_at`] (see the module documentation). +/// +/// # Safety +/// Requires the `aes` target feature. +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +#[inline] +#[target_feature(enable = "aes")] +pub unsafe fn butterfly_lane_pair_neon(top: &mut [F64], bot: &mut [F64], at: usize, twiddle: u64) + requires + at + 2 <= old(top).len(), + at + 2 <= old(bot).len(), + ensures + butterflied(TRANSPOSED, old(top)@, old(bot)@, final(top)@, final(bot)@, at as int, at + 2, twiddle), +{ + let ghost (top0, bot0) = (top@, bot@); + // SAFETY: caller guarantees the pointees; F64 is repr(transparent) u64. + unsafe { + let u = vld1q_u64_at(top, at); + let v = vld1q_u64_at(bot, at); + let m = if TRANSPOSED { veorq_u64(u, v) } else { v }; + // Products m_lane * twiddle: PMULL on the low lanes, PMULL2 on the + // highs (the dup is loop-invariant and hoisted after inlining). + let tw = vdupq_n_u64(twiddle); + proof { + broadcast use axiom_u64x2_as_p64x2; + } + let p0: uint64x2_t = core::mem::transmute(vmull_p64(vgetq_lane_u64::<0>(m), twiddle)); + let p1: uint64x2_t = core::mem::transmute(vmull_high_p64( + core::mem::transmute::(m), + core::mem::transmute::(tw), + )); + let prod = reduce_pair_pmull4(p0, p1); + proof { + lemma_reduced_products(m, twiddle, p0, p1, prod); + } + let (new_u, new_v) = if TRANSPOSED { + (m, veorq_u64(v, prod)) + } else { + let new_u = veorq_u64(u, prod); + (new_u, veorq_u64(v, new_u)) + }; + proof { + let acc = if TRANSPOSED { v } else { u }; + let sum = if TRANSPOSED { new_v } else { new_u }; + lemma_neon_pair(TRANSPOSED, twiddle, u, v, m, acc, prod, sum, new_u, new_v); + } + vst1q_u64_at(top, at, new_u); + vst1q_u64_at(bot, at, new_v); + proof { + assert forall|j: int| at <= j < at + 2 implies ((#[trigger] top@[j]).0, bot@[j].0) == butterfly_spec( + TRANSPOSED, + top0[j].0, + bot0[j].0, + twiddle, + ) by { + let l = j - at; + assert((u64x2(new_u)[l], u64x2(new_v)[l]) == butterfly_spec(TRANSPOSED, u64x2(u)[l], u64x2(v)[l], twiddle)); + } + } + } +} + +} // verus! diff --git a/verification/verus/src/parallel.rs b/verification/verus/src/parallel.rs new file mode 100644 index 0000000000..09141fcdbc --- /dev/null +++ b/verification/verus/src/parallel.rs @@ -0,0 +1,762 @@ +//! The pool primitives of `crates/parallel/src/lib.rs` that the NTT driver uses, with permissions for +//! the memory their tasks touch. +//! +//! A buffer's memory is a map of permissions, one `vstd::raw_ptr::PointsTo` per element, keyed by the +//! element's index from an origin pointer ([`owns`]). Only the holder of an element's permission can +//! read or write it, and a permission cannot be duplicated, so tasks that hold disjoint maps cannot race. +//! +//! The trusted specifications are collected under "Trust base" below: the dispatch of the pool +//! ([`for_each_chunk`]), the passage between a slice and the permissions it stands for +//! ([`slice_as_mut_ptr`], [`slice_as_ptr`], [`from_raw_parts_mut`], [`from_raw_parts`]), and the core +//! functions vstd does not specify (`<*mut T>::add`, `<*const T>::cast_mut`, `div_ceil`, +//! `std::ptr::eq`), plus the worker count. [`for_each`], [`SendPtr`], [`RawMut`], +//! [`Chunks`] and [`chunks_mut`] are verified copies/adapters with explicit specifications. +use vstd::prelude::*; +use vstd::raw_ptr::*; + +verus! { + +broadcast use vstd::raw_ptr::group_raw_ptr_axioms; + +// --------------------------------------------------------------------------------------------- +// Permissions +// --------------------------------------------------------------------------------------------- +/// The pointer to element `k` of the buffer at `o`: `o.add(k)`. +pub open spec fn ptr_at(o: *mut T, k: int) -> *mut T { + ptr_mut_from_data(PtrData:: { addr: (o@.addr + k * size_of::()) as usize, ..o@ }) +} + +/// The elements `lo..lo + len` of the buffer at `o` are initialized, `perms` holds exactly their +/// permissions, and the range does not wrap the address space. +pub open spec fn owns(perms: Map>, o: *mut T, lo: int, len: int) -> bool { + &&& 0 <= lo + &&& 0 <= len + &&& o@.addr + (lo + len) * size_of::() <= usize::MAX + &&& forall|k: int| #[trigger] perms.dom().contains(k) <==> lo <= k < lo + len + &&& forall|k: int| lo <= k < lo + len ==> (#[trigger] perms[k]).ptr() == ptr_at(o, k) && perms[k].is_init() +} + +/// As [`owns`], for a map that may hold more permissions than the range's. +pub open spec fn covers(perms: Map>, o: *mut T, lo: int, len: int) -> bool { + &&& 0 <= lo + &&& 0 <= len + &&& o@.addr + (lo + len) * size_of::() <= usize::MAX + &&& forall|k: int| + lo <= k < lo + len ==> #[trigger] perms.dom().contains(k) && perms[k].ptr() == ptr_at(o, k) && perms[k].is_init() +} + +/// The values of the elements `lo..lo + len`. +pub open spec fn vals(perms: Map>, lo: int, len: int) -> Seq { + Seq::new(len as nat, |j: int| perms[lo + j].value()) +} + +/// The keys of `perms` whose owner lies in `lo..hi`. +pub open spec fn keys_of(perms: Map, owner: spec_fn(int) -> int, lo: int, hi: int) -> Set { + perms.dom().filter(|k: int| lo <= owner(k) < hi) +} + +/// The permissions of the items `lo..hi`, item `owner(k)` holding key `k`. +pub open spec fn claimed(perms: Map, owner: spec_fn(int) -> int, lo: int, hi: int) -> Map { + perms.restrict(keys_of(perms, owner, lo, hi)) +} + +/// Disjoint item ranges cannot receive any of the same element permissions. +pub proof fn lemma_claims_disjoint(perms: Map, owner: spec_fn(int) -> int, + a: int, b: int, c: int, d: int) + requires a <= b <= c <= d, + ensures claimed(perms, owner, a, b).dom().disjoint(claimed(perms, owner, c, d).dom()), +{ + assert forall|k: int| claimed(perms, owner, a, b).dom().contains(k) + implies !claimed(perms, owner, c, d).dom().contains(k) by {} +} + +// --------------------------------------------------------------------------------------------- +// Trust base +// --------------------------------------------------------------------------------------------- +/// `p.add(count)` is the pointer `count` elements on. +/// +/// Rust also requires the result to stay inside the allocation of `p`. Every call in this crate +/// dereferences the result through the permission of that element (or builds a slice of length +/// at least one there), which puts it inside the allocation. +pub assume_specification[ <*mut T>::add ](p: *mut T, count: usize) -> (q: *mut T) + requires + p@.addr + count * size_of::() <= usize::MAX, + ensures + q == ptr_at(p, count as int), +; + +/// `p.cast_mut()` is the same pointer. +pub assume_specification[ <*const T>::cast_mut ](p: *const T) -> (q: *mut T) + ensures + q@ == p@, +; + +/// `a.div_ceil(b)` rounds the quotient up. +pub assume_specification[ ::div_ceil ](a: usize, b: usize) -> (r: usize) + requires + b > 0, + ensures + r == (a + b - 1) / (b as int), +; + +/// `std::ptr::eq` compares addresses and metadata (as vstd's `==` on pointers). +pub assume_specification[ core::ptr::eq:: ](a: *const T, b: *const T) -> (r: bool) + ensures + r == (a@.addr == b@.addr && a@.metadata == b@.metadata), +; + +/// A mutable slice is the permission to its elements, at its address, for as long as it is borrowed. +/// +/// The permissions come back to the slice when the borrow ends: if the map then still holds them, the +/// slice's new contents are their values. (A caller that kept a permission past the borrow could not +/// prove anything about the slice afterwards; every caller here proves the slice's final contents.) +#[verifier::external_body] +pub fn slice_as_mut_ptr<'a, T>(s: &'a mut [T]) -> (r: (*mut T, Tracked<&'a mut Map>>)) + ensures + owns(*r.1@, r.0, 0, old(s)@.len() as int), + vals(*r.1@, 0, old(s)@.len() as int) == old(s)@, + owns(*final(r.1@), r.0, 0, old(s)@.len() as int) ==> final(s)@ == vals(*final(r.1@), 0, old(s)@.len() as int), +{ + (s.as_mut_ptr(), Tracked::assume_new()) +} + +/// A shared slice is a shared permission to its elements, at its address. +#[verifier::external_body] +pub fn slice_as_ptr<'a, T>(s: &'a [T]) -> (r: (*const T, Tracked<&'a Map>>)) + ensures + owns(*r.1@, r.0 as *mut T, 0, s@.len() as int), + vals(*r.1@, 0, s@.len() as int) == s@, +{ + (s.as_ptr(), Tracked::assume_new()) +} + +/// `std::slice::from_raw_parts_mut(ptr, len)`, given the permissions of its elements, which it borrows. +#[verifier::external_body] +pub fn from_raw_parts_mut<'a, T>( + ptr: *mut T, + len: usize, + Ghost(o): Ghost<*mut T>, + Ghost(lo): Ghost, + Tracked(perms): Tracked<&'a mut Map>>, +) -> (s: &'a mut [T]) + requires + owns(*old(perms), o, lo, len as int), + ptr == ptr_at(o, lo), + ensures + s@ == vals(*old(perms), lo, len as int), + owns(*final(perms), o, lo, len as int), + final(s)@ == vals(*final(perms), lo, len as int), +{ + unsafe { std::slice::from_raw_parts_mut(ptr, len) } +} + +/// `std::slice::from_raw_parts(ptr, len)`, given shared permissions of its elements. +#[verifier::external_body] +pub fn from_raw_parts<'a, T>( + ptr: *const T, + len: usize, + Ghost(o): Ghost<*mut T>, + Ghost(lo): Ghost, + Tracked(perms): Tracked<&'a Map>>, +) -> (s: &'a [T]) + requires + covers(*perms, o, lo, len as int), + ptr as *mut T == ptr_at(o, lo), + ensures + s@ == vals(*perms, lo, len as int), +{ + unsafe { std::slice::from_raw_parts(ptr, len) } +} + +/// Worker count including the dispatcher. +#[verifier::external_body] +pub fn num_threads() -> (r: usize) + ensures + r >= 1, +{ + ::parallel::num_threads() +} + +/// The pool's dispatch: `f(lo, hi)` runs on disjoint ranges that tile `0..n_tasks`, each exactly once, +/// concurrently, and the call returns after all of them. +/// +/// Specification: the permissions `perms` are split by item, key `k` going to item `owner(k)`; a +/// call `f(lo, hi)` receives the permissions of its items and returns them, satisfying `post` for every +/// item of its range. The dispatcher gets back the permissions of every item, each satisfying `post`. +/// +/// The body delegates to production's `parallel::for_each_chunk`. Its atomic claim counter, +/// worker lifetime, synchronization and exactly-once behavior are trusted, not verified here. +#[verifier::external_body] +pub fn for_each_chunk>) -> Tracked> + Sync>( + n_tasks: usize, + f: F, + Tracked(perms): Tracked>, + Ghost(owner): Ghost int>, + Ghost(post): Ghost) -> bool>, +) -> (out: Tracked>) + requires + forall|k: int| #[trigger] perms.dom().contains(k) ==> 0 <= owner(k) < n_tasks, + forall|lo: usize, hi: usize| + lo < hi <= n_tasks ==> #[trigger] f.requires((lo, hi, Tracked(claimed(perms, owner, lo as int, hi as int)))), + forall|lo: usize, hi: usize, r: Tracked>| + lo < hi <= n_tasks && #[trigger] f.ensures((lo, hi, Tracked(claimed(perms, owner, lo as int, hi as int))), r) ==> { + &&& r@.dom() == keys_of(perms, owner, lo as int, hi as int) + &&& forall|i: int| lo <= i < hi ==> #[trigger] post(i, r@.restrict(keys_of(perms, owner, i, i + 1))) + }, + ensures + out@.dom() == perms.dom(), + forall|i: int| 0 <= i < n_tasks ==> #[trigger] post(i, out@.restrict(keys_of(perms, owner, i, i + 1))), +{ + ::parallel::for_each_chunk(n_tasks, |lo, hi| { + f(lo, hi, Tracked::assume_new()); + }); + Tracked::assume_new() +} + +// --------------------------------------------------------------------------------------------- +// Verified copies +// --------------------------------------------------------------------------------------------- +/// The permissions of one item, taken out of the claimed range `lo..hi` and put back. +proof fn lemma_claimed_split(perms: Map, owner: spec_fn(int) -> int, lo: int, i: int, hi: int) + requires + lo <= i < hi, + ensures + keys_of(perms, owner, i, i + 1) <= keys_of(perms, owner, lo, hi), + claimed(perms, owner, lo, hi).restrict(keys_of(perms, owner, i, i + 1)) == claimed(perms, owner, i, i + 1), + keys_of(perms, owner, lo, hi) == keys_of(perms, owner, lo, i).union(keys_of(perms, owner, i, hi)), +{ + assert(claimed(perms, owner, lo, hi).restrict(keys_of(perms, owner, i, i + 1)) =~= claimed(perms, owner, i, i + 1)); + assert(keys_of(perms, owner, lo, hi) =~= keys_of(perms, owner, lo, i).union(keys_of(perms, owner, i, hi))); +} + +/// `f(i)` for every `i` in `0..n_tasks`, in parallel. `#[inline]` folds the +/// range-to-index adapter into the monomorphized [`for_each_chunk`]. +/// +/// Specification: as [`for_each_chunk`], one item at a time. `f` takes and returns the permissions of +/// its item (a ghost addition to production's `Fn(usize)`). +#[inline] +pub fn for_each>) -> Tracked> + Sync>( + n_tasks: usize, + f: F, + Tracked(perms): Tracked>, + Ghost(owner): Ghost int>, + Ghost(post): Ghost) -> bool>, +) -> (out: Tracked>) + requires + forall|k: int| #[trigger] perms.dom().contains(k) ==> 0 <= owner(k) < n_tasks, + forall|i: usize| i < n_tasks ==> #[trigger] f.requires((i, Tracked(claimed(perms, owner, i as int, i + 1)))), + forall|i: usize, r: Tracked>| + i < n_tasks && #[trigger] f.ensures((i, Tracked(claimed(perms, owner, i as int, i + 1))), r) ==> { + &&& r@.dom() == keys_of(perms, owner, i as int, i + 1) + &&& post(i as int, r@) + }, + ensures + out@.dom() == perms.dom(), + forall|i: int| 0 <= i < n_tasks ==> #[trigger] post(i, out@.restrict(keys_of(perms, owner, i, i + 1))), +{ + let ghost p0 = perms; + for_each_chunk( + n_tasks, + |start: usize, end: usize, tp: Tracked>| -> (r: Tracked>) + requires + start < end <= n_tasks, + tp@ == claimed(p0, owner, start as int, end as int), + ensures + r@.dom() == keys_of(p0, owner, start as int, end as int), + forall|i: int| start <= i < end ==> #[trigger] post(i, r@.restrict(keys_of(p0, owner, i, i + 1))), + { + // Rewritten from `for i in start..end { f(i); }`: the loop also threads the permissions. + let tracked mut rest = tp.get(); + let tracked mut done = Map::::tracked_empty(); + let mut i = start; + while i < end + invariant + start <= i <= end <= n_tasks, + rest == claimed(p0, owner, i as int, end as int), + done.dom() == keys_of(p0, owner, start as int, i as int), + forall|j: int| start <= j < i ==> #[trigger] post(j, done.restrict(keys_of(p0, owner, j, j + 1))), + forall|i: usize| i < n_tasks ==> #[trigger] f.requires((i, Tracked(claimed(p0, owner, i as int, i + 1)))), + forall|i: usize, r: Tracked>| + i < n_tasks && #[trigger] f.ensures((i, Tracked(claimed(p0, owner, i as int, i + 1))), r) ==> { + &&& r@.dom() == keys_of(p0, owner, i as int, i + 1) + &&& post(i as int, r@) + }, + decreases end - i, + { + proof { + lemma_claimed_split(p0, owner, i as int, i as int, end as int); + lemma_claimed_split(p0, owner, start as int, i as int, i + 1); + } + let tracked one = rest.tracked_remove_keys(keys_of(p0, owner, i as int, i + 1)); + proof { + assert(one == claimed(p0, owner, i as int, i + 1)); + assert(rest =~= claimed(p0, owner, i + 1, end as int)); + } + let Tracked(r) = f(i, Tracked(one)); + let ghost old_done = done; + let ghost rv = r; + proof { + done.tracked_union_prefer_right(r); + assert(done.dom() =~= keys_of(p0, owner, start as int, i + 1)); + assert forall|j: int| start <= j < i + 1 implies #[trigger] post(j, done.restrict(keys_of(p0, owner, j, j + 1))) by { + if j < i { + assert(done.restrict(keys_of(p0, owner, j, j + 1)) =~= old_done.restrict(keys_of(p0, owner, j, j + 1))); + } else { + assert(done.restrict(keys_of(p0, owner, j, j + 1)) =~= rv); + } + } + } + i += 1; + } + Tracked(done) + }, + Tracked(perms), + Ghost(owner), + Ghost(post), + ) +} + +/// A base `*mut` that can cross into the task closures. Sound only because +/// callers partition the allocation by item index; every dereference site +/// restates that. +/// +/// The `Clone`/`Copy` derives are kept; the `Debug` derive and the `Send`/`Sync` impls are outside +/// `verus!` below. +#[derive(Clone, Copy)] +#[verifier::allow(autoderive_clone_without_spec)] +pub struct SendPtr(pub *mut T); + +impl SendPtr { + /// Offset the base by `n` elements. + /// + /// # Safety + /// `n` stays inside the allocation, and any write targets a slot no + /// concurrent task touches. + /// + /// Verified: the address does not wrap, which the caller shows with a permission covering element `n`. + #[inline] + pub const unsafe fn add(&self, n: usize) -> (r: *mut T) + requires + self.0@.addr + n * size_of::() <= usize::MAX, + ensures + r == ptr_at(self.0, n as int), + { + // SAFETY: the caller keeps `n` inside the allocation. + unsafe { self.0.add(n) } + } + + /// Rebuild the `len`-element slice at element offset `off`. + /// + /// # Safety + /// `off`/`len` are in bounds and disjoint from every other concurrent task's + /// slice, and the underlying buffer outlives `'a`. + /// + /// Verified: the caller lends the permissions of exactly the elements `off..off + len` for `'a` + /// (a ghost argument), which makes the slice in bounds and exclusive. + #[inline] + pub unsafe fn slice<'a>(&self, off: usize, len: usize, Tracked(perms): Tracked<&'a mut Map>>) -> (s: &'a mut [T]) + requires + owns(*old(perms), self.0, off as int, len as int), + ensures + s@ == vals(*old(perms), off as int, len as int), + owns(*final(perms), self.0, off as int, len as int), + final(s)@ == vals(*final(perms), off as int, len as int), + { + proof { + lemma_mul_le(off as int, (off + len) as int, size_of::() as int); + } + // SAFETY: the caller guarantees `off..off + len` is in bounds, borrowed by no other task, and alive for `'a`. + unsafe { from_raw_parts_mut(self.0.add(off), len, Ghost(self.0), Ghost(off as int), Tracked(perms)) } + } +} + +/// A mutable slice in raw parts, its address and length; its permissions travel beside it as a ghost +/// argument. +/// +/// Not in production: the verified driver passes its buffer this way where production passes +/// `&mut [F64]`, because the in-place message is that very address, and Verus does not relate the +/// pointers of two `as_mut_ptr` calls on one slice. `len` and `as_mut_ptr` keep the call sites' text. +#[derive(Clone, Copy)] +#[verifier::allow(autoderive_clone_without_spec)] +pub struct RawMut { + pub ptr: *mut T, + pub len: usize, +} + +impl RawMut { + /// The slice's raw parts, and the permissions it lends for `'a`. + pub fn new<'a>(data: &'a mut [T]) -> (r: (Self, Tracked<&'a mut Map>>)) + ensures + r.0.len == old(data)@.len(), + owns(*r.1@, r.0.ptr, 0, old(data)@.len() as int), + vals(*r.1@, 0, old(data)@.len() as int) == old(data)@, + owns(*final(r.1@), r.0.ptr, 0, old(data)@.len() as int) ==> final(data)@ == vals(*final(r.1@), 0, old(data)@.len() as int), + { + let len = data.len(); + let (ptr, perms) = slice_as_mut_ptr(data); + (Self { ptr, len }, perms) + } + + pub fn len(&self) -> (r: usize) + ensures + r == self.len, + { + self.len + } + + pub fn as_mut_ptr(&self) -> (r: *mut T) + ensures + r == self.ptr, + { + self.ptr + } + + /// The slice again, borrowing its permissions for `'a`. + pub fn as_mut_slice<'a>(&self, Tracked(perms): Tracked<&'a mut Map>>) -> (s: &'a mut [T]) + requires + owns(*old(perms), self.ptr, 0, self.len as int), + ensures + s@ == vals(*old(perms), 0, self.len as int), + owns(*final(perms), self.ptr, 0, self.len as int), + final(s)@ == vals(*final(perms), 0, self.len as int), + { + proof { + assert(ptr_at(self.ptr, 0) == self.ptr); + } + from_raw_parts_mut(self.ptr, self.len, Ghost(self.ptr), Ghost(0), Tracked(perms)) + } +} + +/// A `chunks_mut` view that can be handed to tasks: `chunk(i)` is item `i`'s +/// slice. Composes to any number of buffers, which the zip-based helpers do not: +/// a kernel writing four tables at two different widths builds one of these per +/// table and indexes them all by the same item. +#[derive(Clone, Copy)] +#[verifier::allow(autoderive_clone_without_spec)] +pub struct Chunks { + pub base: SendPtr, + pub width: usize, + pub len: usize, +} + +/// The chunk that element `k` falls in. +pub open spec fn chunk_of(width: nat) -> spec_fn(int) -> int { + |k: int| k / (width as int) +} + +/// The length of chunk `i` of a `len`-element buffer cut every `width` elements. +pub open spec fn chunk_len(len: nat, width: nat, i: int) -> int { + if i * width + width <= len { + width as int + } else { + len - i * width + } +} + +impl Chunks { + /// View `data` as `len.div_ceil(width)` chunks of `width` (the last shorter). + /// + /// Rewritten: the `assert!` is a precondition, and the view comes with the permissions of `data` + /// (production's `data.as_mut_ptr()` is [`slice_as_mut_ptr`]), which `data` lends for their lifetime. + pub fn new<'a>(data: &'a mut [T], width: usize) -> (r: (Self, Tracked<&'a mut Map>>)) + requires + width > 0, + ensures + r.0.width == width, + r.0.len == old(data)@.len(), + owns(*r.1@, r.0.base.0, 0, old(data)@.len() as int), + vals(*r.1@, 0, old(data)@.len() as int) == old(data)@, + owns(*final(r.1@), r.0.base.0, 0, old(data)@.len() as int) ==> final(data)@ == vals(*final(r.1@), 0, old(data)@.len() as int), + { + // Rewritten: the length is read before the slice lends its permissions. + let len = data.len(); + let (ptr, perms) = slice_as_mut_ptr(data); + (Self { base: SendPtr(ptr), width, len }, perms) + } + + /// Number of chunks. + pub const fn count(&self) -> (r: usize) + requires + self.width > 0, + ensures + r == (self.len + self.width - 1) / (self.width as int), + { + self.len.div_ceil(self.width) + } + + /// Chunk `i`. + /// + /// # Safety + /// `i < self.count()`, no other live borrow covers chunk `i`, and the + /// underlying buffer outlives `'a`. Calling this once per `i` inside a + /// [`for_each`] body satisfies all three. + /// + /// Verified: the caller lends the permissions of chunk `i` for `'a` (a ghost argument); the + /// `debug_assert!` is proven. + #[inline] + pub unsafe fn get<'a>(&self, i: usize, Tracked(perms): Tracked<&'a mut Map>>) -> (s: &'a mut [T]) + requires + self.width > 0, + i < (self.len + self.width - 1) / (self.width as int), + owns(*old(perms), self.base.0, i * self.width, chunk_len(self.len as nat, self.width as nat, i as int)), + ensures + s@ == vals(*old(perms), i * self.width, chunk_len(self.len as nat, self.width as nat, i as int)), + owns(*final(perms), self.base.0, i * self.width, chunk_len(self.len as nat, self.width as nat, i as int)), + final(s)@ == vals(*final(perms), i * self.width, chunk_len(self.len as nat, self.width as nat, i as int)), + { + proof { + lemma_chunk_start(self.len as nat, self.width as nat, i as int); + } + let start = i * self.width; + assert(start < self.len); + // SAFETY: `i < count()` puts `start` below `len`, so `start..start + min(width, len - start)` lies in + // the slice `new` took; the caller guarantees no other borrow of it and that it outlives `'a`. + unsafe { self.base.slice(start, self.width.min(self.len - start), Tracked(perms)) } + } +} + +/// A chunk below the count starts inside the buffer. +proof fn lemma_chunk_start(len: nat, width: nat, i: int) + requires + width > 0, + 0 <= i < (len + width - 1) / (width as int), + ensures + i * width < len, + 0 < chunk_len(len, width, i) <= width, + i * width + chunk_len(len, width, i) <= len, +{ + vstd::arithmetic::div_mod::lemma_fundamental_div_mod((len + width - 1) as int, width as int); + vstd::arithmetic::div_mod::lemma_mod_bound((len + width - 1) as int, width as int); + let q = (len + width - 1) / (width as int); + assert(i * width + width <= q * width) by (nonlinear_arith) + requires + i + 1 <= q, + width > 0, + ; +} + +/// The permissions of a sub-range of an owned range, and their values. +pub proof fn lemma_restrict_owns(p: Map>, o: *mut T, total: int, keys: Set, lo: int, n: int) + requires + owns(p, o, 0, total), + 0 <= lo, + 0 <= n, + lo + n <= total, + forall|k: int| #[trigger] keys.contains(k) <==> lo <= k < lo + n, + ensures + owns(p.restrict(keys), o, lo, n), + vals(p.restrict(keys), lo, n) == vals(p, 0, total).subrange(lo, lo + n), +{ + lemma_mul_le(lo + n, total, size_of::() as int); + assert forall|k: int| #[trigger] p.restrict(keys).dom().contains(k) <==> lo <= k < lo + n by {} + assert(vals(p.restrict(keys), lo, n) =~= vals(p, 0, total).subrange(lo, lo + n)); +} + +/// `a <= b` scales by a non-negative factor. +pub proof fn lemma_mul_le(a: int, b: int, c: int) + requires + a <= b, + 0 <= c, + ensures + a * c <= b * c, +{ + assert(a * c <= b * c) by (nonlinear_arith) + requires + a <= b, + 0 <= c, + ; +} + +/// An index below `len` lies in a chunk below the count. +proof fn lemma_chunk_index(k: int, len: int, w: int) + requires + 0 <= k < len, + w > 0, + ensures + 0 <= k / w < (len + w - 1) / w, +{ + vstd::arithmetic::div_mod::lemma_fundamental_div_mod(k, w); + vstd::arithmetic::div_mod::lemma_div_pos_is_pos(k, w); + let q = k / w; + vstd::arithmetic::div_mod::lemma_mod_bound(k, w); + assert((q + 1) * w <= len + w - 1) by (nonlinear_arith) + requires + k == w * q + k % w, + 0 <= k % w < w, + k < len, + ; + vstd::arithmetic::div_mod::lemma_div_is_ordered((q + 1) * w, len + w - 1, w); + vstd::arithmetic::div_mod::lemma_div_multiples_vanish(q + 1, w); + assert((q + 1) * w == w * (q + 1)) by (nonlinear_arith); +} + +/// Parallel `data.chunks_mut(chunk).enumerate().for_each(f)`; the final chunk may +/// be shorter. +/// +/// Specification: `f(i, chunk_i)` runs once for every chunk, concurrently; `post(i, old, new)` holds of +/// every chunk's contents before and after its call, given that it holds of every call of `f`. +/// Rewritten: `chunk > 0` is a precondition (production's `Chunks::new` asserts it), and the +/// permissions pass through [`for_each`] (ghost additions). +pub fn chunks_mut(data: &mut [T], chunk: usize, f: F, Ghost(post): Ghost, Seq) -> bool>) +where + F: Fn(usize, &mut [T]) + Sync, + requires + chunk > 0, + forall|i: usize, s: &mut [T]| + i < (old(data)@.len() + chunk - 1) / (chunk as int) && s@ == old(data)@.subrange(i * chunk, i * chunk + chunk_len(old(data)@.len(), chunk as nat, i as int)) + ==> #[trigger] f.requires((i, s)), + forall|i: usize, s: &mut [T]| + #[trigger] f.ensures((i, s), ()) ==> final(s)@.len() == s@.len() && post(i as int, s@, final(s)@), + ensures + final(data)@.len() == old(data)@.len(), + forall|i: int| + 0 <= i < (old(data)@.len() + chunk - 1) / (chunk as int) ==> #[trigger] post( + i, + old(data)@.subrange(i * chunk, i * chunk + chunk_len(old(data)@.len(), chunk as nat, i)), + final(data)@.subrange(i * chunk, i * chunk + chunk_len(old(data)@.len(), chunk as nat, i)), + ), +{ + let ghost d0 = data@; + let ghost len = d0.len() as int; + let (view, Tracked(perms)) = Chunks::new(data, chunk); + let ghost base = view.base; + let ghost owner = chunk_of(chunk as nat); + let ghost item_post = |i: int, m: Map>| + { + let lo = i * chunk; + let n = chunk_len(d0.len(), chunk as nat, i); + owns(m, base.0, lo, n) && post(i, d0.subrange(lo, lo + n), vals(m, lo, n)) + }; + let ghost pre = *perms; + let tracked all = perms.tracked_remove_keys(perms.dom()); + let ghost p0 = all; + let count = view.count(); + proof { + assert(p0 =~= pre); + assert forall|k: int| #[trigger] p0.dom().contains(k) implies 0 <= owner(k) < count by { + lemma_chunk_index(k, len, chunk as int); + } + assert forall|i: usize, k: int| i < count implies (#[trigger] keys_of(p0, owner, i as int, i + 1).contains(k) <==> i + * chunk <= k < i * chunk + chunk_len(d0.len(), chunk as nat, i as int)) by { + lemma_chunk_start(d0.len(), chunk as nat, i as int); + lemma_chunk_keys(k, i as int, chunk as int); + } + } + assert(owns(p0, base.0, 0, len)); + assert(vals(p0, 0, len) == d0); + assert(count == (len + chunk - 1) / (chunk as int)); + assert(view.base == base && view.width == chunk && view.len == len); + assert(forall|i: usize, s: &mut [T]| + i < count && s@ == d0.subrange(i * chunk, i * chunk + chunk_len(d0.len(), chunk as nat, i as int)) + ==> #[trigger] f.requires((i, s))); + assert(forall|i: usize, s: &mut [T]| + #[trigger] f.ensures((i, s), ()) ==> final(s)@.len() == s@.len() && post(i as int, s@, final(s)@)); + let Tracked(out) = for_each( + count, + |i: usize, tp: Tracked>>| -> (r: Tracked>>) + requires + i < count, + tp@ == claimed(p0, owner, i as int, i + 1), + chunk > 0, + view.base == base, + view.width == chunk, + view.len == len, + count == (len + chunk - 1) / (chunk as int), + owns(p0, base.0, 0, len), + vals(p0, 0, len) == d0, + forall|i: usize, k: int| i < count ==> (#[trigger] keys_of(p0, owner, i as int, i + 1).contains(k) <==> i + * chunk <= k < i * chunk + chunk_len(d0.len(), chunk as nat, i as int)), + forall|i: usize, s: &mut [T]| + i < count && s@ == d0.subrange(i * chunk, i * chunk + chunk_len(d0.len(), chunk as nat, i as int)) + ==> #[trigger] f.requires((i, s)), + forall|i: usize, s: &mut [T]| + #[trigger] f.ensures((i, s), ()) ==> final(s)@.len() == s@.len() && post(i as int, s@, final(s)@), + ensures + r@.dom() == keys_of(p0, owner, i as int, i + 1), + item_post(i as int, r@), + { + let tracked mut m = tp.get(); + let ghost lo = i * chunk; + let ghost n = chunk_len(d0.len(), chunk as nat, i as int); + proof { + lemma_chunk_start(d0.len(), chunk as nat, i as int); + lemma_restrict_owns(p0, base.0, len, keys_of(p0, owner, i as int, i + 1), lo, n); + } + // SAFETY: distinct `i` give disjoint in-bounds chunks, and `data` stays + // borrowed for the whole dispatch. + f(i, unsafe { view.get(i, Tracked(&mut m)) }); + proof { + assert(m.dom() =~= keys_of(p0, owner, i as int, i + 1)); + } + Tracked(m) + }, + Tracked(all), + Ghost(owner), + Ghost(item_post), + ); + proof { + perms.tracked_union_prefer_right(out); + assert(perms.dom() =~= out.dom()); + assert forall|k: int| 0 <= k < len implies (#[trigger] perms[k]).ptr() == ptr_at(base.0, k) && perms[k].is_init() by { + let i = owner(k); + lemma_chunk_index(k, len, chunk as int); + assert(p0.dom().contains(k)); + let iu = i as usize; + assert(keys_of(p0, owner, iu as int, iu + 1).contains(k)); + let mi = out.restrict(keys_of(p0, owner, i, i + 1)); + assert(item_post(i, mi)); + assert(mi[k] == out[k]); + assert(perms[k] == out[k]); + } + assert(owns(*perms, base.0, 0, len)); + assert forall|i: int| 0 <= i < count implies #[trigger] post( + i, + d0.subrange(i * chunk, i * chunk + chunk_len(d0.len(), chunk as nat, i)), + vals(*perms, 0, len).subrange(i * chunk, i * chunk + chunk_len(d0.len(), chunk as nat, i)), + ) by { + let lo = i * chunk; + let n = chunk_len(d0.len(), chunk as nat, i); + let mi = out.restrict(keys_of(p0, owner, i, i + 1)); + assert(item_post(i, mi)); + lemma_chunk_start(d0.len(), chunk as nat, i); + assert(vals(mi, lo, n) =~= vals(*perms, 0, len).subrange(lo, lo + n)); + } + } +} + +/// Key `k` falls in chunk `i` exactly when it lies in `i * w .. (i + 1) * w`. +pub proof fn lemma_chunk_keys(k: int, i: int, w: int) + requires + w > 0, + ensures + (i <= k / w < i + 1) <==> (i * w <= k < i * w + w), +{ + vstd::arithmetic::div_mod::lemma_fundamental_div_mod(k, w); + vstd::arithmetic::div_mod::lemma_mod_bound(k, w); + if i <= k / w < i + 1 { + assert(k / w == i); + } else if i * w <= k < i * w + w { + vstd::arithmetic::div_mod::lemma_div_is_ordered(i * w, k, w); + vstd::arithmetic::div_mod::lemma_div_multiples_vanish(i, w); + assert(k / w <= i) by { + vstd::arithmetic::div_mod::lemma_div_is_ordered(k, i * w + w - 1, w); + assert(i * w + w - 1 == i * w + (w - 1)); + vstd::arithmetic::div_mod::lemma_div_multiples_vanish_fancy(i, w - 1, w); + } + } +} + +} // verus! + +impl core::fmt::Debug for SendPtr { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.debug_tuple("SendPtr").field(&self.0).finish() + } +} + +// SAFETY: `SendPtr` grants no access by itself; every dereference is inside an +// `unsafe` block whose comment establishes that the range being touched belongs +// to exactly one task. +unsafe impl Send for SendPtr {} +// SAFETY: sharing the pointer value is as harmless as sending it, for the reason above. +unsafe impl Sync for SendPtr {} diff --git a/verification/verus/src/phi8_tower.rs b/verification/verus/src/phi8_tower.rs new file mode 100644 index 0000000000..9fccff0942 --- /dev/null +++ b/verification/verus/src/phi8_tower.rs @@ -0,0 +1,484 @@ +//! The embedding `φ₈: GF(2^8) → E`, whose image lies in `K`. +//! +//! The executable functions are `crates/primitives/src/field/phi8_tower.rs`, copied with the same bodies; +//! `tests/equivalence/phi8_tower.rs` checks the two agree. +//! +//! Specification: [`phi8`] is the GF(2)-linear map sending the byte `x^i` (bit `i`) to the production +//! constant `PHI_8_BASIS[i]` of `K` ([`phi8_basis`]), and `phi8_192(a)` is `φ₈(a)` embedded in `E` +//! ([`e_from_k`]). The theorems: `φ₈` is a ring homomorphism from `GF(2^8)` ([`f8_mul`]) into `K` +//! ([`k_mul`]) and into `E` ([`e_mul`]), sends one to one, and is injective, so its image is a subfield +//! of `K` with 256 elements. +//! +//! The proof: `φ₈` commutes with the multiplication by `x`, `φ₈(x a) = g φ₈(a)` with `g = φ₈(x)` +//! ([`lemma_phi8_mulx`]); by linearity this needs it only on the eight bytes `x^i`, eight products in `K` +//! of concrete constants, evaluated by [`clmul64_closed`]. Then `φ₈(x^i b) = φ₈(x^i) φ₈(b)` by induction +//! on `i`, and the general product by linearity in the left factor. +use crate::clmul::*; +use crate::gf2_64::*; +use crate::gf2_64x3::*; +use crate::gf2_8::*; +use vstd::prelude::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// `φ₈(x^i)` for `i < 8`: the production `PHI_8_BASIS[i]`. +pub open spec fn phi8_basis(i: u8) -> u64 { + if i == 0 { + 0x0000000000000001u64 + } else if i == 1 { + 0x033ce8beddc8a656u64 + } else if i == 2 { + 0x512620375ed2a108u64 + } else if i == 3 { + 0x0c9e636090aafc01u64 + } else if i == 4 { + 0xba4f3cd82801769cu64 + } else if i == 5 { + 0xba26e7904adb4a47u64 + } else if i == 6 { + 0x467698598926dc01u64 + } else { + 0x4418ae808b28bdd0u64 + } +} + +/// `[a_i] φ₈(x^i)`. +pub open spec fn phi8_term(a: u8, i: u8) -> u64 { + if (a >> i) & 1 == 1 { + phi8_basis(i) + } else { + 0 + } +} + +/// `φ₈` of the low `n` coefficients of `a`: the XOR of `φ₈(x^i)` over the set bits `i < n` of `a`. +pub open spec fn phi8_upto(a: u8, n: nat) -> u64 + decreases n, +{ + if n == 0 { + 0 + } else { + phi8_upto(a, (n - 1) as nat) ^ phi8_term(a, (n - 1) as u8) + } +} + +/// `φ₈(a)`, in `K`. +pub open spec fn phi8(a: u8) -> u64 { + phi8_upto(a, 8) +} + +/// [`phi8`] as a bit-vector term. +pub open spec fn phi8_closed(a: u8) -> u64 { + phi8_term(a, 0) ^ phi8_term(a, 1) ^ phi8_term(a, 2) ^ phi8_term(a, 3) ^ phi8_term(a, 4) ^ phi8_term(a, 5) + ^ phi8_term(a, 6) ^ phi8_term(a, 7) +} + +/// `φ₈(a)`, in `E`. +pub open spec fn phi8_e(a: u8) -> F192 { + e_from_k(phi8(a)) +} + +// --------------------------------------------------------------------------------------------- +// Linearity +// --------------------------------------------------------------------------------------------- +pub proof fn lemma_phi8_closed(a: u8) + ensures + phi8(a) == phi8_closed(a), +{ + reveal_with_fuel(phi8_upto, 9); + let t0 = phi8_term(a, 0); + assert(0u64 ^ t0 == t0) by (bit_vector); +} + +/// `φ₈` is GF(2)-linear: `φ₈(a + b) = φ₈(a) + φ₈(b)`. +pub proof fn lemma_phi8_xor(a: u8, b: u8) + ensures + phi8(a ^ b) == phi8(a) ^ phi8(b), +{ + lemma_phi8_closed(a); + lemma_phi8_closed(b); + lemma_phi8_closed(a ^ b); + assert(phi8_closed(a ^ b) == phi8_closed(a) ^ phi8_closed(b)) by (bit_vector); +} + +/// `φ₈(0) = 0` and `φ₈(1) = 1`. +pub proof fn lemma_phi8_zero_one() + ensures + phi8(0) == 0, + phi8(1) == 1, +{ + lemma_phi8_closed(0); + lemma_phi8_closed(1); + assert(phi8_closed(0) == 0 && phi8_closed(1) == 1) by (bit_vector); +} + +/// `φ₈` is injective: the eight basis images are GF(2)-independent. +pub proof fn lemma_phi8_injective(a: u8, b: u8) + ensures + phi8(a) == phi8(b) ==> a == b, +{ + lemma_phi8_closed(a); + lemma_phi8_closed(b); + assert(phi8_closed(a) == phi8_closed(b) ==> a == b) by (bit_vector); +} + +/// `φ₈(a)` is zero only at zero. +pub proof fn lemma_phi8_nonzero(a: u8) + requires + a != 0, + ensures + phi8(a) != 0, +{ + lemma_phi8_injective(a, 0); + lemma_phi8_zero_one(); +} + +/// `φ₈(x^i)` is the basis constant. +pub proof fn lemma_phi8_monomial(i: u8) + requires + i < 8, + ensures + phi8(1u8 << i) == phi8_basis(i), +{ + lemma_phi8_closed(1u8 << i); + assert(i < 8 ==> phi8_closed(1u8 << i) == phi8_basis(i)) by (bit_vector); +} + +// --------------------------------------------------------------------------------------------- +// Products of concrete elements of K +// --------------------------------------------------------------------------------------------- +/// `[a_i] * b * x^i`, with a machine-word index. +pub open spec fn term64(a: u64, b: u128, i: u64) -> u128 { + if (a >> i) & 1 == 1 { + b << (i as u128) + } else { + 0 + } +} + +/// The 64 partial products of [`clmul`], XORed: a bit-vector term. +pub open spec fn clmul64_closed(a: u64, b: u128) -> u128 { + term64(a, b, 0) ^ term64(a, b, 1) ^ term64(a, b, 2) ^ term64(a, b, 3) ^ term64(a, b, 4) ^ term64(a, b, 5) + ^ term64(a, b, 6) ^ term64(a, b, 7) ^ term64(a, b, 8) ^ term64(a, b, 9) ^ term64(a, b, 10) + ^ term64(a, b, 11) ^ term64(a, b, 12) ^ term64(a, b, 13) ^ term64(a, b, 14) ^ term64(a, b, 15) + ^ term64(a, b, 16) ^ term64(a, b, 17) ^ term64(a, b, 18) ^ term64(a, b, 19) ^ term64(a, b, 20) + ^ term64(a, b, 21) ^ term64(a, b, 22) ^ term64(a, b, 23) ^ term64(a, b, 24) ^ term64(a, b, 25) + ^ term64(a, b, 26) ^ term64(a, b, 27) ^ term64(a, b, 28) ^ term64(a, b, 29) ^ term64(a, b, 30) + ^ term64(a, b, 31) ^ term64(a, b, 32) ^ term64(a, b, 33) ^ term64(a, b, 34) ^ term64(a, b, 35) + ^ term64(a, b, 36) ^ term64(a, b, 37) ^ term64(a, b, 38) ^ term64(a, b, 39) ^ term64(a, b, 40) + ^ term64(a, b, 41) ^ term64(a, b, 42) ^ term64(a, b, 43) ^ term64(a, b, 44) ^ term64(a, b, 45) + ^ term64(a, b, 46) ^ term64(a, b, 47) ^ term64(a, b, 48) ^ term64(a, b, 49) ^ term64(a, b, 50) + ^ term64(a, b, 51) ^ term64(a, b, 52) ^ term64(a, b, 53) ^ term64(a, b, 54) ^ term64(a, b, 55) + ^ term64(a, b, 56) ^ term64(a, b, 57) ^ term64(a, b, 58) ^ term64(a, b, 59) ^ term64(a, b, 60) + ^ term64(a, b, 61) ^ term64(a, b, 62) ^ term64(a, b, 63) +} + +/// [`clmul`] is [`clmul64_closed`]. +pub proof fn lemma_clmul64_closed(a: u64, b: u128) + ensures + clmul(a, b) == clmul64_closed(a, b), +{ + reveal_with_fuel(clmul_upto, 65); + let t0 = term(a, b, 0); + assert(0u128 ^ t0 == t0) by (bit_vector); +} + +/// The product in `K` as a bit-vector term. +pub open spec fn k_mul_closed(a: u64, b: u64) -> u64 { + reduce_formula(clmul64_closed(a, b as u128)) +} + +pub proof fn lemma_k_mul_closed(a: u64, b: u64) + ensures + k_mul(a, b) == k_mul_closed(a, b), +{ + lemma_clmul64_closed(a, b as u128); + lemma_k_mod(clmul(a, b as u128)); +} + +// --------------------------------------------------------------------------------------------- +// φ₈ is multiplicative +// --------------------------------------------------------------------------------------------- +/// `g φ₈(x^i) = φ₈(x * x^i)` for the eight monomials, `g = φ₈(x)`: eight products of constants. +proof fn lemma_phi8_mulx_monomial(i: u8) + requires + i < 8, + ensures + k_mul(phi8_basis(1), phi8_basis(i)) == phi8(mulx8(1u8 << i)), +{ + lemma_k_mul_closed(phi8_basis(1), phi8_basis(i)); + lemma_phi8_closed(mulx8(1u8 << i)); + let g = phi8_basis(1); + if i == 0 { + assert(k_mul_closed(0x033ce8beddc8a656u64, 0x0000000000000001u64) == 0x033ce8beddc8a656u64) + by (bit_vector); + assert(phi8_closed(mulx8(1u8 << 0u8)) == 0x033ce8beddc8a656u64) by (bit_vector); + } else if i == 1 { + assert(k_mul_closed(0x033ce8beddc8a656u64, 0x033ce8beddc8a656u64) == 0x512620375ed2a108u64) + by (bit_vector); + assert(phi8_closed(mulx8(1u8 << 1u8)) == 0x512620375ed2a108u64) by (bit_vector); + } else if i == 2 { + assert(k_mul_closed(0x033ce8beddc8a656u64, 0x512620375ed2a108u64) == 0x0c9e636090aafc01u64) + by (bit_vector); + assert(phi8_closed(mulx8(1u8 << 2u8)) == 0x0c9e636090aafc01u64) by (bit_vector); + } else if i == 3 { + assert(k_mul_closed(0x033ce8beddc8a656u64, 0x0c9e636090aafc01u64) == 0xba4f3cd82801769cu64) + by (bit_vector); + assert(phi8_closed(mulx8(1u8 << 3u8)) == 0xba4f3cd82801769cu64) by (bit_vector); + } else if i == 4 { + assert(k_mul_closed(0x033ce8beddc8a656u64, 0xba4f3cd82801769cu64) == 0xba26e7904adb4a47u64) + by (bit_vector); + assert(phi8_closed(mulx8(1u8 << 4u8)) == 0xba26e7904adb4a47u64) by (bit_vector); + } else if i == 5 { + assert(k_mul_closed(0x033ce8beddc8a656u64, 0xba26e7904adb4a47u64) == 0x467698598926dc01u64) + by (bit_vector); + assert(phi8_closed(mulx8(1u8 << 5u8)) == 0x467698598926dc01u64) by (bit_vector); + } else if i == 6 { + assert(k_mul_closed(0x033ce8beddc8a656u64, 0x467698598926dc01u64) == 0x4418ae808b28bdd0u64) + by (bit_vector); + assert(phi8_closed(mulx8(1u8 << 6u8)) == 0x4418ae808b28bdd0u64) by (bit_vector); + } else { + // x^8 = x^4 + x^3 + x + 1. + assert(k_mul_closed(0x033ce8beddc8a656u64, 0x4418ae808b28bdd0u64) == 0xb5edb70665632ccau64) + by (bit_vector); + assert(phi8_closed(mulx8(1u8 << 7u8)) == 0xb5edb70665632ccau64) by (bit_vector); + } +} + +/// `φ₈(x a) = g φ₈(a)` on the low `n` coefficients of `a`, by linearity from the monomials. +proof fn lemma_phi8_mulx_upto(a: u8, n: u16) + requires + n <= 8, + ensures + phi8(mulx8(low8(a, n))) == k_mul(phi8_basis(1), phi8(low8(a, n))), + decreases n, +{ + let g = phi8_basis(1); + if n == 0 { + assert(((a as u16) & sub(1u16 << 0u16, 1)) as u8 == 0 && mulx8(0) == 0) by (bit_vector); + lemma_phi8_zero_one(); + lemma_k_mul_zero(g); + } else { + let i = sub(n, 1); + lemma_phi8_mulx_upto(a, i); + let e: u8 = if (a >> (i as u8)) & 1 == 1 { + 1u8 << (i as u8) + } else { + 0 + }; + let lo = low8(a, i); + assert(i < 8 ==> ((a as u16) & sub(1u16 << add(i, 1), 1)) as u8 == ((a as u16) & sub(1u16 << i, 1)) as u8 + ^ (if (a >> (i as u8)) & 1 == 1 { + 1u8 << (i as u8) + } else { + 0u8 + })) by (bit_vector); + assert(low8(a, n) == lo ^ e); + assert(mulx8(lo ^ e) == mulx8(lo) ^ mulx8(e)) by (bit_vector); + lemma_phi8_xor(mulx8(lo), mulx8(e)); + lemma_phi8_xor(lo, e); + lemma_k_mul_xor_right(g, phi8(lo), phi8(e)); + if (a >> (i as u8)) & 1 == 1 { + lemma_phi8_mulx_monomial(i as u8); + lemma_phi8_monomial(i as u8); + } else { + assert(mulx8(0) == 0) by (bit_vector); + lemma_phi8_zero_one(); + lemma_k_mul_zero(g); + } + } +} + +/// `φ₈` commutes with the multiplication by `x`: `φ₈(x a) = φ₈(x) φ₈(a)`. +pub proof fn lemma_phi8_mulx(a: u8) + ensures + phi8(mulx8(a)) == k_mul(phi8(2), phi8(a)), +{ + lemma_phi8_mulx_upto(a, 8); + assert(((a as u16) & sub(1u16 << 8u16, 1)) as u8 == a) by (bit_vector); + assert(2u8 == 1u8 << 1u8) by (bit_vector); + lemma_phi8_monomial(1); +} + +/// `φ₈(x^i b) = φ₈(x^i) φ₈(b)`. +proof fn lemma_phi8_mulx_pow(b: u8, i: u8) + requires + i < 8, + ensures + phi8(mulx8_pow(b, i as nat)) == k_mul(phi8(1u8 << i), phi8(b)), + decreases i, +{ + if i == 0 { + assert(1u8 << 0u8 == 1) by (bit_vector); + lemma_phi8_zero_one(); + lemma_k_mul_one(phi8(b)); + } else { + let j = (i - 1) as u8; + lemma_phi8_mulx_pow(b, j); + let m = mulx8_pow(b, j as nat); + assert(mulx8_pow(b, i as nat) == mulx8(m)); + lemma_phi8_mulx(m); + // g (φ₈(x^j) φ₈(b)) = (g φ₈(x^j)) φ₈(b) = φ₈(x^(j+1)) φ₈(b). + lemma_k_mul_assoc(phi8(2), phi8(1u8 << j), phi8(b)); + lemma_phi8_mulx(1u8 << j); + assert(j < 7 ==> mulx8(1u8 << j) == 1u8 << add(j, 1)) by (bit_vector); + } +} + +/// `φ₈(a b) = φ₈(a) φ₈(b)` on the low `n` coefficients of `a`. +proof fn lemma_phi8_mul_upto(a: u8, b: u8, n: u16) + requires + n <= 8, + ensures + phi8(f8_mul(low8(a, n), b)) == k_mul(phi8(low8(a, n)), phi8(b)), + decreases n, +{ + if n == 0 { + assert(((a as u16) & sub(1u16 << 0u16, 1)) as u8 == 0) by (bit_vector); + lemma_f8_mul_zero(b); + lemma_phi8_zero_one(); + lemma_k_mul_zero(phi8(b)); + lemma_k_mul_comm(0, phi8(b)); + } else { + let i = sub(n, 1); + lemma_phi8_mul_upto(a, b, i); + let e: u8 = if (a >> (i as u8)) & 1 == 1 { + 1u8 << (i as u8) + } else { + 0 + }; + let lo = low8(a, i); + assert(i < 8 ==> ((a as u16) & sub(1u16 << add(i, 1), 1)) as u8 == ((a as u16) & sub(1u16 << i, 1)) as u8 + ^ (if (a >> (i as u8)) & 1 == 1 { + 1u8 << (i as u8) + } else { + 0u8 + })) by (bit_vector); + assert(low8(a, n) == lo ^ e); + // (lo + e) b = lo b + e b, in both rings. + lemma_f8_mul_comm(lo ^ e, b); + lemma_f8_mul_xor_right(b, lo, e); + lemma_f8_mul_comm(b, lo); + lemma_f8_mul_comm(b, e); + lemma_phi8_xor(f8_mul(lo, b), f8_mul(e, b)); + lemma_phi8_xor(lo, e); + lemma_k_mul_xor_left(phi8(lo), phi8(e), phi8(b)); + if (a >> (i as u8)) & 1 == 1 { + lemma_f8_mul_monomial(b, i as nat); + lemma_phi8_mulx_pow(b, i as u8); + } else { + lemma_f8_mul_zero(b); + lemma_phi8_zero_one(); + lemma_k_mul_zero(phi8(b)); + lemma_k_mul_comm(0, phi8(b)); + } + } +} + +/// `φ₈` is multiplicative into `K`: `φ₈(a b) = φ₈(a) φ₈(b)`. +pub proof fn lemma_phi8_mul(a: u8, b: u8) + ensures + phi8(f8_mul(a, b)) == k_mul(phi8(a), phi8(b)), +{ + lemma_phi8_mul_upto(a, b, 8); + assert(((a as u16) & sub(1u16 << 8u16, 1)) as u8 == a) by (bit_vector); +} + +/// `φ₈` is a ring homomorphism from `GF(2^8)` into `E`, injective, with image in `K`. +pub proof fn lemma_phi8_e_homomorphism(a: u8, b: u8) + ensures + phi8_e(f8_mul(a, b)) == e_mul(phi8_e(a), phi8_e(b)), + phi8_e(a ^ b) == e_add(phi8_e(a), phi8_e(b)), + phi8_e(1) == F192::ONE, + phi8_e(0) == F192::ZERO, + phi8_e(a) == phi8_e(b) ==> a == b, + phi8_e(a).c1 == 0 && phi8_e(a).c2 == 0, +{ + lemma_phi8_mul(a, b); + lemma_e_from_k_mul(phi8(a), phi8(b)); + lemma_phi8_xor(a, b); + assert(0u64 ^ 0u64 == 0u64) by (bit_vector); + lemma_phi8_zero_one(); + lemma_phi8_injective(a, b); +} + +// --------------------------------------------------------------------------------------------- +// Executable code: `crates/primitives/src/field/phi8_tower.rs` +// --------------------------------------------------------------------------------------------- +/// φ₈(2ᵏ) for k ∈ [0,8): the images of the GF(2⁸) polynomial basis. All in +/// `F64` (`c1 == c2 == 0`). +const PHI_8_BASIS: [u64; 8] = [ + 0x0000000000000001, + 0x033ce8beddc8a656, + 0x512620375ed2a108, + 0x0c9e636090aafc01, + 0xba4f3cd82801769c, + 0xba26e7904adb4a47, + 0x467698598926dc01, + 0x4418ae808b28bdd0, +]; + +const fn build_phi8_table_192() -> (table: [F192; 256]) + ensures + forall|v: int| 0 <= v < 256 ==> #[trigger] table[v] == phi8_e(v as u8), +{ + let mut table = [F192::ZERO; 256]; + let mut value = 1; + assert(table[0] == phi8_e(0)) by { + lemma_phi8_zero_one(); + } + while value < table.len() + invariant + 1 <= value <= 256, + forall|v: int| 0 <= v < value ==> #[trigger] table[v] == phi8_e(v as u8), + decreases 256 - value, + { + let mut c0 = 0u64; + let mut bit = 0; + while bit < PHI_8_BASIS.len() + invariant + 1 <= value < 256, + bit <= 8, + c0 == phi8_upto(value as u8, bit as nat), + decreases 8 - bit, + { + assert(PHI_8_BASIS[bit as int] == phi8_basis(bit as u8)); + assert(value < 256 && bit < 8 ==> ((value & (1usize << bit) != 0) == ((value as u8 >> bit as u8) & 1 + == 1))) by (bit_vector); + if value & (1 << bit) != 0 { + c0 ^= PHI_8_BASIS[bit]; + } else { + assert(c0 ^ 0 == c0) by (bit_vector); + } + bit += 1; + } + table[value] = F192::new(c0, 0, 0); + value += 1; + } + table +} + +/// The unique GF(2^8) subfield embedded in F192. It lies in the F64 base, so +/// both higher extension coordinates are zero. +/// +/// Production's `static` with the initializer `build_phi8_table_192()`, written as Verus's `exec static` with +/// that initializer as its body so that its value has a specification. +pub exec static PHI_8_TABLE_192: [F192; 256] + ensures + forall|v: int| 0 <= v < 256 ==> #[trigger] PHI_8_TABLE_192[v] == phi8_e(v as u8), +{ + build_phi8_table_192() +} + +#[inline] +pub fn phi8_192(a: F8) -> (r: F192) + ensures + r == phi8_e(a.0), +{ + PHI_8_TABLE_192[a.0 as usize] +} + +} // verus! diff --git a/verification/verus/src/skip_domain.rs b/verification/verus/src/skip_domain.rs new file mode 100644 index 0000000000..fc4e13d58f --- /dev/null +++ b/verification/verus/src/skip_domain.rs @@ -0,0 +1,1124 @@ +//! The univariate skip's domain and its Lagrange interpolation. +//! +//! The executable functions are `SkipDomain` of `crates/flock/src/zerocheck/skip_domain.rs`, which is generic over +//! the `Arith` trait of `crates/fiat_shamir/src/arith.rs`, instantiated here at its native arithmetic `Native` +//! (whose trait methods, the defaults included, are copied as inherent methods); `tests/equivalence/skip_domain.rs` +//! checks the two agree. Where Verus rejects the production form (iterator chains, `fold`, `.rev()`, `vec![x]`, +//! the generic parameter, `assert!` on an argument), the copy spells out the same arithmetic; each such spot says +//! what it replaces. +//! +//! Specification: the domain of `l = 2^k` nodes is `S = {s_0, .., s_(l-1)}`, `s_i = φ₈(i)` ([`node`]), an +//! `F_2`-subspace of `K`. Its vanishing polynomial is `V_l(z) = prod_i (z + s_i)` ([`vanishing_spec`]) and its +//! Lagrange basis the textbook one, +//! +//! ```text +//! L_i(z) = prod_{k != i} (z + s_k) / prod_{k != i} (s_i + s_k) ([`lagrange_basis`]) +//! ``` +//! +//! with `L_i(s_j) = [i = j]` ([`lemma_lagrange_basis_at_nodes`]). The theorems: `vanishing` is `V_l`; +//! `lagrange_at` is `sum_i values_i L_i(z)` and `first_round_at` is `sum_i values_i L_(l+i)(z)` over the window +//! of `2l` nodes, for every `z` off the nodes. At a node `s_j` production returns 0 rather than `values_j` (the +//! formula divides by `z + s_j` with `1 / 0 = 0`), so the theorems exclude the nodes. +use crate::gf2_64::*; +use crate::gf2_64x3::*; +use crate::multilinear::*; +use crate::phi8_tower::*; +use vstd::arithmetic::power2::*; +use vstd::prelude::*; + +verus! { + +// --------------------------------------------------------------------------------------------- +// Specification +// --------------------------------------------------------------------------------------------- +/// `prod_{k < n} f(k)`, from the first factor up. +pub open spec fn e_prod_fn(f: spec_fn(int) -> F192, n: nat) -> F192 + decreases n, +{ + if n == 0 { + F192::ONE + } else { + e_mul(e_prod_fn(f, (n - 1) as nat), f(n - 1)) + } +} + +/// Node `k` of the domain, `φ₈(k)`. +pub open spec fn node(k: int) -> F192 { + phi8_e(k as u8) +} + +/// `V_l(z) = prod_{k < l} (z + s_k)`, the vanishing polynomial of the first `l` nodes. +pub open spec fn vanishing_spec(l: nat, z: F192) -> F192 { + e_prod_fn(|k: int| e_add(z, node(k)), l) +} + +/// `f` with its factor at `i` replaced by one: a product of it skips `i`. +pub open spec fn except(f: spec_fn(int) -> F192, i: int) -> spec_fn(int) -> F192 { + |k: int| if k == i { + F192::ONE + } else { + f(k) + } +} + +/// The inverse in `E`, `a^(2^192 - 2)` (zero at zero), what `F192::inv` returns. +pub open spec fn e_inv(a: F192) -> F192 { + e_pow(a, (pow2(192) - 2) as nat) +} + +/// The Lagrange basis polynomial of node `i` among the first `l`, at `z`. +pub open spec fn lagrange_basis(l: nat, i: int, z: F192) -> F192 { + e_mul( + e_prod_fn(except(|k: int| e_add(z, node(k)), i), l), + e_inv(e_prod_fn(except(|k: int| e_add(node(i), node(k)), i), l)), + ) +} + +/// `sum_i values_i L_i(z)` over the first `l` nodes. +pub open spec fn lagrange_sum(l: nat, values: Seq, z: F192) -> F192 { + e_sum_fn(|i: int| e_mul(values[i], lagrange_basis(l, i, z)), values.len()) +} + +/// `sum_i values_i L_(l+i)(z)` over the window of the first `2l` nodes: the interpolant of `values` on the coset +/// `Λ = {s_l, .., s_(2l-1)}` and of zero on `S`. +pub open spec fn window_sum(l: nat, values: Seq, z: F192) -> F192 { + e_sum_fn(|i: int| e_mul(values[i], lagrange_basis(2 * l, l + i, z)), values.len()) +} + +/// `sum_{j < n} c_j x^(2^j)`, a linearized polynomial at `x`. +pub open spec fn lin_upto(c: Seq, x: F192, n: nat) -> F192 { + e_sum_fn(|j: int| e_mul(c[j], e_sq_iter(x, j as nat)), n) +} + +pub open spec fn lin(c: Seq, x: F192) -> F192 { + lin_upto(c, x, c.len()) +} + +/// `z` is none of the first `l` nodes. +pub open spec fn off_nodes(l: nat, z: F192) -> bool { + forall|k: int| 0 <= k < l ==> z != #[trigger] node(k) +} + +pub open spec fn sq(a: F192) -> F192 { + e_mul(a, a) +} + +// --------------------------------------------------------------------------------------------- +// Algebra +// --------------------------------------------------------------------------------------------- +proof fn lemma_xor4(a: u64, b: u64, c: u64, d: u64) + ensures + (a ^ b) ^ (b ^ d) == a ^ d, + (a ^ c) ^ (b ^ d) == (a ^ b) ^ (c ^ d), +{ + assert((a ^ b) ^ (b ^ d) == a ^ d && (a ^ c) ^ (b ^ d) == (a ^ b) ^ (c ^ d)) by (bit_vector); +} + +proof fn lemma_e_add4(a: F192, b: F192, c: F192, d: F192) + ensures + e_add(e_add(a, b), e_add(b, d)) == e_add(a, d), + e_add(e_add(a, c), e_add(b, d)) == e_add(e_add(a, b), e_add(c, d)), +{ + lemma_xor4(a.c0, b.c0, c.c0, d.c0); + lemma_xor4(a.c1, b.c1, c.c1, d.c1); + lemma_xor4(a.c2, b.c2, c.c2, d.c2); +} + +/// `z + s = 0` only when `z = s`. +proof fn lemma_e_add_zero(z: F192, s: F192) + ensures + e_add(z, s) == F192::ZERO <==> z == s, +{ + let (a, b, c, d, e, f) = (z.c0, s.c0, z.c1, s.c1, z.c2, s.c2); + assert((a ^ b == 0 && c ^ d == 0 && e ^ f == 0) <==> (a == b && c == d && e == f)) by (bit_vector); +} + +/// Squaring is additive: `(a + b)^2 = a^2 + b^2`. +pub proof fn lemma_sq_add(a: F192, b: F192) + ensures + sq(e_add(a, b)) == e_add(sq(a), sq(b)), +{ + let s = e_add(a, b); + lemma_e_mul_distrib(s, a, b); + lemma_e_mul_distrib(a, a, b); + lemma_e_mul_distrib(b, a, b); + lemma_e_mul_comm(a, s); + lemma_e_mul_comm(b, s); + lemma_e_mul_comm(a, b); + lemma_e_add4(sq(a), e_mul(a, b), e_mul(b, a), sq(b)); +} + +/// Squaring is multiplicative: `(ab)^2 = a^2 b^2`. +pub proof fn lemma_sq_mul(a: F192, b: F192) + ensures + sq(e_mul(a, b)) == e_mul(sq(a), sq(b)), +{ + let ab = e_mul(a, b); + lemma_e_mul_assoc(a, b, ab); + lemma_e_mul_assoc(b, a, b); + lemma_e_mul_comm(b, a); + lemma_e_mul_assoc(a, b, b); + lemma_e_mul_assoc(a, a, sq(b)); +} + +pub proof fn lemma_sq_iter_add(a: F192, b: F192, n: nat) + ensures + e_sq_iter(e_add(a, b), n) == e_add(e_sq_iter(a, n), e_sq_iter(b, n)), + decreases n, +{ + if n > 0 { + lemma_sq_iter_add(a, b, (n - 1) as nat); + lemma_sq_add(e_sq_iter(a, (n - 1) as nat), e_sq_iter(b, (n - 1) as nat)); + } +} + +pub proof fn lemma_sq_zero_one() + ensures + sq(F192::ZERO) == F192::ZERO, + sq(F192::ONE) == F192::ONE, +{ + lemma_e_mul_zero(F192::ZERO); + lemma_e_mul_one(F192::ONE); +} + +/// `sum_i (g1(i) + g2(i)) = sum_i g1(i) + sum_i g2(i)`. +pub proof fn lemma_sum_add(g1: spec_fn(int) -> F192, g2: spec_fn(int) -> F192, n: nat) + ensures + e_sum_fn(|i: int| e_add(g1(i), g2(i)), n) == e_add(e_sum_fn(g1, n), e_sum_fn(g2, n)), + decreases n, +{ + if n == 0 { + lemma_e_add(F192::ZERO, F192::ZERO, F192::ZERO); + } else { + lemma_sum_add(g1, g2, (n - 1) as nat); + lemma_e_add4(e_sum_fn(g1, (n - 1) as nat), e_sum_fn(g2, (n - 1) as nat), g1(n - 1), g2(n - 1)); + } +} + +/// The square of a sum is the sum of the squares. +pub proof fn lemma_sq_sum(g: spec_fn(int) -> F192, n: nat) + ensures + sq(e_sum_fn(g, n)) == e_sum_fn(|i: int| sq(g(i)), n), + decreases n, +{ + if n == 0 { + lemma_sq_zero_one(); + } else { + lemma_sq_sum(g, (n - 1) as nat); + lemma_sq_add(e_sum_fn(g, (n - 1) as nat), g(n - 1)); + } +} + +// --------------------------------------------------------------------------------------------- +// Products +// --------------------------------------------------------------------------------------------- +pub proof fn lemma_prod_ext(f: spec_fn(int) -> F192, g: spec_fn(int) -> F192, n: nat) + requires + forall|i: int| 0 <= i < n ==> #[trigger] f(i) == g(i), + ensures + e_prod_fn(f, n) == e_prod_fn(g, n), + decreases n, +{ + if n > 0 { + lemma_prod_ext(f, g, (n - 1) as nat); + } +} + +/// A product split after `m` factors. +pub proof fn lemma_prod_split(f: spec_fn(int) -> F192, m: nat, k: nat) + ensures + e_prod_fn(f, m + k) == e_mul(e_prod_fn(f, m), e_prod_fn(shifted(f, m as int), k)), + decreases k, +{ + if k == 0 { + lemma_e_mul_one(e_prod_fn(f, m)); + } else { + lemma_prod_split(f, m, (k - 1) as nat); + assert(shifted(f, m as int)(k - 1) == f(m + k - 1)); + lemma_e_mul_assoc(e_prod_fn(f, m), e_prod_fn(shifted(f, m as int), (k - 1) as nat), f(m + k - 1)); + } +} + +/// One factor out of a product: `prod_k f(k) = f(i) prod_{k != i} f(k)`. +pub proof fn lemma_prod_remove(f: spec_fn(int) -> F192, l: nat, i: int) + requires + 0 <= i < l, + ensures + e_prod_fn(f, l) == e_mul(f(i), e_prod_fn(except(f, i), l)), + decreases l, +{ + let g = except(f, i); + if i == l - 1 { + lemma_prod_ext(f, g, (l - 1) as nat); + lemma_e_mul_one(e_prod_fn(g, (l - 1) as nat)); + lemma_e_mul_comm(e_prod_fn(f, (l - 1) as nat), f(i)); + } else { + lemma_prod_remove(f, (l - 1) as nat, i); + let (a, p, b) = (f(i), e_prod_fn(g, (l - 1) as nat), f(l - 1)); + lemma_e_mul_assoc(a, p, b); + } +} + +/// `k -> i ^ k`, a permutation of every aligned power-of-two range containing `i`. +pub open spec fn xor_perm(g: spec_fn(int) -> F192, i: int) -> spec_fn(int) -> F192 { + |k: int| g(((i as usize) ^ (k as usize)) as int) +} + +proof fn lemma_xor_halves(i: usize, k: usize, s: usize) + requires + s < 8, + k < (1usize << s), + ensures + i < (1usize << s) ==> (i ^ k) < (1usize << s) && (i ^ ((1usize << s) + k) as usize) == (1usize << s) + (i ^ k), + (1usize << s) <= i < 2 * (1usize << s) ==> (i ^ k) == (1usize << s) + ((i - (1usize << s)) as usize ^ k) && (i + ^ ((1usize << s) + k) as usize) == ((i - (1usize << s)) as usize ^ k), +{ + assert(i < (1usize << s) ==> (i ^ k) < (1usize << s) && (i ^ ((1usize << s) + k) as usize) == (1usize << s) + (i + ^ k)) by (bit_vector) + requires + s < 8, + k < (1usize << s), + ; + assert((1usize << s) <= i < 2 * (1usize << s) ==> (i ^ k) == (1usize << s) + ((i - (1usize << s)) as usize ^ k) + && (i ^ ((1usize << s) + k) as usize) == ((i - (1usize << s)) as usize ^ k)) by (bit_vector) + requires + s < 8, + k < (1usize << s), + ; +} + +/// The product over an aligned power-of-two range is invariant under `k -> i ^ k`. +pub proof fn lemma_prod_xor(g: spec_fn(int) -> F192, j: nat, i: int) + requires + j <= 8, + 0 <= i < pow2(j), + ensures + e_prod_fn(xor_perm(g, i), pow2(j)) == e_prod_fn(g, pow2(j)), + decreases j, +{ + lemma_pow2_shl_small(j); + lemma2_to64(); + if j == 0 { + let iu = i as usize; + assert(iu == 0); + assert(iu ^ 0usize == 0usize) by (bit_vector) + requires + iu == 0, + ; + assert(xor_perm(g, i)(0) == g(0)); + assert(e_prod_fn(xor_perm(g, i), 0) == F192::ONE); + assert(e_prod_fn(g, 0) == F192::ONE); + assert(e_prod_fn(xor_perm(g, i), 1) == e_mul(F192::ONE, xor_perm(g, i)(0))); + assert(e_prod_fn(g, 1) == e_mul(F192::ONE, g(0))); + } else { + let s = (j - 1) as usize; + lemma_pow2_shl_small(s as nat); + lemma_pow2_unfold(j); + let m = pow2(s as nat); + let p = xor_perm(g, i); + lemma_prod_split(p, m, m); + lemma_prod_split(g, m, m); + let gh = shifted(g, m as int); + if i < m { + lemma_prod_xor(g, s as nat, i); + lemma_prod_xor(gh, s as nat, i); + assert forall|k: int| 0 <= k < m implies #[trigger] shifted(p, m as int)(k) == xor_perm(gh, i)(k) by { + lemma_xor_halves(i as usize, k as usize, s); + } + lemma_prod_ext(shifted(p, m as int), xor_perm(gh, i), m); + assert forall|k: int| 0 <= k < m implies #[trigger] p(k) == xor_perm(g, i)(k) by {} + } else { + let i2 = i - m; + lemma_prod_xor(g, s as nat, i2); + lemma_prod_xor(gh, s as nat, i2); + assert forall|k: int| 0 <= k < m implies #[trigger] p(k) == xor_perm(gh, i2)(k) by { + lemma_xor_halves(i as usize, k as usize, s); + } + assert forall|k: int| 0 <= k < m implies #[trigger] shifted(p, m as int)(k) == xor_perm(g, i2)(k) by { + lemma_xor_halves(i as usize, k as usize, s); + } + lemma_prod_ext(p, xor_perm(gh, i2), m); + lemma_prod_ext(shifted(p, m as int), xor_perm(g, i2), m); + lemma_e_mul_comm(e_prod_fn(gh, m), e_prod_fn(g, m)); + } + } +} + +proof fn lemma_pow2_shl_small(j: nat) + requires + j <= 8, + ensures + pow2(j) == (1usize << (j as usize)), + pow2(j) <= 256, + pow2(j) >= 1, +{ + lemma2_to64(); + let u = j as usize; + assert(u <= 8 ==> (1usize << u) == (if u == 0 { 1usize } else if u == 1 { 2 } else if u == 2 { 4 } else if u == 3 { 8 } else if u == 4 { 16 } else if u == 5 { 32 } else if u == 6 { 64 } else if u == 7 { 128 } else { 256 })) by (bit_vector); +} + +/// Sums of nodes: `s_i + s_k = s_(i ^ k)`. +pub proof fn lemma_node_add(i: int, k: int) + requires + 0 <= i < 256, + 0 <= k < 256, + ensures + e_add(node(i), node(k)) == node(((i as usize) ^ (k as usize)) as int), +{ + let (iu, ku) = (i as usize, k as usize); + assert(iu < 256 && ku < 256 ==> (iu ^ ku) < 256 && ((iu ^ ku) as u8) == (iu as u8) ^ (ku as u8)) by (bit_vector); + lemma_phi8_xor(i as u8, k as u8); + assert(0u64 ^ 0u64 == 0u64) by (bit_vector); +} + +/// The product of the nonzero nodes of the first `n` is `prod_{k=1}^{n-1} φ₈(k)` in `K`. +proof fn lemma_prod_nonzero_nodes(n: nat) + requires + 1 <= n <= 256, + ensures + e_prod_fn(except(|k: int| node(k), 0), n) == e_from_k(phi8_prod(n)), + decreases n, +{ + let h = except(|k: int| node(k), 0); + if n == 1 { + lemma_e_mul_one(F192::ONE); + assert(e_prod_fn(h, 0) == F192::ONE); + assert(h(0) == F192::ONE); + assert(e_prod_fn(h, 1) == e_mul(e_prod_fn(h, 0), h(0))); + assert(phi8_prod(1) == 1); + assert(e_from_k(1) == F192::ONE); + } else { + lemma_prod_nonzero_nodes((n - 1) as nat); + lemma_e_from_k_mul(phi8_prod((n - 1) as nat), phi8((n - 1) as u8)); + assert(h(n - 1) == e_from_k(phi8((n - 1) as u8))); + assert(e_prod_fn(h, n) == e_mul(e_prod_fn(h, (n - 1) as nat), h(n - 1))); + assert(phi8_prod(n) == k_mul(phi8_prod((n - 1) as nat), phi8((n - 1) as u8))); + } +} + +/// `prod_{k != i} (s_i + s_k)` is the same for every node of a window: the product of its nonzero nodes. +pub proof fn lemma_node_differences(log: nat, i: int) + requires + log <= 8, + 0 <= i < pow2(log), + ensures + e_prod_fn(except(|k: int| e_add(node(i), node(k)), i), pow2(log)) == e_from_k(phi8_prod(pow2(log))), +{ + lemma_pow2_shl_small(log); + let l = pow2(log); + let h = except(|k: int| node(k), 0); + let f = except(|k: int| e_add(node(i), node(k)), i); + assert forall|k: int| 0 <= k < l implies #[trigger] f(k) == xor_perm(h, i)(k) by { + lemma_node_add(i, k); + let (iu, ku) = (i as usize, k as usize); + assert((iu ^ ku) == 0 <==> iu == ku) by (bit_vector); + assert(iu < 256 && ku < 256 ==> (iu ^ ku) < 256) by (bit_vector); + } + lemma_prod_ext(f, xor_perm(h, i), l); + lemma_prod_xor(h, log, i); + lemma_prod_nonzero_nodes(l); +} + +/// `1 / prod_{k != i} (s_i + s_k)` is the window's weight `D`. +pub proof fn lemma_weight_inverts(log: nat, i: int) + requires + log <= 8, + 0 <= i < pow2(log), + ensures + e_inv(e_prod_fn(except(|k: int| e_add(node(i), node(k)), i), pow2(log))) == denominator_spec(log), + e_prod_fn(except(|k: int| e_add(node(i), node(k)), i), pow2(log)) != F192::ZERO, +{ + lemma_node_differences(log, i); + lemma_denominator_inverts(log); + let q = e_from_k(phi8_prod(pow2(log))); + let d = denominator_spec(log); + assert(q != F192::ZERO); + lemma_e_inverse(q); + let w = e_inv(q); + lemma_e_mul_assoc(d, q, w); + lemma_e_mul_one(d); + lemma_e_mul_one(w); +} + +/// The basis is the Lagrange basis: `L_i(s_j) = 1` if `i = j`, else 0. +pub proof fn lemma_lagrange_basis_at_nodes(log: nat, i: int, j: int) + requires + log <= 8, + 0 <= i < pow2(log), + 0 <= j < pow2(log), + ensures + lagrange_basis(pow2(log), i, node(j)) == (if i == j { + F192::ONE + } else { + F192::ZERO + }), +{ + let l = pow2(log); + lemma_weight_inverts(log, i); + let q = e_prod_fn(except(|k: int| e_add(node(i), node(k)), i), l); + let f = |k: int| e_add(node(j), node(k)); + if i == j { + lemma_prod_ext(except(f, i), except(|k: int| e_add(node(i), node(k)), i), l); + lemma_e_inverse(q); + } else { + lemma_prod_remove(except(f, i), l, j); + lemma_e_add(node(j), node(j), node(j)); + lemma_e_mul_zero(e_prod_fn(except(except(f, i), j), l)); + lemma_e_mul_zero(e_inv(q)); + } +} + +/// The vanishing polynomial of `2m` nodes from that of `m`: `V_2m(x) = V_m(x) V_m(x + s_m)`. +pub proof fn lemma_vanishing_double(j: nat, x: F192) + requires + j < 8, + ensures + vanishing_spec(2 * pow2(j), x) == e_mul(vanishing_spec(pow2(j), x), vanishing_spec(pow2(j), e_add(x, node(pow2(j) as int)))), +{ + lemma_pow2_shl_small(j); + lemma2_to64(); + lemma_pow2_strictly_increases(j, 8); + let m = pow2(j); + let f = |k: int| e_add(x, node(k)); + let a = node(m as int); + lemma_prod_split(f, m, m); + assert forall|k: int| 0 <= k < m implies #[trigger] shifted(f, m as int)(k) == e_add(e_add(x, a), node(k)) by { + let (mu, ku, ju) = (m as usize, k as usize, j as usize); + assert(mu == (1usize << ju) && ju < 8 && ku < mu ==> (mu ^ ku) == mu + ku) by (bit_vector); + lemma_node_add(m as int, k); + lemma_e_add(x, a, node(k)); + } + lemma_prod_ext(shifted(f, m as int), |k: int| e_add(e_add(x, a), node(k)), m); +} + +// --------------------------------------------------------------------------------------------- +// Linearized polynomials +// --------------------------------------------------------------------------------------------- +/// A linearized polynomial is additive. +pub proof fn lemma_lin_add(c: Seq, x: F192, y: F192, n: nat) + ensures + lin_upto(c, e_add(x, y), n) == e_add(lin_upto(c, x, n), lin_upto(c, y, n)), + decreases n, +{ + if n == 0 { + lemma_e_add(F192::ZERO, F192::ZERO, F192::ZERO); + } else { + let j = n - 1; + lemma_lin_add(c, x, y, (n - 1) as nat); + lemma_sq_iter_add(x, y, j as nat); + lemma_e_mul_distrib(c[j], e_sq_iter(x, j as nat), e_sq_iter(y, j as nat)); + lemma_e_add4( + lin_upto(c, x, (n - 1) as nat), + lin_upto(c, y, (n - 1) as nat), + e_mul(c[j], e_sq_iter(x, j as nat)), + e_mul(c[j], e_sq_iter(y, j as nat)), + ); + } +} + +/// The step the coefficients take when a basis element joins the subspace: with `c'_k = c_(k-1)^2 + va c_k` (and +/// the top coefficient zero), `lin(c') = lin(c)^2 + va lin(c)`. +pub proof fn lemma_lin_step(c: Seq, va: F192, x: F192) + requires + c.len() >= 1, + c.last() == F192::ZERO, + ensures + lin(Seq::new(c.len(), |k: int| e_add(if k == 0 { F192::ZERO } else { sq(c[k - 1]) }, e_mul(va, c[k]))), x) == e_add( + sq(lin(c, x)), + e_mul(va, lin(c, x)), + ), +{ + let n = c.len(); + let cn = Seq::new(n, |k: int| e_add(if k == 0 { F192::ZERO } else { sq(c[k - 1]) }, e_mul(va, c[k]))); + let xs = |k: int| e_sq_iter(x, k as nat); + let ta = |k: int| e_mul(if k == 0 { F192::ZERO } else { sq(c[k - 1]) }, xs(k)); + let tb = |k: int| e_mul(e_mul(va, c[k]), xs(k)); + let tc = |k: int| e_mul(c[k], xs(k)); + // lin(c') = sum ta + sum tb. + assert forall|k: int| 0 <= k < n implies #[trigger] e_mul(cn[k], e_sq_iter(x, k as nat)) == e_add(ta(k), tb(k)) by { + lemma_e_mul_distrib(xs(k), if k == 0 { F192::ZERO } else { sq(c[k - 1]) }, e_mul(va, c[k])); + lemma_e_mul_comm(xs(k), if k == 0 { F192::ZERO } else { sq(c[k - 1]) }); + lemma_e_mul_comm(xs(k), e_mul(va, c[k])); + lemma_e_mul_comm(xs(k), cn[k]); + } + lemma_sum_ext(|j: int| e_mul(cn[j], e_sq_iter(x, j as nat)), |k: int| e_add(ta(k), tb(k)), n); + lemma_sum_add(ta, tb, n); + // sum tb = va lin(c). + assert forall|k: int| 0 <= k < n implies #[trigger] tb(k) == e_mul(va, tc(k)) by { + lemma_e_mul_assoc(va, c[k], xs(k)); + } + lemma_sum_ext(tb, |k: int| e_mul(va, tc(k)), n); + lemma_sum_scale(va, tc, n); + lemma_sum_ext(tc, |j: int| e_mul(c[j], e_sq_iter(x, j as nat)), n); + // sum ta = sum_{j < n - 1} c_j^2 x^(2^(j+1)). + let td = |j: int| e_mul(sq(c[j]), e_sq_iter(x, (j + 1) as nat)); + lemma_sum_split(ta, 1, (n - 1) as nat); + lemma_e_mul_zero(xs(0)); + lemma_e_add(F192::ZERO, F192::ZERO, F192::ZERO); + assert(e_sum_fn(ta, 0) == F192::ZERO); + assert(ta(0) == F192::ZERO); + assert(e_sum_fn(ta, 1) == F192::ZERO); + assert forall|j: int| 0 <= j < n - 1 implies #[trigger] shifted(ta, 1)(j) == td(j) by { + assert((j + 1) as nat == (1 + j) as nat); + } + lemma_sum_ext(shifted(ta, 1), td, (n - 1) as nat); + lemma_e_add(F192::ZERO, e_sum_fn(td, (n - 1) as nat), F192::ZERO); + // lin(c)^2 = sum_{j < n} c_j^2 x^(2^(j+1)), whose last term is zero. + lemma_sq_sum(tc, n); + assert forall|j: int| 0 <= j < n implies #[trigger] sq(tc(j)) == td(j) by { + lemma_sq_mul(c[j], xs(j)); + } + lemma_sum_ext(|j: int| sq(tc(j)), td, n); + lemma_sq_zero_one(); + lemma_e_mul_zero(e_sq_iter(x, n as nat)); + lemma_e_add(e_sum_fn(td, (n - 1) as nat), F192::ZERO, F192::ZERO); +} + +/// A linearized polynomial with coefficients `1, 0, .., 0` is the identity. +proof fn lemma_lin_first(c: Seq, x: F192, n: nat) + requires + 1 <= n <= c.len(), + c[0] == F192::ONE, + forall|k: int| 1 <= k < c.len() ==> #[trigger] c[k] == F192::ZERO, + ensures + lin_upto(c, x, n) == x, + decreases n, +{ + if n == 1 { + lemma_e_mul_one(x); + lemma_e_add(x, F192::ZERO, F192::ZERO); + assert(lin_upto(c, x, 0) == F192::ZERO); + } else { + lemma_lin_first(c, x, (n - 1) as nat); + lemma_e_mul_zero(e_sq_iter(x, (n - 1) as nat)); + lemma_e_add(x, F192::ZERO, F192::ZERO); + } +} + +// --------------------------------------------------------------------------------------------- +// Executable code +// --------------------------------------------------------------------------------------------- +/// The domain's size exponent in flock (`crates/flock/src/zerocheck.rs`). +pub const K_SKIP: usize = 6; + +/// Plain `F192` arithmetic, for the prover's share of code the verifiers also run. +/// +/// The `Arith` implementation of `fiat_shamir::arith::Native`: its methods, with the trait's default methods it +/// inherits (`zero`, `mul`, `square`, `mul_const`, `add_const`), as inherent methods. +#[derive(Clone, Copy, Debug, Default)] +pub struct Native; + +impl Native { + /// The constant `c`. + pub fn constant(&mut self, c: F192) -> (r: F192) + ensures + r == c, + { + c + } + + /// `a·b + d`. + pub fn mul_add(&mut self, a: F192, b: F192, d: F192) -> (r: F192) + ensures + r == e_add(e_mul(a, b), d), + { + a * b + d + } + + /// `a + d`. + pub fn add(&mut self, a: F192, d: F192) -> (r: F192) + ensures + r == e_add(a, d), + { + a + d + } + + /// `a·c + d` for a constant `c`. + pub fn mul_const_add(&mut self, a: F192, c: F192, d: F192) -> (r: F192) + ensures + r == e_add(e_mul(a, c), d), + { + a * c + d + } + + /// `1 / a`, zero for zero. + pub fn inv(&mut self, a: F192) -> (r: F192) + ensures + r == e_inv(a), + { + proof { + lemma_e_inverse(a); + } + if a.is_zero() { F192::ZERO } else { a.inv() } + } + + /// The constant zero. + pub fn zero(&mut self) -> (r: F192) + ensures + r == F192::ZERO, + { + self.constant(F192::ZERO) + } + + /// `a·b`. + pub fn mul(&mut self, a: F192, b: F192) -> (r: F192) + ensures + r == e_mul(a, b), + { + let zero = self.zero(); + proof { + lemma_e_add(e_mul(a, b), F192::ZERO, F192::ZERO); + } + self.mul_add(a, b, zero) + } + + /// `a^2`. + pub fn square(&mut self, a: F192) -> (r: F192) + ensures + r == e_mul(a, a), + { + self.mul(a, a) + } + + /// `a·c` for a constant `c`. + pub fn mul_const(&mut self, a: F192, c: F192) -> (r: F192) + ensures + r == e_mul(a, c), + { + let zero = self.zero(); + proof { + lemma_e_add(e_mul(a, c), F192::ZERO, F192::ZERO); + } + self.mul_const_add(a, c, zero) + } + + /// `a + c` for a constant `c`. + pub fn add_const(&mut self, a: F192, c: F192) -> (r: F192) + ensures + r == e_add(a, c), + { + let c = self.constant(c); + self.add(a, c) + } +} + +/// The skip domain `S`, the first `2^k_skip` nodes of the phi_8 table: an `F_2`-subspace of `K`, since phi_8 is linear on its index. +/// +/// Its coset `Lambda = S + phi_8(2^k_skip)` holds the zerocheck's first message. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct SkipDomain { + k_skip: usize, +} + +impl SkipDomain { + /// `k_skip`. + pub closed spec fn spec_k_skip(self) -> nat { + self.k_skip as nat + } + + /// The domain the zerocheck skips and the lincheck interpolates over. + /// + /// Verus's `exec const` form (the field is private). + pub exec const FLOCK: Self + ensures + Self::FLOCK.spec_k_skip() == K_SKIP, + { + Self { k_skip: K_SKIP } + } + + /// The domain of `2^k_skip` nodes. + /// + /// Production asserts `k_skip < 8` (the window fits the phi_8 table); here a `requires`. + pub const fn new(k_skip: usize) -> (d: Self) + requires + k_skip < 8, + ensures + d.spec_k_skip() == k_skip, + { + Self { k_skip } + } + + /// The base-two logarithm of its size. + pub const fn k_skip(self) -> (k: usize) + ensures + k == self.spec_k_skip(), + { + self.k_skip + } + + /// Its size, and the zerocheck's first message's. + pub const fn size(self) -> (s: usize) + requires + self.spec_k_skip() < 8, + ensures + s == pow2(self.spec_k_skip()), + { + proof { + lemma_pow2_shl_small(self.k_skip as nat); + } + 1 << self.k_skip + } + + /// The coefficients `c_j` of `V_S(X) = prod_{s in S} (X + s) = sum_j c_j X^(2^j)`, lowest first. + /// + /// Adding a basis element `a` to a subspace takes `V` to `V(X)^2 + V(a) V(X)`, since `V(X + a) = V(X) + V(a)`. + /// + /// `for k in (0..=j + 1).rev()` is a `while` counting down. + fn vanishing_coefficients(self) -> (c: Vec) + requires + self.spec_k_skip() < 8, + ensures + c.len() == self.spec_k_skip() + 1, + c@[self.spec_k_skip() as int] == F192::ONE, + forall|x: F192| #[trigger] lin(c@, x) == vanishing_spec(pow2(self.spec_k_skip()), x), + { + let ghost n = self.k_skip + 1; + let mut c = vec![F192::ZERO; self.k_skip + 1]; + c[0] = F192::ONE; + proof { + lemma2_to64(); + lemma_phi8_zero_one(); + assert forall|t: int| 1 <= t < c.len() implies #[trigger] c@[t] == F192::ZERO by {} + assert forall|x: F192| #[trigger] lin(c@, x) == vanishing_spec(pow2(0), x) by { + lemma_lin_first(c@, x, n as nat); + lemma_e_add(x, F192::ZERO, F192::ZERO); + lemma_e_mul_one(x); + assert(node(0) == F192::ZERO); + let f = |k: int| e_add(x, node(k)); + assert(e_prod_fn(f, 0) == F192::ONE); + assert(e_prod_fn(f, 1) == e_mul(F192::ONE, f(0))); + } + } + for j in 0..self.k_skip + invariant + n == self.k_skip + 1, + self.k_skip < 8, + c.len() == n, + c@[j as int] == F192::ONE, + forall|k: int| j < k < n ==> #[trigger] c@[k] == F192::ZERO, + forall|x: F192| #[trigger] lin(c@, x) == vanishing_spec(pow2(j as nat), x), + { + proof { + lemma_pow2_shl_small(j as nat); + lemma2_to64(); + lemma_pow2_strictly_increases(j as nat, 8); + } + let a = PHI_8_TABLE_192[1 << j]; + let at_a = Self::linearized(&c, a); + let ghost prev = c@; + let ghost cn = Seq::new(n as nat, |k: int| e_add(if k == 0 { F192::ZERO } else { sq(prev[k - 1]) }, e_mul(at_a, prev[k]))); + let mut k = j + 2; + while k > 0 + invariant + n == self.k_skip + 1, + j < self.k_skip, + c.len() == n, + k <= j + 2, + forall|t: int| k <= t <= j + 1 ==> #[trigger] c@[t] == cn[t], + prev.len() == n, + cn.len() == n, + forall|t: int| 0 <= t < n ==> #[trigger] cn[t] == e_add(if t == 0 { F192::ZERO } else { sq(prev[t - 1]) }, e_mul(at_a, prev[t])), + forall|t: int| 0 <= t < k ==> #[trigger] c@[t] == prev[t], + forall|t: int| j + 1 < t < n ==> #[trigger] c@[t] == prev[t], + decreases k, + { + k -= 1; + let squared = if k == 0 { F192::ZERO } else { c[k - 1].square() }; + c[k] = squared + at_a * c[k]; + } + proof { + lemma_sq_zero_one(); + lemma_e_mul_zero(at_a); + lemma_e_add(F192::ZERO, F192::ZERO, F192::ZERO); + lemma_e_add(F192::ONE, F192::ZERO, F192::ZERO); + assert forall|t: int| j + 1 < t < n implies #[trigger] cn[t] == F192::ZERO by {} + assert(c@ =~= cn); + assert(cn[j + 1] == F192::ONE); + lemma_pow2_unfold((j + 1) as nat); + assert forall|x: F192| #[trigger] lin(c@, x) == vanishing_spec(pow2((j + 1) as nat), x) by { + lemma_lin_step(prev, at_a, x); + lemma_lin_add(prev, x, a, n as nat); + lemma_vanishing_double(j as nat, x); + let (v, va) = (lin(prev, x), lin(prev, a)); + assert(v == vanishing_spec(pow2(j as nat), x)); + assert(va == vanishing_spec(pow2(j as nat), a)); + assert(lin(prev, e_add(x, a)) == vanishing_spec(pow2(j as nat), e_add(x, a))); + lemma_e_mul_distrib(v, v, va); + lemma_e_mul_comm(v, va); + } + } + } + c + } + + /// A linearized polynomial `sum_j c_j x^(2^j)` at `x`. + /// + /// `for &cj in c` is a loop over the indices. + fn linearized(c: &[F192], x: F192) -> (r: F192) + ensures + r == lin(c@, x), + { + let (mut power, mut acc) = (x, F192::ZERO); + for j in 0..c.len() + invariant + power == e_sq_iter(x, j as nat), + acc == lin_upto(c@, x, j as nat), + { + acc += c[j] * power; + power = power.square(); + } + acc + } + + /// `V_S(z)`: `k_skip` squarings and as many products by constants of `K`. + /// + /// The generic `A: Arith` is `Native`. `vec![z]` is a push, and the fold over `c[..k_skip]` zipped with the + /// powers is a loop; the `debug_assert_eq!` that `V` is monic is proven (`vanishing_coefficients`). + pub fn vanishing(self, a: &mut Native, z: F192) -> (v: F192) + requires + self.spec_k_skip() < 8, + ensures + v == vanishing_spec(pow2(self.spec_k_skip()), z), + { + let c = self.vanishing_coefficients(); + let mut powers: Vec = Vec::new(); + powers.push(z); + for j in 0..self.k_skip + invariant + self.k_skip < 8, + powers.len() == j + 1, + forall|t: int| 0 <= t <= j ==> #[trigger] powers@[t] == e_sq_iter(z, t as nat), + { + let p = a.square(powers[j]); + powers.push(p); + } + let ghost k = self.k_skip; + let mut acc = powers[self.k_skip]; + proof { + lemma_e_add(e_sq_iter(z, k as nat), F192::ZERO, F192::ZERO); + assert(lin_upto(c@, z, 0) == F192::ZERO); + } + for j in 0..self.k_skip + invariant + k == self.k_skip, + k < 8, + c.len() == k + 1, + powers.len() == k + 1, + forall|t: int| 0 <= t <= k ==> #[trigger] powers@[t] == e_sq_iter(z, t as nat), + acc == e_add(lin_upto(c@, z, j as nat), e_sq_iter(z, k as nat)), + { + proof { + let t = e_mul(c@[j as int], powers@[j as int]); + lemma_e_mul_comm(powers@[j as int], c@[j as int]); + lemma_e_add(t, lin_upto(c@, z, j as nat), e_sq_iter(z, k as nat)); + lemma_e_add(lin_upto(c@, z, j as nat), t, e_sq_iter(z, k as nat)); + } + acc = a.mul_const_add(powers[j], c[j], acc); + } + proof { + lemma_e_mul_one(e_sq_iter(z, k as nat)); + assert(lin(c@, z) == vanishing_spec(pow2(k as nat), z)); + assert(lin(c@, z) == e_add(lin_upto(c@, z, k as nat), e_mul(c@[k as int], e_sq_iter(z, k as nat)))); + } + acc + } + + /// `1 / (z + node)` for each node. + /// + /// The generic `A: Arith` is `Native`; the `map` and `collect` are a loop pushing each inverse. + fn inverses_at(a: &mut Native, z: F192, nodes: &[F192]) -> (r: Vec) + ensures + r.len() == nodes.len(), + forall|i: int| 0 <= i < nodes.len() ==> #[trigger] r@[i] == e_inv(e_add(z, nodes@[i])), + { + let mut out: Vec = Vec::new(); + for i in 0..nodes.len() + invariant + out.len() == i, + forall|t: int| 0 <= t < i ==> #[trigger] out@[t] == e_inv(e_add(z, nodes@[t])), + { + let node = nodes[i]; + let difference = a.add_const(z, node); + out.push(a.inv(difference)); + } + out + } + + /// `1 / (z + s_i)` over `S`, which the Lagrange sum over `S` takes. + pub fn inverses(self, a: &mut Native, z: F192) -> (r: Vec) + requires + self.spec_k_skip() < 8, + ensures + r.len() == pow2(self.spec_k_skip()), + forall|i: int| 0 <= i < r.len() ==> #[trigger] r@[i] == e_inv(e_add(z, node(i))), + { + let size = self.size(); + proof { + lemma_pow2_shl_small(self.k_skip as nat); + } + Self::inverses_at(a, z, &PHI_8_TABLE_192[..size]) + } + + /// `scale * sum_i values_i * inverses_i`. + /// + /// The generic `A: Arith` is `Native`; the `assert_eq!` on the lengths is a `requires`, the fold a loop. + pub fn lagrange_with(a: &mut Native, scale: F192, inverses: &[F192], values: &[F192]) -> (r: F192) + requires + inverses.len() == values.len(), + ensures + r == e_mul(scale, e_sum_fn(|i: int| e_mul(values@[i], inverses@[i]), values.len() as nat)), + { + let zero = a.zero(); + let mut sum = zero; + for i in 0..values.len() + invariant + inverses.len() == values.len(), + sum == e_sum_fn(|t: int| e_mul(values@[t], inverses@[t]), i as nat), + { + proof { + lemma_e_add(e_mul(values@[i as int], inverses@[i as int]), sum, sum); + } + sum = a.mul_add(values[i], inverses[i], sum); + } + a.mul(scale, sum) + } + + /// The first round's message, known on `Lambda` and zero on `S`, interpolated at `z` over the window `S + Lambda`. + /// + /// Its value is `D_2l · V_S(z) · V_Lambda(z) · sum_i values_i / (z + lambda_i)`, `l` the domain's size, with `V_Lambda(z) = V_S(z) + V_S(phi_8(l))`. + /// + /// This barycentric form is the interpolant only for `z` off the window `S + Lambda`: at a node the vanishing factor is zero and `1 / 0` is taken as zero, so it gives 0 rather than the node's value. `z` is the verifier's challenge, so it lands on a node with probability at most `128 / 2^192`, where an honest proof would be rejected. + /// + /// The generic `A: Arith` is `Native`. `z` off the window's nodes and `vanishing = V_S(z)` are `requires`. The + /// coefficients `self.vanishing_coefficients()` are bound to a name before `linearized` reads them. + pub fn first_round_at(self, a: &mut Native, z: F192, vanishing: F192, values: &[F192]) -> (r: F192) + requires + self.spec_k_skip() < 8, + vanishing == vanishing_spec(pow2(self.spec_k_skip()), z), + values.len() == pow2(self.spec_k_skip()), + off_nodes(2 * pow2(self.spec_k_skip()), z), + ensures + r == window_sum(pow2(self.spec_k_skip()), values@, z), + { + let ghost k = self.k_skip as nat; + let size = self.size(); + proof { + lemma_pow2_shl_small(k); + lemma_pow2_shl_small(k + 1); + lemma_pow2_unfold(k + 1); + } + let lambda = &PHI_8_TABLE_192[size..2 * size]; + let c = self.vanishing_coefficients(); + let offset = Self::linearized(&c, lambda[0]); + let on_lambda = a.add_const(vanishing, offset); + let both = a.mul(vanishing, on_lambda); + let scaled = a.mul_const(both, window_denominator(2 * size)); + let inverses = Self::inverses_at(a, z, lambda); + let r = Self::lagrange_with(a, scaled, &inverses, values); + proof { + let l = pow2(k); + let s = node(l as int); + assert(lambda@[0] == s); + assert(lin(c@, s) == vanishing_spec(l, s)); + assert(lin(c@, z) == vanishing_spec(l, z)); + lemma_lin_add(c@, z, s, c.len() as nat); + assert(lin(c@, e_add(z, s)) == vanishing_spec(l, e_add(z, s))); + lemma_vanishing_double(k, z); + assert(both == vanishing_spec(2 * l, z)); + assert(scaled == e_mul(vanishing_spec(pow2(k + 1), z), denominator_spec(k + 1))); + let h = |i: int| e_mul(values@[i], inverses@[i]); + lemma_sum_scale(scaled, h, l); + assert forall|i: int| 0 <= i < l implies #[trigger] e_mul(scaled, h(i)) == e_mul( + values@[i], + lagrange_basis(2 * l, l + i, z), + ) by { + assert(lambda@[i] == node(l + i)); + lemma_lagrange_term(k + 1, l + i, z, values@[i]); + } + lemma_sum_ext(|i: int| e_mul(scaled, h(i)), |i: int| e_mul(values@[i], lagrange_basis(2 * l, l + i, z)), l); + } + r + } + + /// `D_l · V_S(z)`, `l` the domain's size: the scale of the Lagrange sum over `S`. + pub fn lagrange_scale(self, a: &mut Native, vanishing: F192) -> (r: F192) + requires + self.spec_k_skip() < 8, + ensures + r == e_mul(vanishing, denominator_spec(self.spec_k_skip())), + { + let size = self.size(); + proof { + lemma_pow2_shl_small(self.k_skip as nat); + } + a.mul_const(vanishing, window_denominator(size)) + } + + /// `sum_i L_i(z) values_i` over `S`, `L_i` its Lagrange basis: `D_l · V_S(z) · sum_i values_i / (z + s_i)`. + /// + /// This barycentric form is the interpolant only for `z` off `S`: at a node `V_S(z)` is zero and `1 / 0` is taken as zero, so it gives 0 rather than the node's value. `z` is the verifier's challenge, so it lands on a node with probability at most `128 / 2^192`, where an honest proof would be rejected. + /// + /// The generic `A: Arith` is `Native`. `z` off the nodes and `vanishing = V_S(z)` are `requires`. + pub fn lagrange_at(self, a: &mut Native, z: F192, vanishing: F192, values: &[F192]) -> (r: F192) + requires + self.spec_k_skip() < 8, + vanishing == vanishing_spec(pow2(self.spec_k_skip()), z), + values.len() == pow2(self.spec_k_skip()), + off_nodes(pow2(self.spec_k_skip()), z), + ensures + r == lagrange_sum(pow2(self.spec_k_skip()), values@, z), + { + let scaled = self.lagrange_scale(a, vanishing); + let inverses = self.inverses(a, z); + let r = Self::lagrange_with(a, scaled, &inverses, values); + proof { + let log = self.k_skip as nat; + let l = pow2(log); + let h = |i: int| e_mul(values@[i], inverses@[i]); + lemma_sum_scale(scaled, h, l); + assert forall|i: int| 0 <= i < l implies #[trigger] e_mul(scaled, h(i)) == e_mul(values@[i], lagrange_basis(l, i, z)) by { + lemma_lagrange_term(log, i, z, values@[i]); + } + lemma_sum_ext(|i: int| e_mul(scaled, h(i)), |i: int| e_mul(values@[i], lagrange_basis(l, i, z)), l); + } + r + } +} + +/// One term of the Lagrange sum as production computes it: `(V(z) D) (v / (z + s_i)) = v L_i(z)`, for `z` not +/// node `i`. +pub proof fn lemma_lagrange_term(log: nat, i: int, z: F192, v: F192) + requires + log <= 8, + 0 <= i < pow2(log), + z != node(i), + ensures + e_mul(e_mul(vanishing_spec(pow2(log), z), denominator_spec(log)), e_mul(v, e_inv(e_add(z, node(i))))) + == e_mul(v, lagrange_basis(pow2(log), i, z)), +{ + let l = pow2(log); + let f = |k: int| e_add(z, node(k)); + let fi = f(i); + let p = e_prod_fn(except(f, i), l); + let d = denominator_spec(log); + let h = e_inv(fi); + lemma_prod_remove(f, l, i); + lemma_e_add_zero(z, node(i)); + lemma_e_inverse(fi); + lemma_weight_inverts(log, i); + // V h = p (f_i h) = p. + lemma_e_mul_comm(fi, p); + lemma_e_mul_assoc(p, fi, h); + lemma_e_mul_one(p); + let vv = e_mul(fi, p); + assert(e_mul(vv, h) == p); + // (V d)(v h) = v ((V h) d). + lemma_e_mul_assoc(e_mul(vv, d), v, h); + lemma_e_mul_comm(e_mul(vv, d), v); + lemma_e_mul_assoc(v, e_mul(vv, d), h); + lemma_e_mul_assoc(vv, d, h); + lemma_e_mul_comm(d, h); + lemma_e_mul_assoc(vv, h, d); +} + +} // verus! diff --git a/verification/verus/tests/equivalence/bit_fold.rs b/verification/verus/tests/equivalence/bit_fold.rs new file mode 100644 index 0000000000..4cabba3df7 --- /dev/null +++ b/verification/verus/tests/equivalence/bit_fold.rs @@ -0,0 +1,128 @@ +//! The portable `Imp` is private and compiled only without AVX2; it is reached through `BitFold`, `F192Map` and +//! `Sliced`, which on this machine dispatch to the arm the target enables (AVX-512 with GFNI, AVX2, or portable). +use leanvm_verus::bit_fold as verified; +use leanvm_verus::gf2_64x3 as vx; +use primitives::bit_fold as production; +use primitives::field::F192; +use primitives::test_util::Rng; + +const BLOCK: usize = production::BLOCK; + +fn to_v(e: F192) -> vx::F192 { + vx::F192::new(e.c0, e.c1, e.c2) +} + +fn same(v: vx::F192, p: F192) -> bool { + (v.c0, v.c1, v.c2) == (p.c0, p.c1, p.c2) +} + +/// Random weights, then the edge cases: all zero, all ones, one coordinate bit each. +fn weight_sets(rng: &mut Rng, n: usize, random: usize) -> Vec> { + let mut sets: Vec> = (0..random).map(|_| rng.ext_vec(n)).collect(); + sets.push(vec![F192::ZERO; n]); + sets.push(vec![F192::new(u64::MAX, u64::MAX, u64::MAX); n]); + sets.push( + (0..n) + .map(|b| { + let mut w = [0u64; 3]; + w[(b / 64) % 3] = 1 << (b % 64); + F192::new(w[0], w[1], w[2]) + }) + .collect(), + ); + sets +} + +fn check_fold(rng: &mut Rng) { + assert_eq!(verified::BLOCK, BLOCK); + for weights in weight_sets(rng, 8 * CHUNKS, 20) { + let vw: Vec = weights.iter().map(|&w| to_v(w)).collect(); + let (fv, fp) = (verified::BitFold::new(&vw), production::BitFold::new(&weights)); + assert_eq!(fv.n_chunks(), fp.n_chunks()); + for len in [BLOCK, BLOCK, 5, 1, 0] { + let mut rows: Vec<[u8; CHUNKS]> = (0..len).map(|_| std::array::from_fn(|_| rng.next_u8())).collect(); + // Edge rows: all clear, all set, one set bit. + if len == BLOCK { + rows[0] = [0; CHUNKS]; + rows[1] = [0xFF; CHUNKS]; + rows[2] = std::array::from_fn(|j| if j == CHUNKS - 1 { 0x80 } else { 0 }); + } + // Only `out[..rows.len()]` is specified: past it the portable arm keeps what `out` held and the SIMD + // arms store the fold of a zero row. + let before: [F192; BLOCK] = std::array::from_fn(|_| rng.ext()); + let (mut ov, mut op) = (before.map(to_v), before); + fv.fold_block(&rows, &mut ov); + fp.fold_block(&rows, &mut op); + for p in 0..len { + assert!(same(ov[p], op[p]), "CHUNKS={CHUNKS}, len={len}, row {p}"); + } + assert!(ov[len..].iter().zip(&before[len..]).all(|(&v, &b)| same(v, b))); + } + } +} + +#[test] +fn fold_block_matches() { + let mut rng = Rng::new(0xB17_F01D_0E); + check_fold::<8>(&mut rng); + check_fold::<16>(&mut rng); + check_fold::<32>(&mut rng); + check_fold::<64>(&mut rng); + check_fold::<128>(&mut rng); +} + +/// Random values, then zero, all ones and every coordinate vector. +fn blocks(rng: &mut Rng) -> Vec<[F192; BLOCK]> { + let mut out: Vec<[F192; BLOCK]> = (0..4).map(|_| std::array::from_fn(|_| rng.ext())).collect(); + let mut edge = [F192::ZERO; BLOCK]; + edge[1] = F192::new(u64::MAX, u64::MAX, u64::MAX); + edge[2] = F192::new(1, 0, 1 << 63); + out.push(edge); + for chunk in 0..3 { + out.push(std::array::from_fn(|i| { + let b = BLOCK * chunk + i; + let mut w = [0u64; 3]; + w[b / 64] = 1 << (b % 64); + F192::new(w[0], w[1], w[2]) + })); + } + out +} + +#[test] +fn f192_map_matches() { + let mut rng = Rng::new(0x0F19_23A9_0E); + for weights in weight_sets(&mut rng, 192, 20) { + let vw: Vec = weights.iter().map(|&w| to_v(w)).collect(); + let (mv, mp) = (verified::F192Map::new(&vw), production::F192Map::new(&weights)); + let c = rng.ext(); + let (cv, cp) = (mv.after_mul(to_v(c)), mp.after_mul(c)); + for xs in blocks(&mut rng) { + let xv = xs.map(to_v); + let (sv, sp) = (verified::Sliced::new(&xv), production::Sliced::new(&xs)); + for len in [BLOCK, 7, 1, 0] { + let before: Vec = (0..len).map(|_| rng.ext()).collect(); + let mut ov: Vec = before.iter().map(|&b| to_v(b)).collect(); + let mut op = before.clone(); + mv.apply_add(&xv, &mut ov); + mp.apply_add(&xs, &mut op); + assert!(ov.iter().zip(&op).all(|(&v, &p)| same(v, p)), "apply_add, len={len}"); + + let mut ov: Vec = before.iter().map(|&b| to_v(b)).collect(); + let mut op = before.clone(); + mv.apply_sliced_add(&sv, &mut ov); + mp.apply_sliced_add(&sp, &mut op); + assert!( + ov.iter().zip(&op).all(|(&v, &p)| same(v, p)), + "apply_sliced_add, len={len}" + ); + + let mut ov: Vec = before.iter().map(|&b| to_v(b)).collect(); + let mut op = before.clone(); + cv.apply_sliced_add(&sv, &mut ov); + cp.apply_sliced_add(&sp, &mut op); + assert!(ov.iter().zip(&op).all(|(&v, &p)| same(v, p)), "after_mul, len={len}"); + } + } + } +} diff --git a/verification/verus/tests/equivalence/bits.rs b/verification/verus/tests/equivalence/bits.rs new file mode 100644 index 0000000000..74081b488d --- /dev/null +++ b/verification/verus/tests/equivalence/bits.rs @@ -0,0 +1,139 @@ +use leanvm_verus::bits as verified; +use primitives::bits as production; +use primitives::test_util::Rng; + +/// Zero, all ones, the diagonal, the anti-diagonal, one row, one column, then random matrices. +fn matrices() -> impl Iterator { + let edges: [[u64; 64]; 6] = [ + [0; 64], + [u64::MAX; 64], + std::array::from_fn(|r| 1 << r), + std::array::from_fn(|r| 1 << (63 - r)), + std::array::from_fn(|r| if r == 5 { u64::MAX } else { 0 }), + [1 << 37; 64], + ]; + let mut rng = Rng::new(0x6464_7777); + edges + .into_iter() + .chain((0..10_000).map(move |_| std::array::from_fn(|_| rng.next_u64()))) +} + +#[test] +fn transpose_64x64_matches() { + for m in matrices() { + let (mut v, mut p) = (m, m); + verified::transpose_64x64(&mut v); + production::transpose_64x64(&mut p); + assert_eq!(v, p); + } +} + +/// Zero, all ones, one bit per byte, then random blocks. +fn blocks() -> impl Iterator { + let edges: [[u8; 64]; 4] = [ + [0; 64], + [0xFF; 64], + std::array::from_fn(|i| 1 << (i % 8)), + std::array::from_fn(|i| i as u8), + ]; + let mut rng = Rng::new(0xB17_7777); + edges + .into_iter() + .chain((0..10_000).map(move |_| std::array::from_fn(|_| rng.next_u8()))) +} + +/// Production's portable arm is private: the verified copy is compared with the dispatched public function, whichever +/// arm this target compiles. +#[test] +fn bit_transpose_64bytes_matches() { + for input in blocks() { + let (mut v, mut p) = ([0u8; 64], [0u8; 64]); + verified::bit_transpose_64bytes_portable(&input, &mut v); + production::bit_transpose_64bytes(&input, &mut p); + assert_eq!(v, p); + } +} + +/// Production's `transpose_8x8_bits` is private: the verified copy is checked against the definition, and against +/// the public 64-byte transpose, whose row `b` is the 8x8 transpose of byte column `b`. +#[test] +fn transpose_8x8_bits_matches() { + let mut rng = Rng::new(0x88_7777); + for x in [0, u64::MAX, 1, 1 << 63, 0x8040_2010_0804_0201] + .into_iter() + .chain((0..10_000).map(|_| rng.next_u64())) + { + let mut want = 0u64; + for r in 0..8 { + for c in 0..8 { + want |= ((x >> (r * 8 + c)) & 1) << (c * 8 + r); + } + } + assert_eq!(verified::transpose_8x8_bits(x), want, "definition {x:#x}"); + } + for input in blocks() { + let mut p = [0u8; 64]; + production::bit_transpose_64bytes(&input, &mut p); + for b in 0..8 { + let column = u64::from_le_bytes(std::array::from_fn(|x| input[8 * x + b])); + let row = u64::from_le_bytes(p[8 * b..8 * b + 8].try_into().unwrap()); + assert_eq!(verified::transpose_8x8_bits(column), row, "column {b}"); + } + } +} + +/// The definition, one bit at a time. +fn reference(input: &[u8; 64]) -> [u8; 64] { + let mut output = [0u8; 64]; + for x in 0..8 { + for b in 0..8 { + for t in 0..8 { + output[b * 8 + t] |= ((input[x * 8 + b] >> t) & 1) << x; + } + } + } + output +} + +/// Every verified arm this build compiles, against the definition and against production's dispatched function +/// (production's arms are private), like production's `every_arm_matches_reference`. Built with +/// `target-cpu=native` on an AVX-512 VBMI and GFNI machine this runs the three x86 arms; under `--target aarch64-*` +/// the NEON one. +#[test] +fn every_arm_matches_reference() { + for input in blocks() { + let want = reference(&input); + let mut p = [0u8; 64]; + production::bit_transpose_64bytes(&input, &mut p); + assert_eq!(p, want, "production"); + let mut got = [0u8; 64]; + verified::bit_transpose_64bytes(&input, &mut got); + assert_eq!(got, want, "dispatched"); + verified::bit_transpose_64bytes_portable(&input, &mut got); + assert_eq!(got, want, "portable"); + #[cfg(all(target_arch = "x86_64", target_feature = "avx512vbmi", target_feature = "gfni"))] + { + // SAFETY: compiled only with the features enabled. + unsafe { verified::bit_transpose_64bytes_gfni(&input, &mut got) }; + assert_eq!(got, want, "gfni"); + } + #[cfg(all(target_arch = "x86_64", target_feature = "avx2", target_feature = "gfni"))] + { + // SAFETY: compiled only with the features enabled. + unsafe { verified::bit_transpose_64bytes_gfni_avx2(&input, &mut got) }; + assert_eq!(got, want, "gfni avx2"); + } + #[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] + { + // SAFETY: compiled only with the feature enabled. + unsafe { verified::bit_transpose_64bytes_avx2(&input, &mut got) }; + assert_eq!(got, want, "avx2"); + } + #[cfg(target_arch = "aarch64")] + { + // SAFETY: aarch64 always has NEON. + unsafe { verified::bit_transpose_64bytes_neon(&input, &mut got) }; + assert_eq!(got, want, "neon"); + } + } +} diff --git a/verification/verus/tests/equivalence/blake2s.rs b/verification/verus/tests/equivalence/blake2s.rs new file mode 100644 index 0000000000..c6bb06521f --- /dev/null +++ b/verification/verus/tests/equivalence/blake2s.rs @@ -0,0 +1,166 @@ +use leanvm_verus::blake2s as verified; +use primitives::hash as production; +use primitives::test_util::Rng; + +fn random_bytes(rng: &mut Rng, n: usize) -> Vec { + (0..n).map(|_| rng.next_u8()).collect() +} + +#[test] +fn constants_match() { + assert_eq!(verified::IV, production::IV); + assert_eq!(verified::PARAM_IV, production::PARAM_IV); + assert_eq!(verified::SIGMA, production::SIGMA); + assert_eq!(verified::G_LANES, production::G_LANES); + assert_eq!(verified::OUT_LEN, production::OUT_LEN); + assert_eq!(verified::BLOCK_LEN, production::BLOCK_LEN); + assert_eq!(verified::ROUNDS, production::ROUNDS); +} + +/// The trusted specification of `u32::rotate_right` (`0 < n < 32`): the RFC's `(x >> n) ^ (x << (32 - n))`. +#[test] +fn rotate_right_is_the_rfc_rotation() { + let mut rng = Rng::new(0x2077_7777); + let edges = [0u32, 1, u32::MAX, 1 << 31, 0x8000_0001, 0xDEAD_BEEF]; + for x in edges.into_iter().chain((0..10_000).map(|_| rng.next_u32())) { + for n in 1..32u32 { + assert_eq!(x.rotate_right(n), (x >> n) ^ (x << (32 - n)), "{x:#x} >>> {n}"); + } + } +} + +/// Counters at both halves' edges, both flags, then random states and blocks. +#[test] +fn compress_matches() { + let mut rng = Rng::new(0xB2_7777); + let counters = [ + 0u64, + 1, + 64, + u32::MAX as u64, + 1 << 32, + (1 << 32) + 64, + u64::MAX - 63, + u64::MAX, + ]; + for i in 0..20_000 { + let h: [u32; 8] = if i == 0 { + [0; 8] + } else { + std::array::from_fn(|_| rng.next_u32()) + }; + let m: [u32; 16] = match i { + 0 => [0; 16], + 1 => [u32::MAX; 16], + _ => std::array::from_fn(|_| rng.next_u32()), + }; + let t = if i < counters.len() * 4 { + counters[i % counters.len()] + } else { + rng.next_u64() + }; + let last = rng.bit(); + let (mut v, mut p) = (h, h); + verified::compress(&mut v, &m, t, last); + production::compress(&mut p, &m, t, last); + assert_eq!(v, p, "h {h:x?} m {m:x?} t {t} last {last}"); + } +} + +/// Production's `block_words` and `state_bytes` are private: the verified copies are checked against the +/// little-endian conversions they replace, and through `hash` below. +#[test] +fn byte_conversions_are_little_endian() { + let mut rng = Rng::new(0x1E_7777); + for _ in 0..10_000 { + let block: [u8; 64] = std::array::from_fn(|_| rng.next_u8()); + let want: [u32; 16] = std::array::from_fn(|i| u32::from_le_bytes(block[4 * i..4 * i + 4].try_into().unwrap())); + assert_eq!(verified::block_words(&block), want); + let h: [u32; 8] = std::array::from_fn(|_| rng.next_u32()); + let want: Vec = h.iter().flat_map(|w| w.to_le_bytes()).collect(); + assert_eq!(verified::state_bytes(&h).to_vec(), want); + } +} + +/// Every length up to five blocks (both the whole-block fast path and the streaming path), then random lengths. +#[test] +fn hash_matches() { + let mut rng = Rng::new(0x4A54_7777); + let lengths: Vec = (0..=320) + .chain((0..300).map(|_| (rng.next_u32() % 4096) as usize)) + .collect(); + for n in lengths { + let data = random_bytes(&mut rng, n); + assert_eq!(verified::hash(&data), production::hash(&data), "{n} bytes"); + } + for (input, digest) in primitives::test_util::test_vectors() { + assert_eq!(verified::hash(&input), digest, "official vector, {} bytes", input.len()); + } +} + +/// RFC 7693, Appendix B: BLAKE2s-256("abc"). The verified `hash` is proven equal to the specification +/// `blake2s_spec`, so this checks the transcription of the RFC against the RFC's own example. +#[test] +fn rfc_7693_appendix_b() { + let want = [ + 0x50, 0x8C, 0x5E, 0x8C, 0x32, 0x7C, 0x14, 0xE2, 0xE1, 0xA7, 0x2B, 0xA3, 0x4E, 0xEB, 0x45, 0x2F, 0x37, 0x45, + 0x8B, 0x20, 0x9E, 0xD6, 0x3A, 0x29, 0x4D, 0x99, 0x9B, 0x4C, 0x86, 0x67, 0x59, 0x82, + ]; + assert_eq!(verified::hash(b"abc"), want); + assert_eq!(production::hash(b"abc"), want); + let mut hasher = verified::Hasher::new(); + hasher.update(b"a").update(b"bc"); + assert_eq!(hasher.finalize(), want); +} + +/// Random messages fed in random pieces, empty ones included, finalized at every step. +#[test] +fn hasher_matches() { + let mut rng = Rng::new(0x5743_7777); + for _ in 0..500 { + let (mut v, mut p) = (verified::Hasher::new(), production::Hasher::new()); + assert_eq!(v.finalize(), p.finalize()); + for _ in 0..(rng.next_u32() % 12) { + let piece = match rng.next_u32() % 4 { + 0 => 0, + 1 => 64, + 2 => (rng.next_u32() % 70) as usize, + _ => (rng.next_u32() % 300) as usize, + }; + let data = random_bytes(&mut rng, piece); + v.update(&data); + p.update(&data); + assert_eq!(v.finalize(), p.finalize()); + assert_eq!(v.clone().finalize(), p.finalize()); + } + } +} + +#[test] +fn zero_prefix_and_continuation_match() { + let mut rng = Rng::new(0x2E40_7777); + for z in 0..12 { + let state = verified::zero_prefix_state(z); + assert_eq!(state, production::zero_prefix_state(z), "zero prefix {z}"); + for rest_blocks in 1..5 { + let rest = random_bytes(&mut rng, rest_blocks * 64); + let t = (z * 64) as u64; + assert_eq!( + verified::hash_from_state(&rest, &state, t), + production::hash_from_state(&rest, &state, t) + ); + } + } + // Arbitrary states and offsets, the counter's high word included. + for _ in 0..2_000 { + let state: [u32; 8] = std::array::from_fn(|_| rng.next_u32()); + let t = rng.next_u64() >> (rng.next_u32() % 64); + let t = t.min(u64::MAX - 1024); + let n = 64 * (1 + (rng.next_u32() % 8) as usize); + let rest = random_bytes(&mut rng, n); + assert_eq!( + verified::hash_from_state(&rest, &state, t), + production::hash_from_state(&rest, &state, t) + ); + } +} diff --git a/verification/verus/tests/equivalence/blake2s_batch.rs b/verification/verus/tests/equivalence/blake2s_batch.rs new file mode 100644 index 0000000000..67d9073bd1 --- /dev/null +++ b/verification/verus/tests/equivalence/blake2s_batch.rs @@ -0,0 +1,137 @@ +use leanvm_verus::blake2s as scalar; +use leanvm_verus::blake2s_batch::{self as verified, Lanes32, Scalar8}; +use primitives::hash as production; +use primitives::test_util::Rng; + +/// The production walk's layout, written plainly: input `g * 8 + l` of a set is lane `l` of group `g`, its block `b` +/// transposed word by word, every lane starting from `state`, digests written back lane by lane. +fn hash_set_with_verified_compression( + data: &[u8], + len: usize, + state: &[u32; 8], + t_offset: u64, +) -> Vec { + let n_blocks = len / 64; + let mut h: [[Scalar8; 8]; G] = [verified::transpose_words(&[*state; 8]); G]; + for b in 0..n_blocks { + let blocks: [[Scalar8; 16]; G] = std::array::from_fn(|g| { + let rows: [[u32; 16]; 8] = std::array::from_fn(|l| { + std::array::from_fn(|w| { + let at = (g * 8 + l) * len + b * 64 + 4 * w; + u32::from_le_bytes(data[at..at + 4].try_into().unwrap()) + }) + }); + verified::transpose_words(&rows) + }); + let m: [&[Scalar8; 16]; G] = std::array::from_fn(|g| &blocks[g]); + let t = t_offset + ((b + 1) * 64) as u64; + verified::compress_groups::(&mut h, m, t, b + 1 == n_blocks); + } + let mut out = vec![0u8; G * 8 * 32]; + for g in 0..G { + let digests = verified::store_digests(&h[g]); + for l in 0..8 { + let at = (g * 8 + l) * 32; + out[at..at + 32].copy_from_slice(&digests[l]); + } + } + out +} + +/// Production's batched path is private: the verified `compress_groups` (portable backend, 1, 2 and 4 groups), in a +/// plain transposing walk, is compared with production's public batched hash, whichever backend this target +/// dispatches to, on fresh and continued states. +#[test] +fn batched_hash_matches_production() { + fn check(rng: &mut Rng) { + for len in [64usize, 128, 192, 640] { + for continued in [false, true] { + let (state, t_offset) = if continued { + let z = (rng.next_u32() % 4) as usize; + (production::zero_prefix_state(z), (z * 64) as u64) + } else { + (production::PARAM_IV, 0) + }; + let n = G * 8; + let data: Vec = (0..n * len).map(|_| rng.next_u8()).collect(); + let mut want = vec![0u8; n * 32]; + production::hash_many_dyn_from_state(&data, len, &state, t_offset, &mut want); + let got = hash_set_with_verified_compression::(&data, len, &state, t_offset); + assert_eq!(got, want, "G {G} len {len} continued {continued}"); + } + } + } + let mut rng = Rng::new(0xBA7C_7777); + for _ in 0..20 { + check::<1>(&mut rng); + check::<2>(&mut rng); + check::<4>(&mut rng); + } +} + +/// Lane by lane, the batched compression is the verified scalar `compress` (itself checked against production), on +/// arbitrary transposed states and blocks, both counter halves and both flags. +#[test] +fn compress_groups_is_lanewise_compress() { + let mut rng = Rng::new(0x1A2E_7777); + for i in 0..2_000 { + let states: [[[u32; 8]; 8]; 2] = + std::array::from_fn(|_| std::array::from_fn(|_| std::array::from_fn(|_| rng.next_u32()))); + let messages: [[[u32; 16]; 8]; 2] = + std::array::from_fn(|_| std::array::from_fn(|_| std::array::from_fn(|_| rng.next_u32()))); + let h = states.map(|rows| verified::transpose_words(&rows)); + let blocks = messages.map(|rows| verified::transpose_words(&rows)); + let counters = [0, 64, u32::MAX as u64, 1 << 32, (1 << 32) + 64, u64::MAX]; + let t = if i < counters.len() * 2 { + counters[i / 2] + } else { + rng.next_u64() + }; + let last = i % 2 == 0; + let mut got = h; + verified::compress_groups::(&mut got, [&blocks[0], &blocks[1]], t, last); + let mut one = [h[1]]; + verified::compress_groups::(&mut one, [&blocks[1]], t, last); + let row_digests = verified::compress_rows(&states[1], &messages[1], t, last); + for g in 0..2 { + let digests = verified::store_digests(&got[g]); + for l in 0..8 { + let mut want = states[g][l]; + let m = messages[g][l]; + scalar::compress(&mut want, &m, t, last); + assert_eq!( + std::array::from_fn::(|w| got[g][w].0[l]), + want, + "group {g} lane {l}" + ); + assert_eq!(digests[l], scalar::state_bytes(&want), "scatter group {g} lane {l}"); + if g == 1 { + assert_eq!(row_digests[l], digests[l], "composed layout lane {l}"); + } + } + } + for w in 0..8 { + assert_eq!(one[0][w].0, got[1][w].0, "one group against two, word {w}"); + } + } +} + +/// The portable lane arithmetic, against the scalar operations it maps. +#[test] +fn scalar8_lanes_match() { + let mut rng = Rng::new(0x5CA8_7777); + for _ in 0..10_000 { + let a = Scalar8(std::array::from_fn(|_| rng.next_u32())); + let b = Scalar8(std::array::from_fn(|_| rng.next_u32())); + let x = rng.next_u32(); + for l in 0..8 { + assert_eq!(a.add(b).0[l], a.0[l].wrapping_add(b.0[l])); + assert_eq!(a.xor(b).0[l], a.0[l] ^ b.0[l]); + assert_eq!(a.rotr::<16>().0[l], a.0[l].rotate_right(16)); + assert_eq!(a.rotr::<12>().0[l], a.0[l].rotate_right(12)); + assert_eq!(a.rotr::<8>().0[l], a.0[l].rotate_right(8)); + assert_eq!(a.rotr::<7>().0[l], a.0[l].rotate_right(7)); + assert_eq!(Scalar8::splat(x).0[l], x); + } + } +} diff --git a/verification/verus/tests/equivalence/fiat_shamir.rs b/verification/verus/tests/equivalence/fiat_shamir.rs new file mode 100644 index 0000000000..57752d8750 --- /dev/null +++ b/verification/verus/tests/equivalence/fiat_shamir.rs @@ -0,0 +1,57 @@ +//! The verified transcript-step block against `fiat_shamir::step_block`. +//! +//! The circuit's copy (`builder_step_message`) has no public production counterpart (`Builder` lives in +//! `leanvm_core`'s private `rec` module); it is checked against the native block here, and production's +//! `rec::transcript::tests::the_circuit_replays_the_native_transcript` checks the circuit's steps hash to the +//! native ones. +use leanvm_verus::fiat_shamir as verified; +use leanvm_verus::gf2_64::F64 as VF64; +use leanvm_verus::gf2_64x3::F192 as VF192; +use primitives::field::{F192, F64}; +use primitives::test_util::Rng; + +fn to_v(e: F192) -> VF192 { + VF192::new(e.c0, e.c1, e.c2) +} + +fn scalars(rng: &mut Rng) -> Vec { + let n = (rng.next_u32() % 3) as usize; + (0..n) + .map(|_| match rng.next_u32() % 4 { + 0 => F192::ZERO, + 1 => F192::new(u64::MAX, u64::MAX, u64::MAX), + _ => rng.ext(), + }) + .collect() +} + +#[test] +fn step_blocks_match() { + let mut rng = Rng::new(0xF5B1); + let tags = [0, 1, 2, 3, 4, u64::MAX]; + for round in 0..20_000 { + let s = scalars(&mut rng); + let tag = if round % 2 == 0 { + tags[round / 2 % tags.len()] + } else { + rng.next_u64() + }; + let want = fiat_shamir::step_block(&s, F64(tag)).map(|w| w.0); + let vs: Vec = s.iter().copied().map(to_v).collect(); + assert_eq!( + verified::step_block(&vs, VF64(tag)).map(|w| w.0), + want, + "{s:?} {tag:#x}" + ); + assert_eq!(verified::builder_step_message(&vs, tag), want, "{s:?} {tag:#x}"); + } +} + +#[test] +fn constants_match() { + assert_eq!(verified::MAX_PENDING, fiat_shamir::MAX_PENDING); + assert_eq!(verified::DS_OBSERVE.0, fiat_shamir::DS_OBSERVE.0); + assert_eq!(verified::DS_SQUEEZE.0, fiat_shamir::DS_SQUEEZE.0); + assert_eq!(verified::DS_POW_BASE.0, fiat_shamir::DS_POW_BASE.0); + assert_eq!(verified::DS_POW_NONCE.0, fiat_shamir::DS_POW_NONCE.0); +} diff --git a/verification/verus/tests/equivalence/flock_ntt.rs b/verification/verus/tests/equivalence/flock_ntt.rs new file mode 100644 index 0000000000..c05e577cc8 --- /dev/null +++ b/verification/verus/tests/equivalence/flock_ntt.rs @@ -0,0 +1,123 @@ +//! The verified GF(2^8) NTT and LDE table against flock's `zerocheck/ntt.rs` and `zerocheck/ntt/inv_table.rs`. +//! +//! Everything there is `pub(crate)` and flock exports no path to it, so the two production source files are +//! compiled into this test binary as a module of their own (the `#[path]` module below) and called directly: +//! the code under test is production's, byte for byte. Their own `#[cfg(test)]` tests come along and run here +//! too. +//! +//! The twiddle table is private to its struct, so it is checked through `forward` and `inverse`, which read +//! every entry; the LDE table is private too, so it is checked through `apply_scalar` on every single-byte row +//! (which reads one table row) and on random rows, and the dispatched `apply` (the SIMD arms on this machine) +//! against the verified `apply_scalar`. +use leanvm_verus::flock_ntt as verified; +use leanvm_verus::gf2_8::F8 as VF8; +use primitives::field::F8; +use primitives::test_util::Rng; + +#[allow(dead_code, clippy::all)] +#[path = "../../../../crates/flock/src/zerocheck"] +mod production { + pub mod ntt; +} + +use production::ntt::{AdditiveNttGf8 as Production, InvNttTableByteSingleGf8 as ProductionTable}; + +fn random(rng: &mut Rng, n: usize) -> Vec { + (0..n).map(|_| F8(rng.next_u8())).collect() +} + +fn to_verified(v: &[F8]) -> Vec { + v.iter().map(|x| VF8(x.0)).collect() +} + +fn from_verified(v: &[VF8]) -> Vec { + v.iter().map(|x| F8(x.0)).collect() +} + +#[test] +fn forward_and_inverse_match_production() { + // Every domain size up to the whole field, every offset, a few vectors each, and the unit vectors. + let mut rng = Rng::new(0xF8_0717); + for k in 0..=8usize { + for beta in 0..=255u8 { + let (ours, theirs) = ( + verified::AdditiveNttGf8::new(k, VF8(beta)), + Production::new(k, F8(beta)), + ); + assert_eq!(ours.k(), theirs.k()); + let mut inputs: Vec> = (0..2).map(|_| random(&mut rng, 1 << k)).collect(); + if beta % 51 == 0 { + for t in 0..1usize << k { + let mut e = vec![F8::ZERO; 1 << k]; + e[t] = F8::ONE; + inputs.push(e); + } + } + for v in inputs { + let mut want = to_verified(&v); + ours.forward(&mut want); + let mut got = v.clone(); + theirs.forward(&mut got); + assert_eq!(got, from_verified(&want), "forward k={k}, beta={beta:#04x}"); + + let mut want = to_verified(&v); + ours.inverse(&mut want); + let mut got = v.clone(); + theirs.inverse(&mut got); + assert_eq!(got, from_verified(&want), "inverse k={k}, beta={beta:#04x}"); + } + } + } +} + +#[test] +fn lde_table_matches_production() { + // The protocol's pair (S at 0, Λ at 2^k), and random offsets, at every size the table accepts. + let mut rng = Rng::new(0x1DE7); + for k in 3..=7usize { + let ell = 1usize << k; + let mut pairs = vec![(0u8, 1u8 << k)]; + pairs.extend((0..6).map(|_| (rng.next_u8(), rng.next_u8()))); + for (beta_s, beta_l) in pairs { + let ours = verified::InvNttTableByteSingleGf8::new( + &verified::AdditiveNttGf8::new(k, VF8(beta_s)), + &verified::AdditiveNttGf8::new(k, VF8(beta_l)), + ); + let theirs = ProductionTable::new(&Production::new(k, F8(beta_s)), &Production::new(k, F8(beta_l))); + assert_eq!( + (ours.k, ours.ell, ours.n_chunks), + (theirs.k, theirs.ell, theirs.n_chunks) + ); + + let mut rows: Vec> = Vec::new(); + // Single-byte rows: byte `w` at chunk `b`, every `w` and `b`, so every table row at every shift. + for b in 0..ell / 8 { + for w in 0..=255u8 { + let mut bytes = vec![0u8; ell / 8]; + bytes[b] = w; + rows.push(bytes); + } + } + rows.extend((0..256).map(|_| (0..ell / 8).map(|_| rng.next_u8()).collect())); + rows.push(vec![0xFF; ell / 8]); + + for bytes in rows { + let mut want = vec![VF8(0); ell]; + ours.apply_scalar(&bytes, &mut want); + let want = from_verified(&want); + let mut got = vec![F8::ZERO; ell]; + theirs.apply_scalar(&bytes, &mut got); + assert_eq!( + got, want, + "apply_scalar k={k}, betas=({beta_s:#04x}, {beta_l:#04x}), bytes={bytes:02x?}" + ); + let mut got = vec![F8::ZERO; ell]; + theirs.apply(&bytes, &mut got); + assert_eq!( + got, want, + "apply k={k}, betas=({beta_s:#04x}, {beta_l:#04x}), bytes={bytes:02x?}" + ); + } + } + } +} diff --git a/verification/verus/tests/equivalence/gf2_64.rs b/verification/verus/tests/equivalence/gf2_64.rs new file mode 100644 index 0000000000..3dad9c9569 --- /dev/null +++ b/verification/verus/tests/equivalence/gf2_64.rs @@ -0,0 +1,144 @@ +use leanvm_verus::gf2_64 as verified; +use primitives::field::gf2_64 as production; +use primitives::test_util::Rng; + +/// Operands that exercise the reduction's spill and the identities. +const CORNERS: [u64; 9] = [ + 0, + 1, + 2, + u64::MAX, + 1 << 63, + 0xF000_0000_0000_0000, + 0x1B, + 0x8000_0000_0000_001B, + 0x5555_5555_5555_5555, +]; + +fn pairs() -> impl Iterator { + let mut rng = Rng::new(0x6464_6464); + let random: Vec<(u64, u64)> = (0..100_000).map(|_| (rng.next_u64(), rng.next_u64())).collect(); + let corners = CORNERS.iter().flat_map(|&a| CORNERS.iter().map(move |&b| (a, b))); + random.into_iter().chain(corners) +} + +#[test] +fn products_and_reduction_match() { + for (a, b) in pairs() { + assert_eq!( + verified::software::clmul(a, b), + production::software::clmul(a, b), + "clmul {a:#x} {b:#x}" + ); + assert_eq!( + verified::mul_wide(a, b), + production::mul_wide(a, b), + "mul_wide {a:#x} {b:#x}" + ); + assert_eq!( + verified::square_wide(a), + production::square_wide(a), + "square_wide {a:#x}" + ); + let wide = (u128::from(a) << 64) | u128::from(b); + assert_eq!(verified::reduce(wide), production::reduce(wide), "reduce {wide:#x}"); + let p = verified::software::clmul(a, b); + assert_eq!(verified::reduce(p), production::reduce(p), "reduce {p:#x}"); + assert_eq!( + (verified::F64(a) * verified::F64(b)).0, + (production::F64(a) * production::F64(b)).0, + "mul {a:#x} {b:#x}" + ); + assert_eq!( + (verified::F64(a) + verified::F64(b)).0, + (production::F64(a) + production::F64(b)).0 + ); + assert_eq!( + verified::F64(a).square().0, + production::F64(a).square().0, + "square {a:#x}" + ); + } +} + +#[test] +fn inverses_match() { + let mut rng = Rng::new(0x1A7); + for a in (0..2_000).map(|_| rng.next_u64()).chain(CORNERS) { + assert_eq!(verified::F64(a).inv().0, production::F64(a).inv().0, "inv {a:#x}"); + } +} + +#[test] +fn constants_match() { + assert_eq!(verified::R64, production::R64); + assert_eq!(verified::F64::DEGREE, production::F64::DEGREE); + assert_eq!(verified::F64::ZERO.0, production::F64::ZERO.0); + assert_eq!(verified::F64::ONE.0, production::F64::ONE.0); + assert_eq!(verified::F64::G.0, production::F64::G.0); +} + +/// The x86-64 kernels, each against production's on this CPU (whatever the build's flags). +#[cfg(target_arch = "x86_64")] +#[test] +fn x86_kernels_match() { + if !is_x86_feature_detected!("pclmulqdq") || !is_x86_feature_detected!("bmi2") { + eprintln!("this CPU has no pclmulqdq or bmi2: the x86-64 kernels are not compared here"); + return; + } + for (a, b) in pairs() { + unsafe { + assert_eq!( + verified::x86_64::mul(a, b), + production::x86_64::mul(a, b), + "mul {a:#x} {b:#x}" + ); + assert_eq!( + verified::x86_64::clmul(a, b), + production::x86_64::clmul(a, b), + "clmul {a:#x} {b:#x}" + ); + assert_eq!( + verified::x86_64::spread(a), + production::x86_64::spread(a), + "spread {a:#x}" + ); + } + } +} + +/// The aarch64 kernels, each against production's. +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +#[test] +fn aarch64_kernels_match() { + use core::arch::aarch64::uint64x2_t; + use core::mem::transmute; + let w = |v: uint64x2_t| unsafe { transmute::(v) }; + let mut rng = Rng::new(0xA64); + for (a, b) in pairs() { + unsafe { + assert_eq!(w(verified::aarch64::pmull(a, b)), w(production::aarch64::pmull(a, b))); + let (p, q) = (production::aarch64::pmull(a, b), production::aarch64::pmull(b ^ a, a)); + assert_eq!( + w(verified::aarch64::pmull_hi(p, q)), + w(production::aarch64::pmull_hi(p, q)) + ); + assert_eq!( + w(verified::aarch64::reduce_pair_pmull4(p, q)), + w(production::aarch64::reduce_pair_pmull4(p, q)) + ); + let (x, y) = ( + transmute::<[u64; 2], uint64x2_t>([rng.next_u64(), rng.next_u64()]), + transmute([rng.next_u64(), rng.next_u64()]), + ); + assert_eq!( + w(verified::aarch64::reduce_pair_pmull4(x, y)), + w(production::aarch64::reduce_pair_pmull4(x, y)) + ); + assert_eq!( + verified::aarch64::mul_shift_tail(verified::F64(a), verified::F64(b)).0, + production::aarch64::mul_shift_tail(production::F64(a), production::F64(b)).0 + ); + } + } +} diff --git a/verification/verus/tests/equivalence/gf2_64x3.rs b/verification/verus/tests/equivalence/gf2_64x3.rs new file mode 100644 index 0000000000..91eda06d5c --- /dev/null +++ b/verification/verus/tests/equivalence/gf2_64x3.rs @@ -0,0 +1,427 @@ +use leanvm_verus::gf2_64 as vk; +use leanvm_verus::gf2_64x3 as verified; +use primitives::field as production; +use primitives::test_util::Rng; + +const CORNER_WORDS: [u64; 6] = [0, 1, u64::MAX, 1 << 63, 0xF000_0000_0000_0000, 0x1B]; + +fn to_v(e: production::F192) -> verified::F192 { + verified::F192::new(e.c0, e.c1, e.c2) +} + +fn same(v: verified::F192, p: production::F192) -> bool { + (v.c0, v.c1, v.c2) == (p.c0, p.c1, p.c2) +} + +/// The unreduced values compared word for word, through their (identical) `Debug` forms: the production +/// fields are private. +fn same_unreduced(v: &verified::F192Unreduced, p: &production::F192Unreduced) -> bool { + format!("{v:?}") == format!("{p:?}") +} + +fn elements(rng: &mut Rng, n: usize) -> Vec { + let mut out: Vec = (0..n) + .map(|_| production::F192::new(rng.next_u64(), rng.next_u64(), rng.next_u64())) + .collect(); + for &a in &CORNER_WORDS { + for &b in &CORNER_WORDS { + out.push(production::F192::new(a, b, a ^ b)); + out.push(production::F192::new(a, 0, b)); + } + } + out +} + +#[test] +fn products_match() { + let mut rng = Rng::new(0x192); + let xs = elements(&mut rng, 20_000); + let ys = elements(&mut rng, 20_000); + for (&a, &b) in xs.iter().zip(&ys) { + let (va, vb) = (to_v(a), to_v(b)); + assert!(same(va * vb, a * b), "mul {a:?} {b:?}"); + assert!(same( + verified::software::mul(va, vb), + production::gf2_64x3::software::mul(a, b) + )); + assert!( + same_unreduced(&va.mul_unreduced(vb), &a.mul_unreduced(b)), + "mul_unreduced {a:?} {b:?}" + ); + assert!(same_unreduced( + &verified::software::mul_unreduced(va, vb), + &production::gf2_64x3::software::mul_unreduced(a, b) + )); + assert!(same(va.square(), a.square()), "square {a:?}"); + assert!(same(va.frobenius(), a.frobenius())); + assert!(same(va + vb, a + b)); + let k = b.c1; + assert!( + same(va.mul_base(vk::F64(k)), a.mul_base(production::F64(k))), + "mul_base {a:?} {k:#x}" + ); + assert!(same_unreduced( + &va.mul_base_unreduced(vk::F64(k)), + &a.mul_base_unreduced(production::F64(k)) + )); + assert!(same( + verified::F192::from(vk::F64(k)), + production::F192::from(production::F64(k)) + )); + assert!(same_unreduced( + &verified::F192Unreduced::from(va), + &production::F192Unreduced::from(a) + )); + let (mut sv, mut sp) = (va, a); + sv += vb; + sp += b; + assert!(same(sv, sp)); + sv *= vb; + sp *= b; + assert!(same(sv, sp)); + } +} + +#[test] +fn lazy_reduction_matches() { + let mut rng = Rng::new(0x1A2); + let xs = elements(&mut rng, 4_000); + let mut acc_v = verified::F192Unreduced::ZERO; + let mut acc_p = production::F192Unreduced::ZERO; + for w in xs.windows(2) { + let (a, b) = (w[0], w[1]); + acc_v ^= to_v(a).mul_unreduced(to_v(b)); + acc_p ^= a.mul_unreduced(b); + acc_v = acc_v ^ to_v(b).mul_base_unreduced(vk::F64(a.c2)); + acc_p = acc_p ^ b.mul_base_unreduced(production::F64(a.c2)); + assert!(same_unreduced(&acc_v, &acc_p)); + assert!(same(acc_v.reduce(), acc_p.reduce())); + } +} + +#[test] +fn batched_products_match() { + let mut rng = Rng::new(0x48); + let xs = elements(&mut rng, 4_000); + for c in xs.chunks_exact(8) { + let a: [production::F192; 4] = [c[0], c[1], c[2], c[3]]; + let b: [production::F192; 4] = [c[4], c[5], c[6], c[7]]; + let (va, vb) = (a.map(to_v), b.map(to_v)); + let (r2v, r2p) = ( + verified::mul2([va[0], va[1]], [vb[0], vb[1]]), + production::mul2([a[0], a[1]], [b[0], b[1]]), + ); + assert!(r2v.iter().zip(&r2p).all(|(&v, &p)| same(v, p))); + let (r4v, r4p) = (verified::mul4(va, vb), production::mul4(a, b)); + assert!(r4v.iter().zip(&r4p).all(|(&v, &p)| same(v, p))); + let (u4v, u4p) = (verified::mul_unreduced4(va, vb), production::mul_unreduced4(a, b)); + assert!(u4v.iter().zip(&u4p).all(|(v, p)| same_unreduced(v, p))); + let ks: [u64; 8] = std::array::from_fn(|i| c[i].c0 ^ c[i].c2); + let (m8v, m8p) = ( + verified::mul_base8(va[0], ks.map(vk::F64)), + production::mul_base8(a[0], ks.map(production::F64)), + ); + assert!(m8v.iter().zip(&m8p).all(|(&v, &p)| same(v, p))); + } +} + +#[test] +fn weights_and_dot_base_match() { + let mut rng = Rng::new(0xD07); + for len in [0usize, 1, 2, 5, 16] { + let ws = elements(&mut rng, 8 * len); + let ws = &ws[..8 * len]; + let ks: Vec = (0..8 * len).map(|_| rng.next_u64()).collect(); + let packed_v: Vec = ws + .chunks_exact(8) + .map(|c| verified::Weights8::new(&std::array::from_fn(|i| to_v(c[i])))) + .collect(); + let packed_p: Vec = ws + .chunks_exact(8) + .map(|c| production::Weights8::new(&std::array::from_fn(|i| c[i]))) + .collect(); + for (v, p) in packed_v.iter().zip(&packed_p) { + for i in 0..8 { + assert!(same(v.get(i), p.get(i))); + } + } + let kv: Vec = ks.iter().map(|&k| vk::F64(k)).collect(); + let kp: Vec = ks.iter().map(|&k| production::F64(k)).collect(); + let (dv, dp) = (verified::dot_base(&packed_v, &kv), production::dot_base(&packed_p, &kp)); + assert!(same_unreduced(&dv, &dp), "dot_base, len {len}"); + } +} + +#[test] +fn inverses_match() { + let mut rng = Rng::new(0x1A7); + for a in elements(&mut rng, 1_000) { + assert!(same(to_v(a).inv(), a.inv()), "inv {a:?}"); + } +} + +#[test] +fn constants_match() { + assert!(same(verified::F192::ZERO, production::F192::ZERO)); + assert!(same(verified::F192::ONE, production::F192::ONE)); + assert!(same(verified::F192::Y, production::F192::Y)); + assert!(same_unreduced( + &verified::F192Unreduced::ZERO, + &production::F192Unreduced::ZERO + )); +} + +#[cfg(all(target_arch = "x86_64", target_feature = "pclmulqdq"))] +#[test] +fn x86_register_products_and_memory_match() { + use core::mem::MaybeUninit; + use production::gf2_64x3::x86_64 as p; + use verified::x86_64 as v; + let mut rng = Rng::new(0x192_01); + let xs = elements(&mut rng, 4096); + let mut va = v::F192x1Unreduced::zero(); + let mut pa = p::F192x1Unreduced::zero(); + for pair in xs.windows(2) { + let (a, b) = (pair[0], pair[1]); + let (vx, vy) = (v::F192x1::load(&to_v(a)), v::F192x1::new(to_v(b))); + let (px, py) = (p::F192x1::load(&a), p::F192x1::new(b)); + assert!(same(verified::F192::from(vx), a)); + assert!(same(verified::F192::from(vx + vy), production::F192::from(px + py))); + assert!(same(verified::F192::from(vx * vy), production::F192::from(px * py))); + let (vu, pu) = (vx.mul_unreduced(vy), px.mul_unreduced(py)); + assert!(same_unreduced(&vu.into(), &pu.into())); + va ^= vu; + pa ^= pu; + va = va ^ vx.mul_base_unreduced(vk::F64(b.c2)); + pa = pa ^ px.mul_base_unreduced(production::F64(b.c2)); + assert!(same_unreduced(&va.into(), &pa.into())); + let (mut vo, mut po) = (MaybeUninit::uninit(), MaybeUninit::uninit()); + va.reduce().store(&mut vo); + pa.reduce().store(&mut po); + assert!(same(unsafe { vo.assume_init() }, unsafe { po.assume_init() })); + } +} + +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2"))] +#[test] +fn x86_lane_wrappers_transpose_and_accumulation_match() { + use core::mem::MaybeUninit; + let mut rng = Rng::new(0x192_04); + let xs = elements(&mut rng, 4096); + let (mut vu, mut pu) = (verified::F192x4Unreduced::zero(), production::F192x4Unreduced::zero()); + let (mut vs, mut ps) = (verified::MixedSums8::default(), production::MixedSums8::default()); + for c in xs.chunks_exact(16) { + let rows: [[production::F192; 4]; 4] = std::array::from_fn(|i| std::array::from_fn(|j| c[4 * i + j])); + let vr = rows.map(|row| verified::F192x4::load(&row.map(to_v))); + let pr = rows.map(|row| production::F192x4::load(&row)); + let (vt, pt) = (verified::F192x4::transpose(vr), production::F192x4::transpose(pr)); + for i in 0..4 { + let (v, p) = (vt[i].to_array(), pt[i].to_array()); + for j in 0..4 { + assert!(same(v[j], p[j])); + assert_eq!(p[j], rows[j][i]); + } + } + let (v, p) = ( + verified::F192x4::new(rows[0].map(to_v)), + production::F192x4::new(rows[0]), + ); + let (vb, pb) = (verified::F192x4::splat(to_v(c[7])), production::F192x4::splat(c[7])); + for (v, p) in [(v + vb, p + pb), (v * vb, p * pb)] { + let mut vo = [MaybeUninit::uninit(); 4]; + let mut po = [MaybeUninit::uninit(); 4]; + v.store(&mut vo); + p.store(&mut po); + for i in 0..4 { + assert!(same(unsafe { vo[i].assume_init() }, unsafe { po[i].assume_init() })); + } + } + vu ^= v.mul_unreduced(vb); + pu ^= p.mul_unreduced(pb); + vu = vu ^ vr[1].mul_unreduced(vr[2]); + pu = pu ^ pr[1].mul_unreduced(pr[2]); + assert!(same_unreduced(&vu.sum(), &pu.sum())); + for (v, p) in vu.reduce().to_array().into_iter().zip(pu.reduce().to_array()) { + assert!(same(v, p)); + } + let k: [u64; 8] = std::array::from_fn(|i| c[i].c1); + vs.add(to_v(c[0]), k.map(vk::F64)); + ps.add(c[0], k.map(production::F64)); + for (v, p) in vs.reduce().into_iter().zip(ps.reduce()) { + assert!(same(v, p)); + } + } +} + +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx2"))] +#[test] +fn x86_arbitrary_wide_reductions_match() { + use core::arch::x86_64::*; + use core::mem::{transmute, MaybeUninit}; + use production::gf2_64x3::x86_64 as p; + use verified::x86_64 as v; + let mut rng = Rng::new(0x192_ffff); + for n in 0..512 { + let words: [[[u64; 2]; 3]; 4] = std::array::from_fn(|i| { + std::array::from_fn(|j| { + std::array::from_fn(|k| { + if n < 6 { + CORNER_WORDS[(n + i + j + k) % 6] + } else { + rng.next_u64() + } + }) + }) + }); + #[cfg(target_feature = "avx512f")] + let d: v::Wide4 = std::array::from_fn(|c| unsafe { + transmute::<[u64; 8], __m512i>(std::array::from_fn(|j| words[j / 2][c][j % 2])) + }); + #[cfg(not(target_feature = "avx512f"))] + let d: v::Wide4 = std::array::from_fn(|h| { + std::array::from_fn(|c| unsafe { + transmute::<[u64; 4], __m256i>(std::array::from_fn(|j| words[2 * h + j / 2][c][j % 2])) + }) + }); + unsafe { + let mut vo = [MaybeUninit::uninit(); 4]; + let mut po = [MaybeUninit::uninit(); 4]; + v::store_lanes4(v::reduce_lanes4(d), &mut vo); + p::store_lanes4(p::reduce_lanes4(d), &mut po); + for i in 0..4 { + let expected = verified::F192Unreduced { coeffs: words[i] }.reduce(); + assert_eq!(vo[i].assume_init(), expected); + assert!(same(expected, po[i].assume_init())); + } + assert!(same_unreduced(&v::sum_lanes4(d), &p::sum_lanes4(d))); + } + } +} + +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[test] +fn x86_planar_products_and_sums_match() { + use core::arch::x86_64::__m512i; + use core::mem::transmute; + use production::gf2_64x3::x86_64 as p; + use verified::x86_64 as v; + let mut rng = Rng::new(0x192_08); + let xs = elements(&mut rng, 4096); + // SAFETY: this test is compiled only with both required target features. + let (mut vs, mut ps) = unsafe { (v::F192x8Sum::zero(), p::F192x8Sum::zero()) }; + let mut expected = production::F192::ZERO; + for c in xs.chunks_exact(16) { + let pack = |slice: &[production::F192]| -> [__m512i; 3] { + std::array::from_fn(|k| unsafe { + transmute::<[u64; 8], __m512i>(std::array::from_fn(|i| [slice[i].c0, slice[i].c1, slice[i].c2][k])) + }) + }; + let (a, b) = (pack(&c[..8]), pack(&c[8..])); + let results = unsafe { + [ + (v::F192x8(a).add(v::F192x8(b)), p::F192x8(a).add(p::F192x8(b)), false), + (v::F192x8(a).mul(v::F192x8(b)), p::F192x8(a).mul(p::F192x8(b)), true), + ] + }; + for (vr, pr, mul) in results { + let vw: [[u64; 8]; 3] = vr.0.map(|v| unsafe { transmute(v) }); + let pw: [[u64; 8]; 3] = pr.0.map(|v| unsafe { transmute(v) }); + assert_eq!(vw, pw); + for i in 0..8 { + let e = if mul { + production::gf2_64x3::software::mul(c[i], c[8 + i]) + } else { + c[i] + c[8 + i] + }; + assert_eq!([vw[0][i], vw[1][i], vw[2][i]], [e.c0, e.c1, e.c2]); + } + } + unsafe { + vs.mul_add(v::F192x8(a), v::F192x8(b)); + ps.mul_add(p::F192x8(a), p::F192x8(b)); + } + for i in 0..8 { + expected += production::gf2_64x3::software::mul(c[i], c[8 + i]); + } + let (vt, pt) = unsafe { (vs.total(), ps.total()) }; + assert!(same_unreduced(&vt, &pt)); + assert!(same(vt.reduce(), expected)); + } +} + +/// The aarch64 kernels, each against production's, and the register-resident `F192x1` / `F192x1Unreduced` +/// (whose fields are private on both sides) through their conversions to `F192` and `F192Unreduced`. +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +#[test] +fn aarch64_kernels_match() { + use core::mem::MaybeUninit; + use production::gf2_64x3::aarch64 as pa; + use verified::aarch64 as va; + let mut rng = Rng::new(0xA192); + let xs = elements(&mut rng, 10_000); + let ys = elements(&mut rng, 10_000); + let mut acc_v = va::F192x1Unreduced::zero(); + let mut acc_p = pa::F192x1Unreduced::zero(); + assert!(same_unreduced( + &verified::F192Unreduced::from(acc_v), + &production::F192Unreduced::from(acc_p) + )); + for (&a, &b) in xs.iter().zip(&ys) { + let (vx, vy) = (to_v(a), to_v(b)); + let k = a.c2 ^ b.c0; + assert!(same(va::mul(vx, vy), pa::mul(a, b)), "mul {a:?} {b:?}"); + assert!( + same_unreduced(&va::mul_unreduced(vx, vy), &pa::mul_unreduced(a, b)), + "mul_unreduced {a:?} {b:?}" + ); + assert!(same(va::mul_base(vx, vk::F64(k)), pa::mul_base(a, production::F64(k)))); + assert!(same_unreduced( + &va::mul_base_unreduced(vx, vk::F64(k)), + &pa::mul_base_unreduced(a, production::F64(k)) + )); + assert!(same(va::square(vx), pa::square(a)), "square {a:?}"); + // An unreduced sum, as an accumulator holds (production's fields and `from_wide` are private). + let u = va::mul_unreduced(vx, vy) ^ verified::F192Unreduced::from(vy); + let up = pa::mul_unreduced(a, b) ^ production::F192Unreduced::from(b); + assert!(same_unreduced(&u, &up)); + assert!(same(va::reduce(u), pa::reduce(up)), "reduce {u:?}"); + // Registers. + let (x1, y1) = (va::F192x1::load(&vx), va::F192x1::new(vy)); + let (px1, py1) = (pa::F192x1::load(&a), pa::F192x1::new(b)); + assert!(same(verified::F192::from(x1), production::F192::from(px1))); + assert!(same(verified::F192::from(x1 + y1), production::F192::from(px1 + py1))); + assert!(same(verified::F192::from(x1 * y1), production::F192::from(px1 * py1))); + let (mut out_v, mut out_p) = (MaybeUninit::uninit(), MaybeUninit::uninit()); + (x1 * y1).store(&mut out_v); + (px1 * py1).store(&mut out_p); + assert!( + same(unsafe { out_v.assume_init() }, unsafe { out_p.assume_init() }), + "store" + ); + let (pv, pp) = (x1.mul_unreduced(y1), px1.mul_unreduced(py1)); + let (bv, bp) = ( + y1.mul_base_unreduced(vk::F64(k)), + py1.mul_base_unreduced(production::F64(k)), + ); + assert!(same_unreduced( + &verified::F192Unreduced::from(pv ^ bv), + &production::F192Unreduced::from(pp ^ bp) + )); + assert!(same( + verified::F192::from((pv ^ bv).reduce()), + production::F192::from((pp ^ bp).reduce()) + )); + acc_v ^= pv; + acc_v ^= bv; + acc_p ^= pp; + acc_p ^= bp; + assert!(same_unreduced( + &verified::F192Unreduced::from(acc_v), + &production::F192Unreduced::from(acc_p) + )); + assert!(same( + verified::F192::from(acc_v.reduce()), + production::F192::from(acc_p.reduce()) + )); + } +} diff --git a/verification/verus/tests/equivalence/gf2_8.rs b/verification/verus/tests/equivalence/gf2_8.rs new file mode 100644 index 0000000000..5c40bfcd35 --- /dev/null +++ b/verification/verus/tests/equivalence/gf2_8.rs @@ -0,0 +1,135 @@ +use leanvm_verus::gf2_8 as verified; +use primitives::field::gf2_8 as production; + +/// Every pair of bytes. Production's `clmul8` / `clmul8_software` are private: the verified `clmul8_software` is +/// checked through production's public `gf8_reduce` against production's `F8` product. +#[test] +fn products_match_exhaustively() { + for a in 0..=u8::MAX { + for b in 0..=u8::MAX { + let want = (production::F8(a) * production::F8(b)).0; + assert_eq!((verified::F8(a) * verified::F8(b)).0, want, "mul {a:#x} {b:#x}"); + assert_eq!( + production::gf8_reduce(verified::clmul8_software(a, b)), + want, + "clmul8 {a:#x} {b:#x}" + ); + let (mut v, mut p) = (verified::F8(a), production::F8(a)); + v *= verified::F8(b); + p *= production::F8(b); + assert_eq!(v.0, p.0, "mul_assign {a:#x} {b:#x}"); + assert_eq!( + (verified::F8(a) + verified::F8(b)).0, + (production::F8(a) + production::F8(b)).0 + ); + let (mut v, mut p) = (verified::F8(a), production::F8(a)); + v += verified::F8(b); + p += production::F8(b); + assert_eq!(v.0, p.0, "add_assign {a:#x} {b:#x}"); + } + } +} + +/// Every `u16`, including the degree-15 inputs outside the documented `deg <= 14`. +#[test] +fn reduction_matches_on_every_u16() { + for p in 0..=u16::MAX { + assert_eq!(verified::gf8_reduce(p), production::gf8_reduce(p), "reduce {p:#x}"); + } +} + +#[test] +fn inverses_match_exhaustively() { + for a in 0..=u8::MAX { + assert_eq!(verified::F8(a).inv().0, production::F8(a).inv().0, "inv {a:#x}"); + } +} + +#[test] +fn constants_match() { + assert_eq!(verified::F8::ZERO.0, production::F8::ZERO.0); + assert_eq!(verified::F8::ONE.0, production::F8::ONE.0); +} + +/// `clmul8_neon` (production's is private) against the verified portable `clmul8_software`, which the test above +/// ties to production, and through production's `gf8_reduce` against production's product, which dispatches to +/// production's `clmul8_neon` in this build. +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +#[test] +fn clmul8_neon_matches_exhaustively() { + for a in 0..=u8::MAX { + for b in 0..=u8::MAX { + let got = unsafe { verified::clmul8_neon(a, b) }; + assert_eq!(got, verified::clmul8_software(a, b), "clmul8_neon {a:#x} {b:#x}"); + assert_eq!(verified::clmul8(a, b), got); + assert_eq!(production::gf8_reduce(got), (production::F8(a) * production::F8(b)).0); + } + } +} + +/// The NEON reduction on every 16-bit polynomial lane, and the NEON product on every pair of bytes. +#[cfg(target_arch = "aarch64")] +#[test] +fn neon_helpers_match_exhaustively() { + use core::arch::aarch64::uint8x16_t; + use core::mem::transmute; + let b16 = |v: uint8x16_t| unsafe { transmute::(v) }; + for first in (0..=u16::MAX).step_by(16) { + let p: [u16; 16] = std::array::from_fn(|i| first + i as u16); + let (lo, hi): ([u16; 8], [u16; 8]) = (p[..8].try_into().unwrap(), p[8..].try_into().unwrap()); + let (c0, c1) = unsafe { + ( + transmute::<[u16; 8], uint8x16_t>(lo), + transmute::<[u16; 8], uint8x16_t>(hi), + ) + }; + let (v, q) = unsafe { + ( + b16(verified::neon::gf8_reduce_vec16(c0, c1)), + b16(production::neon::gf8_reduce_vec16(c0, c1)), + ) + }; + assert_eq!(v, q, "gf8_reduce_vec16 first={first:#06x}"); + } + for b in 0..=u8::MAX { + for a0 in (0..=u8::MAX).step_by(16) { + let a: [u8; 16] = std::array::from_fn(|i| a0.wrapping_add(i as u8)); + let bs: [u8; 16] = std::array::from_fn(|i| b.wrapping_add((i as u8).wrapping_mul(17))); + let (va, vb) = unsafe { + ( + transmute::<[u8; 16], uint8x16_t>(a), + transmute::<[u8; 16], uint8x16_t>(bs), + ) + }; + let (v, q) = unsafe { + ( + b16(verified::neon::gf8_mul_vec16(va, vb)), + b16(production::neon::gf8_mul_vec16(va, vb)), + ) + }; + assert_eq!(v, q, "gf8_mul_vec16 a={a:x?} b={bs:x?}"); + } + } +} + +/// The AVX2 product on every pair of bytes (built with AVX2, like production's `avx2` module). +#[cfg(all(target_arch = "x86_64", target_feature = "avx2"))] +#[test] +fn avx2_gf8_mul_vec32_matches_exhaustively() { + use core::arch::x86_64::__m256i; + use core::mem::transmute; + for b in 0..=u8::MAX { + for a0 in (0..256usize).step_by(32) { + let a: [u8; 32] = std::array::from_fn(|i| (a0 + i) as u8); + let bs: [u8; 32] = std::array::from_fn(|i| b.wrapping_add((i as u8).wrapping_mul(9))); + let (va, vb) = unsafe { (transmute::<[u8; 32], __m256i>(a), transmute::<[u8; 32], __m256i>(bs)) }; + let (v, q) = unsafe { + ( + transmute::<__m256i, [u8; 32]>(verified::avx2::gf8_mul_vec32(va, vb)), + transmute::<__m256i, [u8; 32]>(production::avx2::gf8_mul_vec32(va, vb)), + ) + }; + assert_eq!(v, q, "gf8_mul_vec32 a={a:x?} b={bs:x?}"); + } + } +} diff --git a/verification/verus/tests/equivalence/intrinsics_aarch64.rs b/verification/verus/tests/equivalence/intrinsics_aarch64.rs new file mode 100644 index 0000000000..6a30735329 --- /dev/null +++ b/verification/verus/tests/equivalence/intrinsics_aarch64.rs @@ -0,0 +1,115 @@ +//! The AArch64 intrinsic specifications against the hardware (natively on an ARM64 runner, or under +//! `qemu-aarch64-static`). +//! +//! Each test runs the real intrinsic on edge and random operands, reads its result through the same `transmute` the +//! specification's view is defined by, and compares every lane with the specification. A test whose ISA extension +//! this CPU lacks says so and passes. +use core::arch::aarch64::*; +use core::mem::transmute; +use leanvm_verus::gf2_64::software::clmul; +use primitives::test_util::Rng; +use std::arch::is_aarch64_feature_detected; + +const N: usize = 20_000; + +const EDGES: [u64; 8] = [ + 0, + 1, + u64::MAX, + 1 << 63, + 0x1B, + 0x5555_5555_5555_5555, + 0xAAAA_AAAA_AAAA_AAAA, + 1 << 32, +]; + +fn words(rng: &mut Rng, k: usize) -> [u64; W] { + if k < EDGES.len() * EDGES.len() { + std::array::from_fn(|i| EDGES[(k / EDGES.len() * (i + 1) + k % EDGES.len() * i) % EDGES.len()]) + } else { + std::array::from_fn(|_| rng.next_u64()) + } +} + +fn has(feature: bool, name: &str) -> bool { + if !feature { + eprintln!("this CPU has no {name}: its intrinsics are not tested here"); + } + feature +} + +fn v(w: [u64; 2]) -> uint64x2_t { + unsafe { transmute(w) } +} +fn w(v: uint64x2_t) -> [u64; 2] { + unsafe { transmute(v) } +} + +#[test] +fn neon_moves_and_xor() { + let mut rng = Rng::new(0x4E_0); + for k in 0..N { + let (a, b) = (words::<2>(&mut rng, k), words::<2>(&mut rng, k + 3)); + unsafe { + assert_eq!(w(vdupq_n_u64(a[0])), [a[0], a[0]]); + assert_eq!(w(veorq_u64(v(a), v(b))), [a[0] ^ b[0], a[1] ^ b[1]]); + assert_eq!(w(vzip1q_u64(v(a), v(b))), [a[0], b[0]]); + assert_eq!(vgetq_lane_u64::<0>(v(a)), a[0]); + assert_eq!(vgetq_lane_u64::<1>(v(a)), a[1]); + // The layout axioms. + let x = u128::from(a[0]) | u128::from(a[1]) << 64; + assert_eq!(w(transmute::(x)), a); + assert_eq!(transmute::(v(a)), x); + assert_eq!( + transmute::(transmute::(v(a))), + a + ); + } + } +} + +#[test] +fn pmull() { + if !has(is_aarch64_feature_detected!("aes"), "aes (PMULL)") { + return; + } + #[target_feature(enable = "aes")] + fn run(a: [u64; 2], b: [u64; 2]) { + assert_eq!(vmull_p64(a[0], b[0]), clmul(a[0], b[0]), "vmull_p64"); + let (pa, pb) = unsafe { + ( + transmute::<[u64; 2], poly64x2_t>(a), + transmute::<[u64; 2], poly64x2_t>(b), + ) + }; + assert_eq!(vmull_high_p64(pa, pb), clmul(a[1], b[1]), "vmull_high_p64"); + } + let mut rng = Rng::new(0x9_0001); + for k in 0..N { + let (a, b) = (words::<2>(&mut rng, k), words::<2>(&mut rng, k + 5)); + unsafe { run(a, b) }; + } +} + +#[test] +fn sha3_eor3() { + if !has(is_aarch64_feature_detected!("sha3"), "sha3 (EOR3)") { + return; + } + #[target_feature(enable = "sha3")] + fn run(a: [u64; 2], b: [u64; 2], c: [u64; 2]) { + assert_eq!( + w(veor3q_u64(v(a), v(b), v(c))), + [a[0] ^ b[0] ^ c[0], a[1] ^ b[1] ^ c[1]] + ); + } + let mut rng = Rng::new(0xE0_3); + for k in 0..N { + let (a, b, c) = ( + words::<2>(&mut rng, k), + words::<2>(&mut rng, k + 5), + words::<2>(&mut rng, k + 2), + ); + unsafe { run(a, b, c) }; + } +} diff --git a/verification/verus/tests/equivalence/intrinsics_aarch64_bits.rs b/verification/verus/tests/equivalence/intrinsics_aarch64_bits.rs new file mode 100644 index 0000000000..eb6e7ca686 --- /dev/null +++ b/verification/verus/tests/equivalence/intrinsics_aarch64_bits.rs @@ -0,0 +1,139 @@ +//! The AArch64 intrinsic specifications of the bit transposes, and their byte loads and stores, against the hardware +//! (natively on an ARM64 runner, or under `qemu-aarch64-static`). +//! +//! Each test runs the real intrinsic on edge and random operands, reads its result through the same `transmute` the +//! specification's view is defined by, and compares every lane with the executable twin of the specification. +use core::arch::aarch64::*; +use core::mem::transmute; +use leanvm_verus::intrinsics::aarch64_bits as spec; +use primitives::test_util::Rng; + +const N: usize = 20_000; + +const EDGES: [u64; 8] = [ + 0, + 1, + u64::MAX, + 1 << 63, + 0x00AA_00AA_00AA_00AA, + 0x5555_5555_5555_5555, + 0xF0F0_F0F0_0000_CCCC, + 1 << 32, +]; + +fn words(rng: &mut Rng, k: usize) -> [u64; 2] { + if k < EDGES.len() * EDGES.len() { + [EDGES[k / EDGES.len()], EDGES[k % EDGES.len()]] + } else { + [rng.next_u64(), rng.next_u64()] + } +} + +fn bytes(rng: &mut Rng) -> [u8; B] { + std::array::from_fn(|_| rng.next_u8()) +} + +fn v64(w: [u64; 2]) -> uint64x2_t { + unsafe { transmute(w) } +} +fn w64(v: uint64x2_t) -> [u64; 2] { + unsafe { transmute(v) } +} +fn v8(b: [u8; 16]) -> uint8x16_t { + unsafe { transmute(b) } +} +fn b8(v: uint8x16_t) -> [u8; 16] { + unsafe { transmute(v) } +} + +#[test] +fn neon_table_lookup() { + let mut rng = Rng::new(0x7B_4); + for k in 0..N { + let table = bytes::<64>(&mut rng); + // Indices: the identity, past the table (64 and up return zero), the bit transpose's own, then random bytes, + // about three quarters of which are past the table. + let idx: [u8; 16] = match k { + 0 => std::array::from_fn(|i| i as u8), + 1 => std::array::from_fn(|i| 56 + i as u8), + 2 => [0, 8, 16, 24, 32, 40, 48, 56, 1, 9, 17, 25, 33, 41, 49, 57], + 3 => [63, 64, 65, 127, 128, 255, 0, 15, 16, 31, 32, 47, 48, 62, 200, 1], + _ if k % 2 == 0 => std::array::from_fn(|_| rng.next_u8() & 63), + _ => bytes::<16>(&mut rng), + }; + let t = uint8x16x4_t( + v8(table[0..16].try_into().unwrap()), + v8(table[16..32].try_into().unwrap()), + v8(table[32..48].try_into().unwrap()), + v8(table[48..64].try_into().unwrap()), + ); + let r = b8(unsafe { vqtbl4q_u8(t, v8(idx)) }); + for i in 0..16 { + assert_eq!( + r[i], + spec::model_tbl4_byte(&table, idx[i]), + "vqtbl4q_u8 byte {i}, index {}", + idx[i] + ); + } + } +} + +#[test] +fn neon_word_lanes() { + let mut rng = Rng::new(0x5_4); + for k in 0..N { + let (a, b) = (words(&mut rng, k), words(&mut rng, k + 3)); + unsafe { + assert_eq!(w64(vandq_u64(v64(a), v64(b))), [a[0] & b[0], a[1] & b[1]], "vandq_u64"); + macro_rules! shr { + ($($n:literal),*) => {$( + assert_eq!(w64(vshrq_n_u64::<$n>(v64(a))), [spec::model_shr_n(a[0], $n), spec::model_shr_n(a[1], $n)], "vshrq_n_u64 {}", $n); + )*}; + } + shr!(1, 7, 14, 28, 32, 63, 64); + macro_rules! shl { + ($($n:literal),*) => {$( + assert_eq!(w64(vshlq_n_u64::<$n>(v64(a))), [spec::model_shl_n(a[0], $n), spec::model_shl_n(a[1], $n)], "vshlq_n_u64 {}", $n); + )*}; + } + shl!(0, 1, 7, 14, 28, 32, 63); + // The reinterprets: bytes and words, little-endian. + let ab: [u8; 16] = transmute(a); + let r = b8(vreinterpretq_u8_u64(v64(a))); + for i in 0..16 { + assert_eq!(r[i], spec::model_u64x2_byte(&a, i), "vreinterpretq_u8_u64 byte {i}"); + } + let back = w64(vreinterpretq_u64_u8(v8(ab))); + for i in 0..16 { + assert_eq!(spec::model_u64x2_byte(&back, i), ab[i], "vreinterpretq_u64_u8 byte {i}"); + } + } + } +} + +#[test] +fn neon_byte_memory() { + let mut rng = Rng::new(0x1D_4); + for _ in 0..N { + let (block, small, fill) = (bytes::<64>(&mut rng), bytes::<16>(&mut rng), bytes::<16>(&mut rng)); + assert_eq!(b8(spec::vld1q_u8_16(&small)), small, "vld1q_u8_16"); + for offset in [0, 1, 15, 16, 32, 47, 48] { + assert_eq!( + b8(spec::vld1q_u8_at(&block, offset))[..], + block[offset..offset + 16], + "vld1q_u8_at {offset}" + ); + let mut out = block; + spec::vst1q_u8_at(&mut out, offset, v8(fill)); + for k in 0..64 { + let want = if (offset..offset + 16).contains(&k) { + fill[k - offset] + } else { + block[k] + }; + assert_eq!(out[k], want, "vst1q_u8_at {offset} byte {k}"); + } + } + } +} diff --git a/verification/verus/tests/equivalence/intrinsics_aarch64_gfneon.rs b/verification/verus/tests/equivalence/intrinsics_aarch64_gfneon.rs new file mode 100644 index 0000000000..69e4d6ea28 --- /dev/null +++ b/verification/verus/tests/equivalence/intrinsics_aarch64_gfneon.rs @@ -0,0 +1,184 @@ +//! The AArch64 intrinsic specifications of `src/intrinsics/aarch64_gfneon.rs` (and the memory helpers of +//! `gf2_64x3::aarch64`) against the hardware, natively on an ARM64 runner or under `qemu-aarch64-static`. +//! +//! Each test runs the real intrinsic on edge and random operands, reads its result through the same `transmute` +//! the specification's view is defined by, and compares every lane with the specification's executable twin. +use core::arch::aarch64::*; +use core::mem::{transmute, MaybeUninit}; +use leanvm_verus::gf2_64x3::aarch64::{load_f192, store_f192}; +use leanvm_verus::gf2_64x3::F192; +use leanvm_verus::intrinsics::aarch64_gfneon::*; +use primitives::test_util::Rng; + +const N: usize = 20_000; + +const EDGES: [u64; 8] = [ + 0, + 1, + u64::MAX, + 1 << 63, + 0x1B, + 0x5555_5555_5555_5555, + 0xAAAA_AAAA_AAAA_AAAA, + 0x8080_8080_8080_8080, +]; + +fn words(rng: &mut Rng, k: usize) -> [u64; W] { + if k < EDGES.len() * EDGES.len() { + std::array::from_fn(|i| EDGES[(k / EDGES.len() * (i + 1) + k % EDGES.len() * i) % EDGES.len()]) + } else { + std::array::from_fn(|_| rng.next_u64()) + } +} + +/// Sixteen bytes: the little-endian bytes of two words. +fn bytes16(rng: &mut Rng, k: usize) -> [u8; 16] { + unsafe { transmute::<[u64; 2], [u8; 16]>(words::<2>(rng, k)) } +} + +fn v(w: [u64; 2]) -> uint64x2_t { + unsafe { transmute(w) } +} +fn w(v: uint64x2_t) -> [u64; 2] { + unsafe { transmute(v) } +} +fn vb(b: [u8; 16]) -> uint8x16_t { + unsafe { transmute(b) } +} +fn b16(v: uint8x16_t) -> [u8; 16] { + unsafe { transmute(v) } +} +fn b8(v: uint8x8_t) -> [u8; 8] { + unsafe { transmute(v) } +} +fn p8(v: poly8x8_t) -> [u8; 8] { + unsafe { transmute(v) } +} +fn h8(v: uint16x8_t) -> [u16; 8] { + unsafe { transmute(v) } +} +fn ph8(v: poly16x8_t) -> [u16; 8] { + unsafe { transmute(v) } +} + +#[test] +fn neon_u64_lane_moves() { + let mut rng = Rng::new(0x6F_0); + for k in 0..N { + let (a, b) = (words::<2>(&mut rng, k), words::<2>(&mut rng, k + 3)); + unsafe { + assert_eq!( + transmute::(vreinterpretq_p64_u64(v(a))), + a, + "vreinterpretq_p64_u64" + ); + assert_eq!( + transmute::(vcreate_u64(a[0])), + [a[0]], + "vcreate_u64" + ); + let (x, y) = ( + transmute::<[u64; 1], uint64x1_t>([a[1]]), + transmute::<[u64; 1], uint64x1_t>([b[0]]), + ); + assert_eq!(w(vcombine_u64(x, y)), [a[1], b[0]], "vcombine_u64"); + let ext0: [u64; 2] = std::array::from_fn(|i| model_ext_u64_lane(a, b, 0, i)); + let ext1: [u64; 2] = std::array::from_fn(|i| model_ext_u64_lane(a, b, 1, i)); + assert_eq!(w(vextq_u64::<0>(v(a), v(b))), ext0, "vextq_u64::<0>"); + assert_eq!(w(vextq_u64::<1>(v(a), v(b))), ext1, "vextq_u64::<1>"); + assert_eq!(w(vdupq_laneq_u64::<0>(v(a))), [a[0], a[0]], "vdupq_laneq_u64::<0>"); + assert_eq!(w(vdupq_laneq_u64::<1>(v(a))), [a[1], a[1]], "vdupq_laneq_u64::<1>"); + // The layout axiom: `[u64; 2]` read as a register has those lanes. + assert_eq!(w(transmute::<[u64; 2], uint64x2_t>(a)), a); + } + } +} + +#[test] +fn neon_byte_lanes() { + let mut rng = Rng::new(0x6F_1); + for k in 0..N { + let (a, b) = (bytes16(&mut rng, k), bytes16(&mut rng, k + 5)); + unsafe { + let uzp1: [u8; 16] = std::array::from_fn(|i| model_uzp_u8_lane(&a, &b, 0, i)); + let uzp2: [u8; 16] = std::array::from_fn(|i| model_uzp_u8_lane(&a, &b, 1, i)); + assert_eq!(b16(vuzp1q_u8(vb(a), vb(b))), uzp1, "vuzp1q_u8"); + assert_eq!(b16(vuzp2q_u8(vb(a), vb(b))), uzp2, "vuzp2q_u8"); + assert_eq!( + b16(veorq_u8(vb(a), vb(b))), + std::array::from_fn(|i| a[i] ^ b[i]), + "veorq_u8" + ); + assert_eq!(b8(vget_low_u8(vb(a))), std::array::from_fn(|i| a[i]), "vget_low_u8"); + assert_eq!( + b8(vget_high_u8(vb(a))), + std::array::from_fn(|i| a[i + 8]), + "vget_high_u8" + ); + assert_eq!(p8(vdup_n_p8(a[3])), [a[3]; 8], "vdup_n_p8"); + // The layout axioms: a `uint8x8_t` read as a `poly8x8_t`, and a `u64` read as a `poly8x8_t`. + let lo = vget_low_u8(vb(a)); + assert_eq!(p8(transmute::(lo)), b8(lo)); + let x = u64::from_le_bytes(std::array::from_fn(|i| b[i])); + assert_eq!( + p8(transmute::(x)), + std::array::from_fn(|i| (x >> (8 * i)) as u8) + ); + } + } + // The two constants the reduction reinterprets. + for (x, c) in [(0x8d8d8d8d8d8d8d8d_u64, 0x8d_u8), (0x1b1b1b1b1b1b1b1b_u64, 0x1b_u8)] { + assert_eq!(p8(unsafe { transmute::(x) }), [c; 8]); + } +} + +#[test] +fn neon_u16_lanes() { + let mut rng = Rng::new(0x6F_2); + for k in 0..N { + let a = bytes16(&mut rng, k); + let h: [u16; 8] = unsafe { transmute(a) }; + unsafe { + let hv = transmute::<[u16; 8], uint16x8_t>(h); + let ph = transmute::<[u16; 8], poly16x8_t>(h); + assert_eq!(h8(vreinterpretq_u16_p16(ph)), ph8(ph), "vreinterpretq_u16_p16"); + let bytes: [u8; 16] = std::array::from_fn(|i| model_u16_byte(&h, i)); + assert_eq!(b16(vreinterpretq_u8_u16(hv)), bytes, "vreinterpretq_u8_u16"); + assert_eq!(h8(vshlq_n_u16::<1>(hv)), h.map(|x| x << 1), "vshlq_n_u16::<1>"); + assert_eq!(h8(vshlq_n_u16::<7>(hv)), h.map(|x| x << 7), "vshlq_n_u16::<7>"); + assert_eq!(vgetq_lane_u16::<0>(hv), h[0], "vgetq_lane_u16::<0>"); + assert_eq!(vgetq_lane_u16::<5>(hv), h[5], "vgetq_lane_u16::<5>"); + } + } +} + +/// PMULL on bytes, on every pair of bytes in every lane position. +#[test] +fn neon_vmull_p8() { + for a in 0..=u8::MAX { + for b0 in (0..=u8::MAX).step_by(8) { + let x = [a; 8]; + let y: [u8; 8] = std::array::from_fn(|i| b0.wrapping_add(i as u8 * 37)); + let got = ph8(unsafe { vmull_p8(transmute::<[u8; 8], poly8x8_t>(x), transmute::<[u8; 8], poly8x8_t>(y)) }); + let want: [u16; 8] = std::array::from_fn(|i| model_vmull_p8_lane(x[i], y[i])); + assert_eq!(got, want, "vmull_p8 {a:#x} {y:x?}"); + } + } +} + +/// The trusted memory helpers of `F192x1::load` and `F192x1::store`. +#[test] +fn f192_load_store_helpers() { + let mut rng = Rng::new(0x6F_3); + for k in 0..N { + let [c0, c1, c2] = words::<3>(&mut rng, k); + let e = F192::new(c0, c1, c2); + let (r01, r22) = load_f192(&e); + assert_eq!((w(r01), w(r22)), ([c0, c1], [c2, c2]), "load_f192"); + let (x, y) = (words::<2>(&mut rng, k + 7), words::<2>(&mut rng, k + 1)); + let mut out = MaybeUninit::::uninit(); + store_f192(&mut out, v(x), v(y)); + let got = unsafe { out.assume_init() }; + assert_eq!((got.c0, got.c1, got.c2), (x[0], x[1], y[0]), "store_f192"); + } +} diff --git a/verification/verus/tests/equivalence/intrinsics_aarch64_nttsimd.rs b/verification/verus/tests/equivalence/intrinsics_aarch64_nttsimd.rs new file mode 100644 index 0000000000..9a1641639c --- /dev/null +++ b/verification/verus/tests/equivalence/intrinsics_aarch64_nttsimd.rs @@ -0,0 +1,33 @@ +//! The memory helpers of the NEON NTT butterfly kernels (`intrinsics::aarch64_nttsimd`) against their +//! specifications: the load returns the two words at its offset, read through `transmute` as `u64x2` is, and +//! the store writes exactly those two words and leaves the rest of the row as it was. +use core::arch::aarch64::*; +use core::mem::transmute; +use leanvm_verus::gf2_64::F64; +use leanvm_verus::intrinsics::aarch64_nttsimd as helpers; +use primitives::test_util::Rng; + +const N: usize = 2_000; + +#[test] +fn neon_row_loads_and_stores() { + let mut rng = Rng::new(0x1D1); + for _ in 0..N { + let row: Vec = (0..9).map(|_| F64(rng.next_u64())).collect(); + for at in 0..=7 { + let got = unsafe { transmute::(helpers::vld1q_u64_at(&row, at)) }; + assert_eq!(got, [row[at].0, row[at + 1].0], "vld1q_u64_at at {at}"); + let words = [rng.next_u64(), rng.next_u64()]; + let mut stored = row.clone(); + unsafe { helpers::vst1q_u64_at(&mut stored, at, transmute::<[u64; 2], uint64x2_t>(words)) }; + for j in 0..row.len() { + let want = if (at..at + 2).contains(&j) { + words[j - at] + } else { + row[j].0 + }; + assert_eq!(stored[j].0, want, "vst1q_u64_at at {at} word {j}"); + } + } + } +} diff --git a/verification/verus/tests/equivalence/intrinsics_x86.rs b/verification/verus/tests/equivalence/intrinsics_x86.rs new file mode 100644 index 0000000000..7eb5baa6d4 --- /dev/null +++ b/verification/verus/tests/equivalence/intrinsics_x86.rs @@ -0,0 +1,328 @@ +//! The x86-64 intrinsic specifications against the hardware. +//! +//! Each test runs the real intrinsic on edge and random operands, reads its result through the same `transmute` the +//! specification's view is defined by, and compares every lane with the executable twin of the specification. A test +//! whose ISA extension this CPU lacks says so and passes. +use core::arch::x86_64::*; +use core::mem::transmute; +use leanvm_verus::intrinsics::x86 as spec; +use primitives::test_util::Rng; + +const N: usize = 20_000; + +/// Edge words: zero, one, all ones, the top bit, the reduction constant, alternating bits, a single middle bit. +const EDGES: [u64; 8] = [ + 0, + 1, + u64::MAX, + 1 << 63, + 0x1B, + 0x5555_5555_5555_5555, + 0xAAAA_AAAA_AAAA_AAAA, + 1 << 32, +]; + +/// `n` words: the edge words cycled, then random ones. +fn words(rng: &mut Rng, k: usize) -> [u64; W] { + if k < EDGES.len() * EDGES.len() { + std::array::from_fn(|i| EDGES[(k / EDGES.len() * (i + 1) + k % EDGES.len() * i) % EDGES.len()]) + } else { + std::array::from_fn(|_| rng.next_u64()) + } +} + +fn has(feature: bool, name: &str) -> bool { + if !feature { + eprintln!("this CPU has no {name}: its intrinsics are not tested here"); + } + feature +} + +fn m128(w: [u64; 2]) -> __m128i { + unsafe { transmute(w) } +} +fn m256(w: [u64; 4]) -> __m256i { + unsafe { transmute(w) } +} +fn m512(w: [u64; 8]) -> __m512i { + unsafe { transmute(w) } +} +fn w128(v: __m128i) -> [u64; 2] { + unsafe { transmute(v) } +} +fn w256(v: __m256i) -> [u64; 4] { + unsafe { transmute(v) } +} +fn w512(v: __m512i) -> [u64; 8] { + unsafe { transmute(v) } +} + +#[test] +fn sse2_moves_and_xor() { + let mut rng = Rng::new(0x5_5E2); + for k in 0..N { + let (a, b) = (words::<2>(&mut rng, k), words::<2>(&mut rng, k + 3)); + unsafe { + assert_eq!(w128(_mm_cvtsi64_si128(a[0] as i64)), [a[0], 0]); + assert_eq!(_mm_cvtsi128_si64(m128(a)) as u64, a[0]); + assert_eq!(w128(_mm_set_epi64x(a[1] as i64, a[0] as i64)), a); + assert_eq!(w128(_mm_xor_si128(m128(a), m128(b))), [a[0] ^ b[0], a[1] ^ b[1]]); + let lo = w128(_mm_unpacklo_epi64(m128(a), m128(b))); + let hi = w128(_mm_unpackhi_epi64(m128(a), m128(b))); + for i in 0..2 { + assert_eq!(lo[i], spec::model_unpacklo_lane(&a, &b, i)); + assert_eq!(hi[i], spec::model_unpackhi_lane(&a, &b, i)); + } + // The layout axioms of a 128-bit register. + for k in 0..16 { + assert_eq!( + transmute::<__m128i, [u8; 16]>(m128(a))[k], + (a[k / 8] >> (8 * (k % 8))) as u8 + ); + } + assert_eq!( + transmute::<__m128i, u128>(m128(a)), + u128::from(a[0]) | u128::from(a[1]) << 64 + ); + } + } +} + +macro_rules! check_clmul { + ($f:ident, $view:ident, $make:ident, $a:expr, $b:expr, $($imm:literal),*) => {$({ + let r = $view($f::<$imm>($make($a), $make($b))); + for i in 0..r.len() { + assert_eq!(r[i], spec::model_clmul_lane(&$a, &$b, $imm as u8, i), "{} imm {:#x} lane {i}", stringify!($f), $imm); + } + })*}; +} + +#[test] +fn pclmulqdq() { + if !has(is_x86_feature_detected!("pclmulqdq"), "pclmulqdq") { + return; + } + #[target_feature(enable = "pclmulqdq")] + fn run(a: [u64; 2], b: [u64; 2]) { + check_clmul!(_mm_clmulepi64_si128, w128, m128, a, b, 0x00, 0x01, 0x10, 0x11); + } + let mut rng = Rng::new(0xC1_0001); + for k in 0..N { + let (a, b) = (words::<2>(&mut rng, k), words::<2>(&mut rng, k + 5)); + unsafe { run(a, b) }; + } +} + +#[test] +fn bmi2_pdep() { + if !has(is_x86_feature_detected!("bmi2"), "bmi2") { + return; + } + #[target_feature(enable = "bmi2")] + fn run(a: u64, mask: u64) { + assert_eq!(_pdep_u64(a, mask), spec::model_pdep(a, mask), "pdep {a:#x} {mask:#x}"); + } + let mut rng = Rng::new(0xBD_E9); + for k in 0..N { + let [a, mask] = words::<2>(&mut rng, k); + unsafe { run(a, mask) }; + } +} + +#[test] +fn avx2() { + if !has(is_x86_feature_detected!("avx2"), "avx2") { + return; + } + #[target_feature(enable = "avx2")] + fn run(a: [u64; 4], b: [u64; 4], x: [u64; 2]) { + let (va, vb) = (m256(a), m256(b)); + let lanes = |r: __m256i, f: &dyn Fn(usize) -> u64, name: &str| { + let r = w256(r); + for i in 0..4 { + assert_eq!(r[i], f(i), "{name} lane {i}"); + } + }; + lanes(_mm256_xor_si256(va, vb), &|i| a[i] ^ b[i], "xor"); + lanes(_mm256_and_si256(va, vb), &|i| a[i] & b[i], "and"); + lanes(_mm256_add_epi64(va, vb), &|i| a[i].wrapping_add(b[i]), "add"); + lanes( + _mm256_unpacklo_epi64(va, vb), + &|i| spec::model_unpacklo_lane(&a, &b, i), + "unpacklo", + ); + lanes( + _mm256_unpackhi_epi64(va, vb), + &|i| spec::model_unpackhi_lane(&a, &b, i), + "unpackhi", + ); + lanes(_mm256_set1_epi64x(a[0] as i64), &|_| a[0], "set1"); + lanes( + _mm256_set_epi64x(a[3] as i64, a[2] as i64, a[1] as i64, a[0] as i64), + &|i| a[i], + "set", + ); + lanes(_mm256_broadcastsi128_si256(m128(x)), &|i| x[i % 2], "broadcastsi128"); + macro_rules! shifts { + ($($imm:literal),*) => {$( + lanes(_mm256_slli_epi64::<$imm>(va), &|i| spec::model_shl_imm(a[i], $imm), "slli"); + lanes(_mm256_srli_epi64::<$imm>(va), &|i| spec::model_shr_imm(a[i], $imm), "srli"); + )*}; + } + shifts!(0, 1, 3, 4, 8, 32, 56, 60, 61, 63, 64, 200); + macro_rules! permutes { + ($($imm:literal),*) => {$( + lanes(_mm256_permute4x64_epi64::<$imm>(va), &|i| spec::model_permute4x64_lane(&a, $imm, i), "permute4x64"); + )*}; + } + permutes!(0x00, 0x1B, 0x4E, 0x8D, 0xB1, 0xD8, 0xE4, 0xFF, 0x44, 0xEE); + let (ba, bb) = (unsafe { transmute::<[u64; 4], [u8; 32]>(a) }, unsafe { + transmute::<[u64; 4], [u8; 32]>(b) + }); + let r = unsafe { transmute::<__m256i, [u8; 32]>(_mm256_shuffle_epi8(va, vb)) }; + for k in 0..32 { + assert_eq!( + r[k], + spec::model_shuffle_epi8_lane(&ba, &bb, k), + "shuffle_epi8 byte {k}" + ); + } + // The layout axioms of a 256-bit register: as pairs, as bytes. + for k in 0..32 { + assert_eq!( + unsafe { transmute::<__m256i, [u8; 32]>(va) }[k], + (a[k / 8] >> (8 * (k % 8))) as u8 + ); + } + assert_eq!( + unsafe { transmute::<__m256i, [[u64; 2]; 2]>(va) }, + [[a[0], a[1]], [a[2], a[3]]] + ); + } + let mut rng = Rng::new(0xA_2); + for k in 0..N { + let (a, b, x) = ( + words::<4>(&mut rng, k), + words::<4>(&mut rng, k + 7), + words::<2>(&mut rng, k + 1), + ); + unsafe { run(a, b, x) }; + } +} + +#[test] +fn vpclmulqdq_avx2() { + if !has( + is_x86_feature_detected!("vpclmulqdq") && is_x86_feature_detected!("avx2"), + "vpclmulqdq with avx2", + ) { + return; + } + #[target_feature(enable = "vpclmulqdq,avx2")] + fn run(a: [u64; 4], b: [u64; 4]) { + check_clmul!(_mm256_clmulepi64_epi128, w256, m256, a, b, 0x00, 0x01, 0x10, 0x11); + } + let mut rng = Rng::new(0xC1_0256); + for k in 0..N { + let (a, b) = (words::<4>(&mut rng, k), words::<4>(&mut rng, k + 3)); + unsafe { run(a, b) }; + } +} + +#[test] +fn avx512f() { + if !has(is_x86_feature_detected!("avx512f"), "avx512f") { + return; + } + #[target_feature(enable = "avx512f")] + fn run(a: [u64; 8], b: [u64; 8], c: [u64; 8]) { + let (va, vb, vc) = (m512(a), m512(b), m512(c)); + let lanes = |r: __m512i, f: &dyn Fn(usize) -> u64, name: &str| { + let r = w512(r); + for i in 0..8 { + assert_eq!(r[i], f(i), "{name} lane {i}"); + } + }; + lanes(_mm512_xor_si512(va, vb), &|i| a[i] ^ b[i], "xor"); + lanes( + _mm512_unpacklo_epi64(va, vb), + &|i| spec::model_unpacklo_lane(&a, &b, i), + "unpacklo", + ); + lanes( + _mm512_unpackhi_epi64(va, vb), + &|i| spec::model_unpackhi_lane(&a, &b, i), + "unpackhi", + ); + lanes(_mm512_set1_epi64(a[0] as i64), &|_| a[0], "set1"); + let (e0, e1, e2, e3) = (a[0] as i64, a[1] as i64, a[2] as i64, a[3] as i64); + let (e4, e5, e6, e7) = (a[4] as i64, a[5] as i64, a[6] as i64, a[7] as i64); + lanes(_mm512_set_epi64(e7, e6, e5, e4, e3, e2, e1, e0), &|i| a[i], "set"); + lanes( + _mm512_permutex2var_epi64(va, vc, vb), + &|i| spec::model_permutex2var_lane(&a, &c, &b, i), + "permutex2var", + ); + macro_rules! shifts { + ($($imm:literal),*) => {$( + lanes(_mm512_slli_epi64::<$imm>(va), &|i| spec::model_shl_imm(a[i], $imm), "slli"); + lanes(_mm512_srli_epi64::<$imm>(va), &|i| spec::model_shr_imm(a[i], $imm), "srli"); + )*}; + } + shifts!(0, 1, 3, 4, 8, 32, 56, 60, 61, 63, 64, 200); + macro_rules! shuffles { + ($($imm:literal),*) => {$( + lanes(_mm512_shuffle_i64x2::<$imm>(va, vb), &|i| spec::model_shuffle_i64x2_lane(&a, &b, $imm, i), "shuffle_i64x2"); + )*}; + } + shuffles!(0x00, 0x44, 0xEE, 0x88, 0xDD, 0x4E, 0xE4, 0x1B, 0xFF); + macro_rules! ternary { + ($($imm:literal),*) => {$( + lanes(_mm512_ternarylogic_epi64::<$imm>(va, vb, vc), &|i| spec::model_ternlog($imm, a[i], b[i], c[i]), "ternarylogic"); + )*}; + } + ternary!(0x00, 0x96, 0x78, 0xE8, 0xCA, 0xF0, 0xCC, 0xAA, 0x3C, 0x6A, 0xFE, 0x80, 0x01, 0xFF); + // The layout axioms of a 512-bit register. + assert_eq!( + unsafe { transmute::<__m512i, [[u64; 2]; 4]>(va) }, + [[a[0], a[1]], [a[2], a[3]], [a[4], a[5]], [a[6], a[7]]] + ); + assert_eq!(w512(unsafe { transmute::<[u64; 8], __m512i>(a) }), a); + for k in 0..64 { + assert_eq!( + unsafe { transmute::<__m512i, [u8; 64]>(va) }[k], + (a[k / 8] >> (8 * (k % 8))) as u8 + ); + } + } + let mut rng = Rng::new(0xA_512); + for k in 0..N { + let (a, b) = (words::<8>(&mut rng, k), words::<8>(&mut rng, k + 7)); + // Index words: random low nibbles, the rest random too (the instruction reads bits 0 to 3 only). + let c = if k % 2 == 0 { + words::<8>(&mut rng, k + 2) + } else { + std::array::from_fn(|_| rng.next_u64() & 15) + }; + unsafe { run(a, b, c) }; + } +} + +#[test] +fn vpclmulqdq_avx512() { + if !has( + is_x86_feature_detected!("vpclmulqdq") && is_x86_feature_detected!("avx512f"), + "vpclmulqdq with avx512f", + ) { + return; + } + #[target_feature(enable = "vpclmulqdq,avx512f")] + fn run(a: [u64; 8], b: [u64; 8]) { + check_clmul!(_mm512_clmulepi64_epi128, w512, m512, a, b, 0x00, 0x01, 0x10, 0x11); + } + let mut rng = Rng::new(0xC1_0512); + for k in 0..N { + let (a, b) = (words::<8>(&mut rng, k), words::<8>(&mut rng, k + 3)); + unsafe { run(a, b) }; + } +} diff --git a/verification/verus/tests/equivalence/intrinsics_x86_bits.rs b/verification/verus/tests/equivalence/intrinsics_x86_bits.rs new file mode 100644 index 0000000000..e3d7e8a16a --- /dev/null +++ b/verification/verus/tests/equivalence/intrinsics_x86_bits.rs @@ -0,0 +1,250 @@ +//! The x86-64 intrinsic specifications of the bit transposes, and their byte loads and stores, against the hardware. +//! +//! Each test runs the real intrinsic on edge and random operands, reads its result through the same `transmute` the +//! specification's view is defined by, and compares every lane with the executable twin of the specification. A test +//! whose ISA extension this CPU lacks says so and passes. +use core::arch::x86_64::*; +use core::mem::transmute; +use leanvm_verus::intrinsics::x86_bits as spec; +use primitives::test_util::Rng; + +const N: usize = 20_000; + +/// Edge words: zero, all ones, the GFNI unit matrix, the identity matrix of the other bit order, one bit per byte, +/// alternating bits, the AES S-box's affine matrix, a single bit. +const EDGES: [u64; 8] = [ + 0, + u64::MAX, + 0x8040_2010_0804_0201, + 0x0102_0408_1020_4080, + 0x0101_0101_0101_0101, + 0x5555_5555_5555_5555, + 0xF1E3_C78F_1F3E_7CF8, + 1 << 37, +]; + +/// The edge words cycled, then random ones. +fn words(rng: &mut Rng, k: usize) -> [u64; W] { + if k < EDGES.len() * EDGES.len() { + std::array::from_fn(|i| EDGES[(k / EDGES.len() * (i + 1) + k % EDGES.len() * i) % EDGES.len()]) + } else { + std::array::from_fn(|_| rng.next_u64()) + } +} + +fn bytes(rng: &mut Rng) -> [u8; B] { + std::array::from_fn(|_| rng.next_u8()) +} + +fn has(feature: bool, name: &str) -> bool { + if !feature { + eprintln!("this CPU has no {name}: its intrinsics are not tested here"); + } + feature +} + +fn m256(w: [u64; 4]) -> __m256i { + unsafe { transmute(w) } +} +fn m512(w: [u64; 8]) -> __m512i { + unsafe { transmute(w) } +} +fn b256(v: __m256i) -> [u8; 32] { + unsafe { transmute(v) } +} +fn b512(v: __m512i) -> [u8; 64] { + unsafe { transmute(v) } +} + +/// GF2P8AFFINEQB from Intel's pseudocode, independently of the specification's closed form: bit `i` of the result is +/// the parity of `A.byte[7 - i] & x`, XOR bit `i` of the immediate. +fn affine_reference(a: u64, x: u8, imm8: u8) -> u8 { + (0..8).fold(0u8, |r, i| { + let row = (a >> (8 * (7 - i))) as u8; + r | ((((row & x).count_ones() as u8) & 1) ^ ((imm8 >> i) & 1)) << i + }) +} + +#[test] +fn affine_byte_matches_intel_pseudocode() { + let mut rng = Rng::new(0xAFF_0); + for k in 0..N { + let a = words::<1>(&mut rng, k)[0]; + for x in [0u8, 1, 0x80, 0xFF, 0x55, rng.next_u8()] { + for imm8 in [0u8, 1, 0x63, 0x80, 0xFF, rng.next_u8()] { + assert_eq!( + spec::model_affine_byte(a, x, imm8), + affine_reference(a, x, imm8), + "a={a:#x} x={x:#x} imm8={imm8:#x}" + ); + } + } + } +} + +#[test] +fn avx2_byte_unpacks_and_memory() { + if !has(is_x86_feature_detected!("avx2"), "avx2") { + return; + } + #[target_feature(enable = "avx2")] + fn run(a: [u64; 4], b: [u64; 4], block: [u8; 64], small: [u8; 16], fill: [u8; 64]) { + let (ba, bb): ([u8; 32], [u8; 32]) = unsafe { (transmute(a), transmute(b)) }; + let (lo, hi) = ( + b256(_mm256_unpacklo_epi8(m256(a), m256(b))), + b256(_mm256_unpackhi_epi8(m256(a), m256(b))), + ); + for k in 0..32 { + assert_eq!( + lo[k], + spec::model_unpacklo_epi8_lane(&ba, &bb, k), + "unpacklo_epi8 byte {k}" + ); + assert_eq!( + hi[k], + spec::model_unpackhi_epi8_lane(&ba, &bb, k), + "unpackhi_epi8 byte {k}" + ); + } + unsafe { + assert_eq!( + transmute::<__m128i, [u8; 16]>(spec::load128_bytes(&small)), + small, + "load128_bytes" + ); + for offset in [0, 1, 7, 16, 31, 32] { + let v = spec::load256_bytes_at(&block, offset); + assert_eq!(b256(v)[..], block[offset..offset + 32], "load256_bytes_at {offset}"); + let mut out = fill; + spec::store256_bytes_at(&mut out, offset, m256(a)); + for k in 0..64 { + let want = if (offset..offset + 32).contains(&k) { + ba[k - offset] + } else { + fill[k] + }; + assert_eq!(out[k], want, "store256_bytes_at {offset} byte {k}"); + } + } + } + } + let mut rng = Rng::new(0xB_256); + for k in 0..N { + let (a, b) = (words::<4>(&mut rng, k), words::<4>(&mut rng, k + 3)); + let (block, small, fill) = (bytes::<64>(&mut rng), bytes::<16>(&mut rng), bytes::<64>(&mut rng)); + unsafe { run(a, b, block, small, fill) }; + } +} + +#[test] +fn avx512vbmi_byte_permute_and_memory() { + if !has( + is_x86_feature_detected!("avx512vbmi") && is_x86_feature_detected!("avx512f"), + "avx512vbmi", + ) { + return; + } + #[target_feature(enable = "avx512f,avx512vbmi")] + fn run(idx: [u8; 64], a: [u8; 64], fill: [u8; 64]) { + let (vi, va): (__m512i, __m512i) = unsafe { (transmute(idx), transmute(a)) }; + let r = b512(_mm512_permutexvar_epi8(vi, va)); + for k in 0..64 { + assert_eq!( + r[k], + spec::model_permutexvar_epi8_lane(&idx, &a, k), + "permutexvar_epi8 byte {k}" + ); + } + unsafe { + assert_eq!(b512(spec::load512_bytes(&a)), a, "load512_bytes"); + let mut out = fill; + spec::store512_bytes(&mut out, va); + assert_eq!(out, a, "store512_bytes"); + } + } + let mut rng = Rng::new(0xB_512); + for k in 0..N { + // Indices: the identity, the reversal, every byte the same, then random (the top two bits random too, which + // the instruction ignores). + let idx: [u8; 64] = match k { + 0 => std::array::from_fn(|i| i as u8), + 1 => std::array::from_fn(|i| 63 - i as u8), + 2 => [0xC5; 64], + _ => bytes::<64>(&mut rng), + }; + let (a, fill) = (bytes::<64>(&mut rng), bytes::<64>(&mut rng)); + unsafe { run(idx, a, fill) }; + } +} + +macro_rules! check_affine { + ($f:ident, $view:ident, $reg:ident, $x:expr, $a:expr, $($imm:literal),*) => {$( + let r = $view($f::<$imm>($reg($x), $reg($a))); + let xb: Vec = $x.iter().flat_map(|w: &u64| w.to_le_bytes()).collect(); + for k in 0..r.len() { + assert_eq!(r[k], spec::model_gf2p8affine_lane(&xb, &$a, $imm, k), "{} imm {} byte {k}", stringify!($f), $imm); + } + )*}; +} + +#[test] +fn gfni_affine_avx2() { + if !has( + is_x86_feature_detected!("gfni") && is_x86_feature_detected!("avx2"), + "gfni with avx2", + ) { + return; + } + #[target_feature(enable = "gfni,avx2")] + fn run(x: [u64; 4], a: [u64; 4]) { + check_affine!( + _mm256_gf2p8affine_epi64_epi8, + b256, + m256, + x, + a, + 0, + 1, + 0x63, + 0x80, + 0xAA, + 0xFF + ); + } + let mut rng = Rng::new(0x6F_256); + for k in 0..N { + let (x, a) = (words::<4>(&mut rng, k + 5), words::<4>(&mut rng, k)); + unsafe { run(x, a) }; + } +} + +#[test] +fn gfni_affine_avx512() { + if !has( + is_x86_feature_detected!("gfni") && is_x86_feature_detected!("avx512f"), + "gfni with avx512f", + ) { + return; + } + #[target_feature(enable = "gfni,avx512f")] + fn run(x: [u64; 8], a: [u64; 8]) { + check_affine!( + _mm512_gf2p8affine_epi64_epi8, + b512, + m512, + x, + a, + 0, + 1, + 0x63, + 0x80, + 0xAA, + 0xFF + ); + } + let mut rng = Rng::new(0x6F_512); + for k in 0..N { + let (x, a) = (words::<8>(&mut rng, k + 5), words::<8>(&mut rng, k)); + unsafe { run(x, a) }; + } +} diff --git a/verification/verus/tests/equivalence/intrinsics_x86_gfneon.rs b/verification/verus/tests/equivalence/intrinsics_x86_gfneon.rs new file mode 100644 index 0000000000..7b43bc1114 --- /dev/null +++ b/verification/verus/tests/equivalence/intrinsics_x86_gfneon.rs @@ -0,0 +1,63 @@ +//! The AVX2 byte intrinsic specifications of `src/intrinsics/x86_gfneon.rs` against the hardware. +//! +//! Each test runs the real intrinsic on edge and random operands, reads its result through `transmute`, and +//! compares every byte with the specification (its executable twin where there is one). A machine without AVX2 +//! says so and passes. +use core::arch::x86_64::*; +use core::mem::transmute; +use leanvm_verus::intrinsics::x86_gfneon::model_cmpgt_epi8_lane; +use primitives::test_util::Rng; +use std::arch::is_x86_feature_detected; + +const N: usize = 20_000; + +/// 32 bytes: every byte value appears in the first eight vectors, in every lane position over the edge rounds. +fn bytes(rng: &mut Rng, k: usize) -> [u8; 32] { + if k < 256 { + std::array::from_fn(|i| (k + 37 * i) as u8) + } else { + std::array::from_fn(|_| rng.next_u64() as u8) + } +} + +#[target_feature(enable = "avx2")] +fn run(a: [u8; 32], b: [u8; 32]) { + let r = |x: __m256i| unsafe { transmute::<__m256i, [u8; 32]>(x) }; + let (va, vb) = unsafe { (transmute::<[u8; 32], __m256i>(a), transmute::<[u8; 32], __m256i>(b)) }; + assert_eq!( + unsafe { transmute::<__m256i, [u64; 4]>(_mm256_setzero_si256()) }, + [0; 4], + "_mm256_setzero_si256" + ); + assert_eq!(r(_mm256_set1_epi8(a[0] as i8)), [a[0]; 32], "_mm256_set1_epi8"); + assert_eq!( + r(_mm256_add_epi8(va, vb)), + std::array::from_fn(|i| a[i].wrapping_add(b[i])), + "_mm256_add_epi8" + ); + assert_eq!( + r(_mm256_cmpgt_epi8(va, vb)), + std::array::from_fn(|i| model_cmpgt_epi8_lane(a[i], b[i])), + "_mm256_cmpgt_epi8" + ); +} + +#[test] +fn avx2_byte_ops() { + if !is_x86_feature_detected!("avx2") { + eprintln!("this CPU has no avx2: its intrinsics are not tested here"); + return; + } + let mut rng = Rng::new(0x8B_0); + for k in 0..N { + let (a, b) = ( + bytes(&mut rng, k), + bytes(&mut rng, (k * 7 + 3) % 256 + if k < 256 { 0 } else { 256 }), + ); + unsafe { run(a, b) }; + } + // The compare against zero, the use the kernel makes of it, for every byte. + for x in 0..=u8::MAX { + unsafe { run([0; 32], [x; 32]) }; + } +} diff --git a/verification/verus/tests/equivalence/intrinsics_x86_gfx86.rs b/verification/verus/tests/equivalence/intrinsics_x86_gfx86.rs new file mode 100644 index 0000000000..988eda6974 --- /dev/null +++ b/verification/verus/tests/equivalence/intrinsics_x86_gfx86.rs @@ -0,0 +1,161 @@ +//! Hardware checks of every additional GF192 intrinsic assumption. Undefined upper +//! words of the widening cast are deliberately not inspected. +use core::arch::x86_64::*; +use core::mem::{transmute, zeroed}; +use leanvm_verus::intrinsics::{x86 as base, x86_gfx86 as model}; +use primitives::test_util::Rng; + +fn words(rng: &mut Rng, n: usize) -> [u64; N] { + const E: [u64; 8] = [ + 0, + 1, + u64::MAX, + 1 << 63, + 0x1b, + 0x5555_5555_5555_5555, + 0xAAAA_AAAA_AAAA_AAAA, + 1 << 32, + ]; + std::array::from_fn(|i| if n < 64 { E[(n + i * 3) % 8] } else { rng.next_u64() }) +} +fn w128(v: __m128i) -> [u64; 2] { + unsafe { transmute(v) } +} +fn w256(v: __m256i) -> [u64; 4] { + unsafe { transmute(v) } +} +fn w512(v: __m512i) -> [u64; 8] { + unsafe { transmute(v) } +} + +macro_rules! shuffle { + ($f:ident,$view:ident,$a:expr,$($imm:literal),*) => {$( + let actual = $view($f::<$imm>(transmute($a))); + for i in 0..actual.len() { assert_eq!(actual[i],model::model_shuffle_epi32_lane(&$a,$imm,i)); } + )*}; +} +macro_rules! shifts { + ($a:expr,$($imm:literal),*) => {$( + let left = w128(_mm_slli_epi64::<$imm>(transmute($a))); + let right = w128(_mm_srli_epi64::<$imm>(transmute($a))); + for i in 0..2 { + assert_eq!(left[i],base::model_shl_imm($a[i],$imm)); + assert_eq!(right[i],base::model_shr_imm($a[i],$imm)); + } + )*}; +} +#[test] +fn sse2_additional_assumptions() { + let mut rng = Rng::new(0x192_128); + for n in 0..2048 { + let a = words::<2>(&mut rng, n); + unsafe { + assert_eq!(w128(_mm_setzero_si128()), [0; 2]); + assert_eq!(w128(_mm_set1_epi64x(a[0] as i64)), [a[0]; 2]); + shifts!(a, 0, 1, 3, 4, 31, 32, 60, 61, 63, 64, 65, 255); + shuffle!(_mm_shuffle_epi32, w128, a, 0, 0x1b, 0x4e, 0x55, 0x88, 0xaa, 0xe4, 0xff); + } + } +} +macro_rules! pair_moves { + ($a:expr,$b:expr,$($imm:literal),*) => {$( + let blend = w256(_mm256_blend_epi32::<$imm>(transmute($a),transmute($b))); + let perm = w256(_mm256_permute2x128_si256::<$imm>(transmute($a),transmute($b))); + for i in 0..4 { + assert_eq!(blend[i],model::model_blend_epi32_lane(&$a,&$b,$imm,i)); + assert_eq!(perm[i],model::model_permute2x128_lane(&$a,&$b,$imm,i)); + } + )*}; +} +#[test] +fn avx2_additional_assumptions() { + if !is_x86_feature_detected!("avx2") { + eprintln!("AVX2 not available; hardware test skipped"); + return; + } + #[target_feature(enable = "avx2")] + unsafe fn run(a: [u64; 4], b: [u64; 4]) { + shuffle!( + _mm256_shuffle_epi32, + w256, + a, + 0, + 0x1b, + 0x4e, + 0x55, + 0x88, + 0xaa, + 0xe4, + 0xff + ); + pair_moves!(a, b, 0, 1, 2, 3, 8, 0x20, 0x31, 0x44, 0x88, 0xc3, 0xff); + let p = _mm256_permutevar_pd(transmute(a), transmute(b)); + let actual: [u64; 4] = transmute(p); + for i in 0..4 { + assert_eq!(actual[i], model::model_permutevar_pd_lane(&a, &b, i)); + } + assert_eq!(w256(_mm256_castpd_si256(p)), actual); + assert_eq!(w128(_mm256_castsi256_si128(transmute(a))), [a[0], a[1]]); + assert_eq!(w128(_mm256_extracti128_si256::<0>(transmute(a))), [a[0], a[1]]); + assert_eq!(w128(_mm256_extracti128_si256::<1>(transmute(a))), [a[2], a[3]]); + let z: [[__m256i; 6]; 2] = zeroed(); + for half in z { + for v in half { + assert_eq!(w256(v), [0; 4]); + } + } + } + let mut rng = Rng::new(0x192_256); + for n in 0..2048 { + unsafe { + run(words(&mut rng, n), words(&mut rng, n + 5)); + } + } +} +#[test] +fn avx512_additional_assumptions() { + if !is_x86_feature_detected!("avx512f") { + eprintln!("AVX512F not available; hardware test skipped"); + return; + } + #[target_feature(enable = "avx512f")] + unsafe fn run(a: [u64; 8]) { + shuffle!( + _mm512_shuffle_epi32, + w512, + a, + 0, + 0x1b, + 0x4e, + 0x55, + 0x88, + 0xaa, + 0xe4, + 0xff + ); + assert_eq!(w512(_mm512_setzero_si512()), [0; 8]); + assert_eq!( + w512(_mm512_broadcast_i32x4(transmute([a[0], a[1]]))), + [a[0], a[1], a[0], a[1], a[0], a[1], a[0], a[1]] + ); + let low = [a[0], a[1], a[2], a[3]]; + assert_eq!(w256(_mm512_castsi512_si256(transmute(a))), low); + let widened = _mm512_castsi256_si512(transmute(low)); + assert_eq!(w256(_mm512_castsi512_si256(widened)), low); + assert_eq!(w256(_mm512_extracti64x4_epi64::<0>(transmute(a))), low); + assert_eq!( + w256(_mm512_extracti64x4_epi64::<1>(transmute(a))), + [a[4], a[5], a[6], a[7]] + ); + let z: [__m512i; 6] = zeroed(); + for v in z { + assert_eq!(w512(v), [0; 8]); + } + } + let mut rng = Rng::new(0x192_512); + for n in 0..2048 { + unsafe { + run(words(&mut rng, n)); + } + } +} diff --git a/verification/verus/tests/equivalence/intrinsics_x86_nttsimd.rs b/verification/verus/tests/equivalence/intrinsics_x86_nttsimd.rs new file mode 100644 index 0000000000..09c8f19de8 --- /dev/null +++ b/verification/verus/tests/equivalence/intrinsics_x86_nttsimd.rs @@ -0,0 +1,66 @@ +//! The memory helpers of the x86-64 NTT butterfly kernels (`intrinsics::x86_nttsimd`) against their +//! specifications: each load returns the words (or bytes) at its offset, read through `transmute` as the views +//! are, and each store writes exactly its words and leaves the rest of the row as it was. Each helper is compiled +//! where its target feature is, so each test runs in the builds that have it. +use core::arch::x86_64::*; +use core::mem::transmute; +use leanvm_verus::gf2_64::F64; +use leanvm_verus::intrinsics::x86_nttsimd as helpers; +use primitives::test_util::Rng; + +const N: usize = 2_000; + +fn random_row(rng: &mut Rng, n: usize) -> Vec { + (0..n).map(|_| F64(rng.next_u64())).collect() +} + +/// Every offset of a row of `W + 7` words: the load gives words `at .. at + W`; the store of random words +/// changes exactly those. +macro_rules! check_load_store { + ($w:literal, $reg:ty, $load:ident, $store:ident) => {{ + let mut rng = Rng::new(0x10AD + $w); + for _ in 0..N { + let row = random_row(&mut rng, $w + 7); + for at in 0..=7 { + let got = unsafe { transmute::<$reg, [u64; $w]>(helpers::$load(&row, at)) }; + let want: [u64; $w] = std::array::from_fn(|j| row[at + j].0); + assert_eq!(got, want, "{} at {at}", stringify!($load)); + let words: [u64; $w] = std::array::from_fn(|_| rng.next_u64()); + let mut stored = row.clone(); + unsafe { helpers::$store(&mut stored, at, transmute::<[u64; $w], $reg>(words)) }; + for j in 0..row.len() { + let want = if (at..at + $w).contains(&j) { + words[j - at] + } else { + row[j].0 + }; + assert_eq!(stored[j].0, want, "{} at {at} word {j}", stringify!($store)); + } + } + } + }}; +} + +#[cfg(target_feature = "avx512f")] +#[test] +fn avx512_row_loads_and_stores() { + check_load_store!(8, __m512i, loadu512_at, storeu512_at); +} + +#[cfg(target_feature = "avx")] +#[test] +fn avx_row_loads_and_stores() { + check_load_store!(4, __m256i, loadu256_at, storeu256_at); +} + +#[test] +fn byte_table_load() { + let mut rng = Rng::new(0x128); + for _ in 0..N { + let bytes: [u8; 16] = std::array::from_fn(|_| rng.next_u64() as u8); + assert_eq!( + unsafe { transmute::<__m128i, [u8; 16]>(helpers::loadu128_bytes(&bytes)) }, + bytes + ); + } +} diff --git a/verification/verus/tests/equivalence/main.rs b/verification/verus/tests/equivalence/main.rs new file mode 100644 index 0000000000..ef21995fc8 --- /dev/null +++ b/verification/verus/tests/equivalence/main.rs @@ -0,0 +1,38 @@ +//! Differential checks between executable verified copies, intrinsic models and production. +//! +//! Edge, exhaustive small-domain and deterministic random cases can detect divergence but do not prove +//! source equivalence or guarantee that future copy drift will be caught. + +mod bit_fold; +mod bits; +mod blake2s; +mod blake2s_batch; +mod fiat_shamir; +mod flock_ntt; +mod gf2_64; +mod gf2_64x3; +mod gf2_8; +#[cfg(target_arch = "aarch64")] +mod intrinsics_aarch64; +#[cfg(target_arch = "aarch64")] +mod intrinsics_aarch64_bits; +#[cfg(target_arch = "aarch64")] +mod intrinsics_aarch64_gfneon; +#[cfg(target_arch = "aarch64")] +mod intrinsics_aarch64_nttsimd; +#[cfg(target_arch = "x86_64")] +mod intrinsics_x86; +#[cfg(target_arch = "x86_64")] +mod intrinsics_x86_bits; +#[cfg(target_arch = "x86_64")] +mod intrinsics_x86_gfneon; +#[cfg(target_arch = "x86_64")] +mod intrinsics_x86_gfx86; +#[cfg(target_arch = "x86_64")] +mod intrinsics_x86_nttsimd; +mod multilinear; +mod ntt; +mod ntt_driver; +mod ntt_simd; +mod phi8_tower; +mod skip_domain; diff --git a/verification/verus/tests/equivalence/multilinear.rs b/verification/verus/tests/equivalence/multilinear.rs new file mode 100644 index 0000000000..9ef10fbdd9 --- /dev/null +++ b/verification/verus/tests/equivalence/multilinear.rs @@ -0,0 +1,190 @@ +//! The verified `eq` tables, their marginalizations, the split table and the MLE evaluation against +//! `primitives::multilinear`. +//! +//! `fill_eq_doubling`, `fold_low_k`, `fold_ladder` and `packed_eq` are private in production; they are reached +//! through `fill_eq_table_uninit` (the doubling below `EQ_PAR_LEN` entries) and `mle_eval` (the other three). +use leanvm_verus::gf2_64::F64 as VF64; +use leanvm_verus::gf2_64x3::F192 as VF192; +use leanvm_verus::multilinear as verified; +use primitives::field::{F192, F64}; +use primitives::multilinear as production; +use primitives::test_util::Rng; +use std::mem::MaybeUninit; + +fn to_v(e: F192) -> VF192 { + VF192::new(e.c0, e.c1, e.c2) +} + +fn to_vs(v: &[F192]) -> Vec { + v.iter().copied().map(to_v).collect() +} + +fn from_vs(v: &[VF192]) -> Vec { + v.iter().map(|e| F192::new(e.c0, e.c1, e.c2)).collect() +} + +/// Random elements with zero and one mixed in, so the products hit their edge cases. +fn point(rng: &mut Rng, n: usize) -> Vec { + (0..n) + .map(|_| match rng.next_u32() % 5 { + 0 => F192::ZERO, + 1 => F192::ONE, + _ => rng.ext(), + }) + .collect() +} + +#[test] +fn interpolations_match() { + let mut rng = Rng::new(0x1A7E); + for _ in 0..20_000 { + let (lo, hi, t) = (rng.ext(), rng.ext(), point(&mut rng, 1)[0]); + assert_eq!( + from_vs(&[verified::interp(to_v(lo), to_v(hi), to_v(t))]), + [production::interp(lo, hi, t)] + ); + let (a, b) = (rng.next_u64(), rng.next_u64()); + assert_eq!( + from_vs(&[verified::interp_k(VF64(a), VF64(b), to_v(t))]), + [production::interp_k(F64(a), F64(b), t)] + ); + } +} + +#[test] +fn eq_evals_match() { + let mut rng = Rng::new(0xE0E7); + for n in 0..12 { + for _ in 0..200 { + let (r, x) = (point(&mut rng, n), point(&mut rng, n)); + assert_eq!( + from_vs(&[verified::eq_eval(&to_vs(&r), &to_vs(&x))]), + [production::eq_eval(&r, &x)], + "n={n}" + ); + } + } +} + +#[test] +fn eq_tables_match() { + // Up to 17 variables: from 16 on, production takes the parallel tensor pass. + let mut rng = Rng::new(0xE07A); + for n in 0..=17 { + let rounds = if n < 12 { 20 } else { 2 }; + for _ in 0..rounds { + let (r, seed) = (point(&mut rng, n), rng.ext()); + assert_eq!( + from_vs(&verified::eq_table(&to_vs(&r))), + production::eq_table(&r), + "n={n}" + ); + let want = production::eq_table_seeded(&r, seed); + assert_eq!( + from_vs(&verified::eq_table_seeded(&to_vs(&r), to_v(seed))), + want, + "n={n}" + ); + + let mut out = vec![MaybeUninit::::uninit(); 1 << n]; + production::fill_eq_table_uninit(&r, seed, &mut out); + // SAFETY: the fill writes every entry. + let filled: Vec = out.iter().map(|e| unsafe { e.assume_init() }).collect(); + let mut vout = vec![VF192::ZERO; 1 << n]; + verified::fill_eq_table_uninit(&to_vs(&r), to_v(seed), &mut vout); + assert_eq!(from_vs(&vout), filled, "n={n}"); + } + } +} + +#[test] +fn shrinks_match() { + let mut rng = Rng::new(0x5A81); + for n in 0..=12 { + for _ in 0..10 { + // Any table, not only an `eq` table: the copies are the pairwise sums of whatever they get. + let t: Vec = if rng.bit() { + production::eq_table_seeded(&point(&mut rng, n), rng.ext()) + } else { + rng.ext_vec(1 << n) + }; + let (mut p, mut v) = (t.clone(), to_vs(&t)); + production::shrink_eq_low(&mut p); + verified::shrink_eq_low(&mut v); + assert_eq!(from_vs(&v), p, "low n={n}"); + let (mut p, mut v) = (t.clone(), to_vs(&t)); + production::shrink_eq_high(&mut p); + verified::shrink_eq_high(&mut v); + assert_eq!(from_vs(&v), p, "high n={n}"); + } + } +} + +#[test] +fn split_tables_match() { + let mut rng = Rng::new(0x5B17); + for n in 0..=12 { + let r = point(&mut rng, n); + let vr = to_vs(&r); + for max in [0, 1, 3, 5, 12, 20] { + let (p, v) = ( + production::SplitEq::with_low_vars(&r, max), + verified::SplitEq::with_low_vars(&vr, max), + ); + assert_eq!( + (from_vs(&v.low), from_vs(&v.high), v.low_log()), + (p.low.clone(), p.high.clone(), p.low_log()) + ); + for x in 0..1usize << n { + assert_eq!(from_vs(&[v.at(x)]), [p.at(x)], "n={n} max_low={max} x={x}"); + } + let (p, v) = ( + production::SplitEq::with_high_vars(&r, max), + verified::SplitEq::with_high_vars(&vr, max), + ); + assert_eq!( + (from_vs(&v.low), from_vs(&v.high), v.low_log()), + (p.low.clone(), p.high.clone(), p.low_log()) + ); + for x in 0..1usize << n { + assert_eq!(from_vs(&[v.at(x)]), [p.at(x)], "n={n} max_high={max} x={x}"); + } + } + } +} + +#[test] +fn mle_evals_match() { + // Below three variables production folds the table directly; from three on it packs the low table, and from + // eleven on there is more than one row. + let mut rng = Rng::new(0x31E0); + for n in 0..=14 { + for _ in 0..(if n < 10 { 20 } else { 3 }) { + let table: Vec = (0..1usize << n).map(|_| F64(rng.next_u64())).collect(); + let p = point(&mut rng, n); + let vt: Vec = table.iter().map(|w| VF64(w.0)).collect(); + assert_eq!( + from_vs(&[verified::mle_eval(&vt, &to_vs(&p))]), + [production::mle_eval(&table, &p)], + "n={n}" + ); + } + } +} + +#[test] +fn window_denominators_match() { + for log in 0..=8 { + let size = 1usize << log; + assert_eq!( + from_vs(&[verified::window_denominator(size)]), + [production::window_denominator(size)], + "size {size}" + ); + assert_eq!( + from_vs(&[verified::DENOMINATORS[log]]), + [production::window_denominator(size)], + "log {log}" + ); + } +} diff --git a/verification/verus/tests/equivalence/ntt.rs b/verification/verus/tests/equivalence/ntt.rs new file mode 100644 index 0000000000..6c84c48b71 --- /dev/null +++ b/verification/verus/tests/equivalence/ntt.rs @@ -0,0 +1,193 @@ +//! The verified NTT pieces against `pcs::ntt::AdditiveNttF64`. +//! +//! The table, `span_get`, `twiddle`, `twiddles_radix8`, the butterflies and the fused row groups are +//! private in production, so they are checked through the public encoder: the verified scalar reference, +//! and a small driver built from the verified radix-8/radix-4 groups, must both reproduce +//! `encode_interleaved_in_place` word for word. +use leanvm_verus::gf2_64::F64 as VF64; +use leanvm_verus::ntt as verified; +use pcs::ntt::AdditiveNttF64 as Production; +use primitives::field::F64; +use primitives::test_util::Rng; + +fn random(rng: &mut Rng, n: usize) -> Vec { + (0..n).map(|_| F64(rng.next_u64())).collect() +} + +fn to_verified(v: &[F64]) -> Vec { + v.iter().map(|x| VF64(x.0)).collect() +} + +fn from_verified(v: &[VF64]) -> Vec { + v.iter().map(|x| F64(x.0)).collect() +} + +#[test] +fn forward_reference_matches_encoder() { + // Every lane count encodes `lanes` independent transforms; the domain size is the table's. + let mut rng = Rng::new(0x4E77); + for log_d in 1..=12usize { + for lanes in [1usize, 2, 3, 5, 8] { + let msg = random(&mut rng, lanes << log_d); + let mut want = to_verified(&msg); + verified::forward_scalar_from_layer(&verified::AdditiveNttF64::standard(log_d), &mut want, lanes, 0); + let mut got = msg; + Production::standard(log_d).encode_interleaved_in_place(&mut got, lanes, 0); + assert_eq!(got, from_verified(&want), "log_d={log_d}, lanes={lanes}"); + } + } +} + +#[test] +fn forward_reference_matches_encoder_at_a_rate() { + // The encoder reads the message from the first replica; the reference replicates it, then runs the + // layers from the rate layer on. + let mut rng = Rng::new(0x4A7E); + for (log_d, lanes, log_inv_rate) in [ + (3usize, 1usize, 1usize), + (6, 3, 2), + (9, 8, 1), + (10, 2, 3), + (12, 4, 2), + (4, 8, 4), + ] { + let msg_len = (lanes << log_d) >> log_inv_rate; + let msg = random(&mut rng, msg_len); + let mut want: Vec = to_verified(&msg).repeat(1 << log_inv_rate); + verified::forward_scalar_from_layer( + &verified::AdditiveNttF64::standard(log_d), + &mut want, + lanes, + log_inv_rate, + ); + let mut got = vec![F64::ZERO; msg_len << log_inv_rate]; + got[..msg_len].copy_from_slice(&msg); + Production::standard(log_d).encode_interleaved_in_place(&mut got, lanes, log_inv_rate); + assert_eq!( + got, + from_verified(&want), + "log_d={log_d}, lanes={lanes}, rate={log_inv_rate}" + ); + } +} + +#[test] +fn inverse_matches_production_and_undoes_the_forward_reference() { + let mut rng = Rng::new(0x1A7F); + for dim in [1usize, 4, 9, 12] { + let (ours, theirs) = (verified::AdditiveNttF64::standard(dim), Production::standard(dim)); + for log_d in 0..=dim { + for _ in 0..4 { + let orig = random(&mut rng, 1 << log_d); + let mut want = to_verified(&orig); + ours.inverse_transform(&mut want); + let mut got = orig.clone(); + theirs.inverse_transform(&mut got); + assert_eq!(got, from_verified(&want), "dim={dim}, log_d={log_d}"); + + if log_d == dim { + let mut back = to_verified(&orig); + verified::forward_scalar_from_layer(&ours, &mut back, 1, 0); + ours.inverse_transform(&mut back); + assert_eq!(from_verified(&back), orig, "roundtrip dim={dim}"); + } + } + } + } +} + +/// One single-lane transform built from the verified fused groups, planned as `run_layers` plans a +/// single sweep: three layers at a time while blocks span at least 8 rows, then two, then one. +fn fused_forward(ntt: &verified::AdditiveNttF64, data: &mut [VF64]) { + let log_d = verified::log2_strict_usize(data.len()); + let mut layer = 0; + while layer < log_d { + let num_blocks = 1usize << layer; + let block_size = 1usize << (log_d - layer); + if layer + 2 < log_d && block_size >= 8 { + let eighth = block_size >> 3; + for block in 0..num_blocks { + let t = ntt.twiddles_radix8(layer, block); + for r in 0..eighth { + let at = |k: usize| block * block_size + r + k * eighth; + let mut rows = [0, 1, 2, 3, 4, 5, 6, 7].map(|k| data[at(k)]); + verified::radix8_butterflies(&mut rows, &t); + for (k, v) in rows.into_iter().enumerate() { + data[at(k)] = v; + } + } + } + layer += 3; + } else if layer + 1 < log_d && block_size >= 4 { + let quarter = block_size >> 2; + for block in 0..num_blocks { + let (t_outer, t_inner_a, t_inner_b) = ( + ntt.twiddle(layer, block), + ntt.twiddle(layer + 1, 2 * block), + ntt.twiddle(layer + 1, 2 * block + 1), + ); + for r in 0..quarter { + let at = |k: usize| block * block_size + r + k * quarter; + let mut rows = [0, 1, 2, 3].map(|k| data[at(k)]); + verified::radix4_butterflies(&mut rows, t_outer, t_inner_a, t_inner_b); + for (k, v) in rows.into_iter().enumerate() { + data[at(k)] = v; + } + } + } + layer += 2; + } else { + let half = block_size >> 1; + for block in 0..num_blocks { + let twiddle = ntt.twiddle(layer, block); + let (top, bot) = data[block * block_size..(block + 1) * block_size].split_at_mut(half); + verified::butterfly_lanes(top, bot, twiddle); + } + layer += 1; + } + } +} + +#[test] +fn fused_groups_match_encoder() { + let mut rng = Rng::new(0xF05E); + for log_d in 1..=13usize { + for _ in 0..4 { + let msg = random(&mut rng, 1 << log_d); + let ntt = verified::AdditiveNttF64::standard(log_d); + let mut fused = to_verified(&msg); + fused_forward(&ntt, &mut fused); + let mut scalar = to_verified(&msg); + verified::forward_scalar_from_layer(&ntt, &mut scalar, 1, 0); + let mut got = msg; + Production::standard(log_d).encode_interleaved_in_place(&mut got, 1, 0); + assert_eq!(got, from_verified(&fused), "fused, log_d={log_d}"); + assert_eq!(got, from_verified(&scalar), "scalar, log_d={log_d}"); + } + } +} + +#[test] +fn transposed_butterfly_matches_its_formula() { + // Production's transposed lane butterfly is crate-private and reached only through the WHIR prover, + // so the verified one is checked against its documented formula in the production field: + // s = u + v; new_u = s; new_v = v + s*t. + let mut rng = Rng::new(0x7A5); + for _ in 0..10_000 { + let (u, v, t) = (F64(rng.next_u64()), F64(rng.next_u64()), F64(rng.next_u64())); + let s = u + v; + let (mut top, mut bot) = ([VF64(u.0)], [VF64(v.0)]); + verified::transposed_butterfly_lanes(&mut top, &mut bot, VF64(t.0)); + assert_eq!((top[0].0, bot[0].0), (s.0, (v + s * t).0)); + } +} + +#[test] +fn log2_matches_production() { + for k in 0..usize::BITS as usize { + assert_eq!( + verified::log2_strict_usize(1 << k), + primitives::log2_strict_usize(1 << k) + ); + } +} diff --git a/verification/verus/tests/equivalence/ntt_driver.rs b/verification/verus/tests/equivalence/ntt_driver.rs new file mode 100644 index 0000000000..f791448b04 --- /dev/null +++ b/verification/verus/tests/equivalence/ntt_driver.rs @@ -0,0 +1,75 @@ +//! Permission-erased driver refinement against the actual production encoder. +//! Each child initializes the real fixed pool with its own worker configuration. +use leanvm_verus::gf2_64::F64 as VerifiedWord; +use leanvm_verus::ntt::{forward_scalar_from_layer, AdditiveNttF64 as Verified}; +use pcs::ntt::AdditiveNttF64 as Production; +use primitives::field::F64; +use primitives::test_util::Rng; + +fn case(rng: &mut Rng, dim: usize, log_d: usize, lanes: usize, start: usize, plans: &[(usize, usize)]) { + let len = lanes << log_d; + let message: Vec<_> = (0..(len >> start)).map(|_| F64(rng.next_u64())).collect(); + let mut production = vec![F64(u64::MAX); len]; + production[..message.len()].copy_from_slice(&message); + Production::standard(dim).encode_interleaved_in_place(&mut production, lanes, start); + let input: Vec<_> = message.iter().map(|x| VerifiedWord(x.0)).cycle().take(len).collect(); + let ntt = Verified::standard(dim); + let mut reference = input.clone(); + forward_scalar_from_layer(&ntt, &mut reference, lanes, start); + assert_eq!( + production.iter().map(|x| x.0).collect::>(), + reference.iter().map(|x| x.0).collect::>(), + "production/reference dim={dim} log_d={log_d} lanes={lanes} start={start}" + ); + for &(deep, width) in plans { + let mut actual = input.clone(); + ntt.transform(&mut actual, log_d, lanes, start, deep, width); + assert_eq!( + actual.iter().map(|x| x.0).collect::>(), + production.iter().map(|x| x.0).collect::>(), + "driver/production dim={dim} log_d={log_d} lanes={lanes} start={start} deep={deep} width={width}" + ); + } +} + +#[test] +fn driver_matches_production_across_plans_and_threads() { + if std::env::var_os("VERUS_DRIVER_CHILD").is_none() { + for workers in [1, 2, 4] { + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "ntt_driver::driver_matches_production_across_plans_and_threads", + "--nocapture", + ]) + .env("VERUS_DRIVER_CHILD", "1") + .env("LEANVM_NUM_THREADS", workers.to_string()) + .status() + .unwrap(); + assert!(status.success(), "driver child with {workers} workers failed"); + } + return; + } + let mut rng = Rng::new(0xD215_EE77); + for log_d in 0..=8 { + for lanes in [1, 3, 8] { + for start in 0..=log_d { + let plans: Vec<_> = (start..=log_d) + .flat_map(|deep| [1, 2, 3, 7].map(|width| (deep, width))) + .collect(); + // A nonempty table also covers the zero-layer domain. + case(&mut rng, (log_d + 2).max(1), log_d, lanes, start, &plans); + } + } + } + for (log_d, lanes, start) in [(12, 1, 0), (12, 3, 2), (13, 56, 1), (16, 3, 3)] { + case( + &mut rng, + log_d, + log_d, + lanes, + start, + &[(start, 3), ((start + log_d) / 2, 3), (log_d, 5)], + ); + } +} diff --git a/verification/verus/tests/equivalence/ntt_simd.rs b/verification/verus/tests/equivalence/ntt_simd.rs new file mode 100644 index 0000000000..5f5391aae3 --- /dev/null +++ b/verification/verus/tests/equivalence/ntt_simd.rs @@ -0,0 +1,197 @@ +//! The verified SIMD butterfly kernels and `lane_butterflies` against production. +//! +//! Production's kernels and `lane_butterflies` are private. Each verified kernel is run on random and edge rows, +//! offsets and twiddles (0, 1, all ones, ...) and compared with production's butterfly in the production field +//! (`u' = u + v t, v' = v + u'`; transposed `s = u + v, u' = s, v' = v + s t`), in both directions; production's +//! own kernels are reached through `encode_interleaved_in_place`, whose row pairs of 8 or more lanes go through +//! them, and which a layer-by-layer driver built from the verified `butterfly_lanes` must reproduce. Each test +//! runs the arm its build compiles: natively, with the AVX2 flags, and under qemu for NEON. +use leanvm_verus::gf2_64::F64 as VF64; +use leanvm_verus::ntt as verified; +use pcs::ntt::AdditiveNttF64 as Production; +use primitives::field::F64; +use primitives::test_util::Rng; + +/// Edge words: zero, one, all ones, the top bit, the reduction constant, alternating bits, every top nibble. +const EDGES: [u64; 10] = [ + 0, + 1, + u64::MAX, + 1 << 63, + 0x1B, + 0x5555_5555_5555_5555, + 0xAAAA_AAAA_AAAA_AAAA, + 0xF << 60, + 0x7 << 60, + 1 << 60, +]; + +/// Twiddles: the edge ones first (0, 1, all ones, ...), then random. +fn twiddles(rng: &mut Rng) -> Vec { + let mut t = EDGES.to_vec(); + t.extend((0..40).map(|_| rng.next_u64())); + t +} + +/// Rows of `n` words: edge words cycled from `k`, or random. +fn row(rng: &mut Rng, n: usize, k: usize) -> Vec { + if k % 3 == 0 { + (0..n).map(|i| EDGES[(k / 3 + i * (k % 7 + 1)) % EDGES.len()]).collect() + } else { + (0..n).map(|_| rng.next_u64()).collect() + } +} + +/// Production's butterfly of one lane, in the production field. +fn production_butterfly(transposed: bool, u: u64, v: u64, t: u64) -> (u64, u64) { + let (u, v, t) = (F64(u), F64(v), F64(t)); + if transposed { + let s = u + v; + (s.0, (v + s * t).0) + } else { + let nu = u + v * t; + (nu.0, (v + nu).0) + } +} + +/// Runs `kernel` on `width` words at every offset of `n`-word rows, and checks every word: production's butterfly +/// on `at .. at + width`, unchanged elsewhere. +fn check_kernel(name: &str, width: usize, transposed: bool, kernel: impl Fn(&mut [VF64], &mut [VF64], usize, u64)) { + let mut rng = Rng::new(0x5_1AD + width as u64 + u64::from(transposed)); + let n = width + 5; + for (k, t) in twiddles(&mut rng).into_iter().enumerate() { + for rep in 0..30 { + let (u, v) = (row(&mut rng, n, k + rep), row(&mut rng, n, k + 2 * rep + 1)); + for at in 0..=n - width { + let mut top: Vec = u.iter().map(|&x| VF64(x)).collect(); + let mut bot: Vec = v.iter().map(|&x| VF64(x)).collect(); + kernel(&mut top, &mut bot, at, t); + for j in 0..n { + let want = if (at..at + width).contains(&j) { + production_butterfly(transposed, u[j], v[j], t) + } else { + (u[j], v[j]) + }; + assert_eq!( + (top[j].0, bot[j].0), + want, + "{name} transposed={transposed} t={t:#x} at={at} word {j}" + ); + } + } + } + } +} + +#[cfg(all(target_arch = "x86_64", target_feature = "vpclmulqdq", target_feature = "avx512f"))] +#[test] +fn avx512_kernel_matches_production_butterfly() { + use leanvm_verus::ntt_simd::butterfly_lanes_avx512; + check_kernel("avx512", 8, false, |t, b, at, tw| unsafe { + butterfly_lanes_avx512::(t, b, at, tw) + }); + check_kernel("avx512", 8, true, |t, b, at, tw| unsafe { + butterfly_lanes_avx512::(t, b, at, tw) + }); +} + +#[cfg(all( + target_arch = "x86_64", + target_feature = "vpclmulqdq", + target_feature = "avx2", + not(target_feature = "avx512f") +))] +#[test] +fn avx2_kernel_matches_production_butterfly() { + use leanvm_verus::ntt_simd::{butterfly_lanes_avx2, SPILL}; + // The listed nibble table is production's formula. + for n in 0..16u8 { + assert_eq!(SPILL[n as usize], n ^ (n >> 1) ^ (n >> 3)); + } + check_kernel("avx2", 4, false, |t, b, at, tw| unsafe { + butterfly_lanes_avx2::(t, b, at, tw) + }); + check_kernel("avx2", 4, true, |t, b, at, tw| unsafe { + butterfly_lanes_avx2::(t, b, at, tw) + }); +} + +#[cfg(all(target_arch = "aarch64", target_feature = "aes"))] +#[test] +fn neon_kernels_match_production_butterfly() { + use leanvm_verus::ntt_simd::{butterfly_lane_pair_neon, butterfly_lanes_neon_8}; + check_kernel("neon_8", 8, false, |t, b, at, tw| unsafe { + butterfly_lanes_neon_8::(t, b, at, tw) + }); + check_kernel("neon_8", 8, true, |t, b, at, tw| unsafe { + butterfly_lanes_neon_8::(t, b, at, tw) + }); + check_kernel("pair", 2, false, |t, b, at, tw| unsafe { + butterfly_lane_pair_neon::(t, b, at, tw) + }); + check_kernel("pair", 2, true, |t, b, at, tw| unsafe { + butterfly_lane_pair_neon::(t, b, at, tw) + }); +} + +/// `lane_butterflies` (through `butterfly_lanes` and `transposed_butterfly_lanes`), whatever arm this build +/// dispatches to, on every row length up to 40: the SIMD blocks, the NEON pair tail and the scalar tail. +#[test] +fn lane_butterflies_match_production_butterfly() { + let mut rng = Rng::new(0x1A_4E5); + for n in 0..=40usize { + for (k, t) in twiddles(&mut rng).into_iter().enumerate() { + let (u, v) = (row(&mut rng, n, k), row(&mut rng, n, k + 1)); + for transposed in [false, true] { + let mut top: Vec = u.iter().map(|&x| VF64(x)).collect(); + let mut bot: Vec = v.iter().map(|&x| VF64(x)).collect(); + if transposed { + verified::transposed_butterfly_lanes(&mut top, &mut bot, VF64(t)); + } else { + verified::butterfly_lanes(&mut top, &mut bot, VF64(t)); + } + for j in 0..n { + assert_eq!( + (top[j].0, bot[j].0), + production_butterfly(transposed, u[j], v[j], t), + "n={n} transposed={transposed} t={t:#x} word {j}" + ); + } + } + } + } +} + +/// Production's encoder runs its own `lane_butterflies` (and so its SIMD kernels) on row pairs of `lanes` words; a +/// layer-by-layer forward transform made of the verified `butterfly_lanes` on the same row pairs must agree. +#[test] +fn verified_lane_butterflies_reproduce_the_encoder() { + let mut rng = Rng::new(0xE1C0); + for log_d in 1..=9usize { + for lanes in [8usize, 9, 10, 11, 12, 16, 17, 24, 31] { + let msg: Vec = (0..lanes << log_d).map(|_| F64(rng.next_u64())).collect(); + let ntt = verified::AdditiveNttF64::standard(log_d); + let mut data: Vec = msg.iter().map(|x| VF64(x.0)).collect(); + for layer in 0..log_d { + let block_size = 1usize << (log_d - layer); + let half = block_size / 2; + for block in 0..1usize << layer { + let twiddle = ntt.twiddle(layer, block); + let rows = &mut data[block * block_size * lanes..(block + 1) * block_size * lanes]; + let (tops, bots) = rows.split_at_mut(half * lanes); + for r in 0..half { + verified::butterfly_lanes( + &mut tops[r * lanes..(r + 1) * lanes], + &mut bots[r * lanes..(r + 1) * lanes], + twiddle, + ); + } + } + } + let mut got = msg; + Production::standard(log_d).encode_interleaved_in_place(&mut got, lanes, 0); + let want: Vec = data.iter().map(|x| F64(x.0)).collect(); + assert_eq!(got, want, "log_d={log_d}, lanes={lanes}"); + } + } +} diff --git a/verification/verus/tests/equivalence/phi8_tower.rs b/verification/verus/tests/equivalence/phi8_tower.rs new file mode 100644 index 0000000000..c32aa3732e --- /dev/null +++ b/verification/verus/tests/equivalence/phi8_tower.rs @@ -0,0 +1,17 @@ +use leanvm_verus::gf2_8 as v8; +use leanvm_verus::phi8_tower as verified; +use primitives::field as production; + +/// Every byte: the table and `phi8_192` (domain of 256 elements, so exhaustive). +#[test] +fn table_and_embedding_match() { + for a in 0..=255u8 { + let (v, p) = (verified::phi8_192(v8::F8(a)), production::phi8_192(production::F8(a))); + assert_eq!((v.c0, v.c1, v.c2), (p.c0, p.c1, p.c2), "phi8_192({a:#04x})"); + let (tv, tp) = ( + verified::PHI_8_TABLE_192[a as usize], + production::PHI_8_TABLE_192[a as usize], + ); + assert_eq!((tv.c0, tv.c1, tv.c2), (tp.c0, tp.c1, tp.c2), "PHI_8_TABLE_192[{a}]"); + } +} diff --git a/verification/verus/tests/equivalence/skip_domain.rs b/verification/verus/tests/equivalence/skip_domain.rs new file mode 100644 index 0000000000..4345adbc48 --- /dev/null +++ b/verification/verus/tests/equivalence/skip_domain.rs @@ -0,0 +1,86 @@ +//! The verified skip domain against `flock::zerocheck::SkipDomain` at the native arithmetic. +//! +//! Production's public surface is `SkipDomain::FLOCK` with `vanishing` and `lagrange_at` (which run the private +//! `vanishing_coefficients`, `linearized`, `inverses`, `lagrange_scale` and `lagrange_with`); those are compared +//! directly. `SkipDomain::new` and `first_round_at` are crate-private and only run inside a whole zerocheck, so for +//! every other size, and for `first_round_at`, the copy is compared with what production's own tests compare +//! production with (`skip_domain::tests`): the product `prod_i (z + s_i)` and the weights of +//! `primitives::multilinear::skip_lagrange_weights`. +use fiat_shamir::arith::Native; +use flock::zerocheck::{SkipDomain, K_SKIP}; +use leanvm_verus::gf2_64x3::F192 as VF192; +use leanvm_verus::skip_domain as verified; +use primitives::field::{F192, PHI_8_TABLE_192}; +use primitives::multilinear::skip_lagrange_weights; +use primitives::test_util::Rng; + +fn to_v(e: F192) -> VF192 { + VF192::new(e.c0, e.c1, e.c2) +} + +fn from_v(e: VF192) -> F192 { + F192::new(e.c0, e.c1, e.c2) +} + +/// Random points, then every node of the largest window (where production returns zero). +fn points(rng: &mut Rng) -> Vec { + let mut out = rng.ext_vec(40); + out.extend_from_slice(&PHI_8_TABLE_192[..]); + out +} + +#[test] +fn the_flock_domain_matches() { + let mut rng = Rng::new(0x5D0); + let domain = SkipDomain::FLOCK; + let vdomain = verified::SkipDomain::FLOCK; + assert_eq!( + (verified::K_SKIP, vdomain.k_skip(), vdomain.size()), + (K_SKIP, K_SKIP, 1 << K_SKIP) + ); + for z in points(&mut rng) { + let vanishing = domain.vanishing(&mut Native, z); + let vvanishing = vdomain.vanishing(&mut verified::Native, to_v(z)); + assert_eq!(from_v(vvanishing), vanishing, "z={z:?}"); + let values = rng.ext_vec(1 << K_SKIP); + let vvalues: Vec = values.iter().copied().map(to_v).collect(); + let want = domain.lagrange_at(&mut Native, z, vanishing, &values); + let got = vdomain.lagrange_at(&mut verified::Native, to_v(z), vvanishing, &vvalues); + assert_eq!(from_v(got), want, "z={z:?}"); + } +} + +#[test] +fn every_size_matches_the_references() { + let mut rng = Rng::new(0x5D1); + for k_skip in 0..8 { + let domain = verified::SkipDomain::new(k_skip); + let size = domain.size(); + for z in rng.ext_vec(8) { + let vz = to_v(z); + let vanishing = domain.vanishing(&mut verified::Native, vz); + let product = PHI_8_TABLE_192[..size].iter().fold(F192::ONE, |acc, &s| acc * (z + s)); + assert_eq!(from_v(vanishing), product, "k_skip {k_skip}"); + + let values = rng.ext_vec(size); + let vvalues: Vec = values.iter().copied().map(to_v).collect(); + let lagrange = + (skip_lagrange_weights(k_skip, z).iter().zip(&values)).fold(F192::ZERO, |acc, (&w, &v)| acc + w * v); + assert_eq!( + from_v(domain.lagrange_at(&mut verified::Native, vz, vanishing, &vvalues)), + lagrange, + "k_skip {k_skip}" + ); + + let window = skip_lagrange_weights(k_skip + 1, z)[size..] + .iter() + .zip(&values) + .fold(F192::ZERO, |acc, (&w, &v)| acc + w * v); + assert_eq!( + from_v(domain.first_round_at(&mut verified::Native, vz, vanishing, &vvalues)), + window, + "k_skip {k_skip}" + ); + } + } +} diff --git a/verification/verus/verify.sh b/verification/verus/verify.sh new file mode 100755 index 0000000000..4fdde03888 --- /dev/null +++ b/verification/verus/verify.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Check the proofs with the pinned Verus release, installing it under $VERUS_HOME (default ~/.verus) if missing. +# +# verification/verus/verify.sh # all modules, every configuration +# verification/verus/verify.sh gf2_64 # one module (and the modules it uses), every configuration +# VERUS_CONFIGS="portable neon" verification/verus/verify.sh # some configurations +# +# The kernels pick their SIMD arm with `cfg(target_feature)`, so Verus sees one arm per build. Each configuration +# below is one build, with the flags of a CPU class production targets (the CI legs' flags): together they check +# every arm, the portable one included. The NEON ones are cross-checked from this host with `--target`. +set -euo pipefail + +# The release, and the Rust toolchain it is built against (its `version.json`). +VERUS_RELEASE=0.2026.10.04.426d8b0 +VERUS_TOOLCHAIN=1.98.1 + +# name | target triple | RUSTFLAGS +CONFIGS=( + "portable|x86_64-unknown-linux-gnu|-C target-cpu=x86-64" + "haswell|x86_64-unknown-linux-gnu|-C target-cpu=haswell" + "avx2-vpclmulqdq|x86_64-unknown-linux-gnu|-C target-cpu=x86-64-v3 -C target-feature=+vpclmulqdq,+pclmulqdq" + "avx2-gfni|x86_64-unknown-linux-gnu|-C target-cpu=x86-64-v3 -C target-feature=+gfni,+vpclmulqdq,+pclmulqdq,+aes" + "avx512|x86_64-unknown-linux-gnu|-C target-cpu=x86-64-v4 -C target-feature=+avx512vbmi,+vpclmulqdq,+pclmulqdq,+gfni,+aes" + "neon|aarch64-unknown-linux-gnu|-C target-cpu=neoverse-n2 -C target-feature=+aes,+sha3" + "neon-no-sha3|aarch64-unknown-linux-gnu|-C target-feature=+aes" +) + +home="${VERUS_HOME:-$HOME/.verus}/verus-$VERUS_RELEASE" +if [ ! -x "$home/verus-x86-linux/verus" ]; then + mkdir -p "$home" + curl -fsSL -o "$home/verus.zip" \ + "https://github.com/verus-lang/verus/releases/download/release/$VERUS_RELEASE/verus-$VERUS_RELEASE-x86-linux.zip" + unzip -q "$home/verus.zip" -d "$home" + rm "$home/verus.zip" +fi +rustup toolchain install "$VERUS_TOOLCHAIN" --profile minimal --no-self-update >/dev/null 2>&1 +rustup target add --toolchain "$VERUS_TOOLCHAIN" aarch64-unknown-linux-gnu >/dev/null 2>&1 +export PATH="$home/verus-x86-linux:$PATH" + +cd "$(dirname "$0")" +threads=(--num-threads "${VERUS_THREADS:-4}") +selected="${VERUS_CONFIGS:-}" +target_root="${CARGO_TARGET_DIR:-target/verus}" +# `cargo verus focus` can replay a cached root-package result with different forwarded module arguments. +# Keep each requested module separate from other focus queries and from the whole-crate proof. +scope=all +if [ $# -gt 0 ]; then + scope="focus/$1" +fi +for config in "${CONFIGS[@]}"; do + IFS='|' read -r name triple flags <<<"$config" + if [ -n "$selected" ] && [[ " $selected " != *" $name "* ]]; then + continue + fi + echo "== $name ($flags)" + # Isolate the ISA flags and proof selection, while reusing each selection's dependency cache. + export RUSTFLAGS="$flags" CARGO_TARGET_DIR="$target_root/$name/$scope" + if [ $# -gt 0 ]; then + cargo verus focus --target "$triple" -- "${threads[@]}" --verify-module "$1" + else + cargo verus verify --target "$triple" -- "${threads[@]}" + fi +done