From c7d00ad94e6db827abd5c72e61ef0381cce8bfcc Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Wed, 11 Nov 2026 02:59:32 +0200
Subject: [PATCH 1/7] perf(encode): seed only the dense block tail after a
 dfast scan

After its scan the dfast matcher indexed every position from where the
scan stopped to the block's end. Upstream's loops end without that, and
on incompressible input the scan stops a step short of the end, the step
having grown to hundreds of bytes by then: hundreds of inserts a frame
that nothing searches when the frame is one block. The seed now indexes
the dense tail alone, the last starts the scan cannot reach, which the
next block may match into; the position the scan stopped at no longer
travels out of the loop.

100 KiB random at level 3: -11% cycles, -24% instructions, 1.21x -> 1.07x
of libzstd; z000033 whole and in 10 KiB frames at level 3 flat. Bytes over
the ten largest decodecorpus files: levels 1, 2 and 5 identical, levels 3
and 4 +43 / +44 of 3.1 MB.

Part of #653
---
 zstd/src/encoding/dfast/mod.rs             | 60 ++++++++-----------
 zstd/src/encoding/match_generator/tests.rs | 68 ++++++++--------------
 2 files changed, 47 insertions(+), 81 deletions(-)

diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs
index 03fd873f..c16e0fdc 100644
--- a/zstd/src/encoding/dfast/mod.rs
+++ b/zstd/src/encoding/dfast/mod.rs
@@ -1967,14 +1967,19 @@ impl DfastMatchGenerator {
         pos
     }
 
+    /// Index the block's dense tail, the last starts the scan cannot reach,
+    /// so the next block can match into them.
     pub(crate) fn seed_remaining_hashable_starts(
         &mut self,
         current_abs_start: usize,
         current_len: usize,
-        pos: usize,
     ) {
-        let boundary_tail_start = current_len.saturating_sub(Self::BOUNDARY_DENSE_TAIL_LEN);
-        let mut seed_pos = pos.min(current_len).min(boundary_tail_start);
+        // The tail only, never the gap the scan's step skipped before it
+        // stopped: upstream's loops index neither, and on incompressible input
+        // that gap is the step, hundreds of bytes by a block's end. Seeding it
+        // cost 11% of the cycles of a 100 KiB random frame at level 3 for 43
+        // bytes over 3.1 MB of corpus at levels 3-4.
+        let mut seed_pos = current_len.saturating_sub(Self::BOUNDARY_DENSE_TAIL_LEN);
         // Every start with a whole short key; past it nothing is hashable.
         while seed_pos + self.mls <= current_len {
             self.insert_position(current_abs_start + seed_pos);
@@ -2856,16 +2861,9 @@ macro_rules! start_matching_fast_loop_body {
             // load bound — using it here read up to 3 bytes past
             // `history.len()` on tiny blocks (CI fuzz `interop` crash,
             // 7-byte input).
-            // NOTE: when this guard fires on the very first outer-iter
-            // (tiny block, `$current_len < 9`), we `break 'outer` BEFORE
-            // `tail_seed_anchor` is even declared. The post-loop
-            // `seed_remaining_hashable_starts(.., pos)` then runs with
-            // `pos` at its initial value (1 on first frame, or the
-            // post-match cursor from a previous outer iter); that's the
-            // correct tail seed for tiny blocks — `seed_pos = pos.min(
-            // $current_len).min(boundary_tail_start)` plus the
-            // `seed_pos + DFAST_SHORT_HASH_LOOKAHEAD <= $current_len`
-            // guard inside the seeder keeps every insert in-bounds.
+            // On a tiny block this fires on the first pass; the post-loop
+            // tail seed then indexes the block's dense tail, its own
+            // whole-key guard keeping every insert in bounds.
             if pos + HASH_READ_SIZE > $current_len {
                 break 'outer;
             }
@@ -2996,7 +2994,7 @@ macro_rules! start_matching_fast_loop_body {
                 // `curr`, zstd_double_fast.c:184), NOT on the (rep1 `+1` / catch-up
                 // adjusted) match start.
                 Committed(MatchCandidate, u8),
-                Tail(usize),
+                Tail,
             }
 
             let inner_exit: InnerExit = 'inner: loop {
@@ -3472,11 +3470,7 @@ macro_rules! start_matching_fast_loop_body {
                 // leaving a remainder inside the build-to-build drift measured
                 // on this host. Kept for the op-reduction, not on a speed claim.
                 if ip1 > scan_limit {
-                    // First position the fast loop did NOT pack into the
-                    // hash tables. `seed_remaining_hashable_starts` will
-                    // pick up from `ip0` instead of restarting at the
-                    // outer-entry `pos`.
-                    break 'inner InnerExit::Tail(ip0);
+                    break 'inner InnerExit::Tail;
                 }
             };
 
@@ -3623,18 +3617,14 @@ macro_rules! start_matching_fast_loop_body {
                     }
                     }
                 }
-                InnerExit::Tail(seed) => {
-                    // Inner loop ran out of safe scan room without
-                    // committing. Hand the tail seeder the first
-                    // un-scanned position so it does not redundantly
-                    // re-pack everything the fast loop already wrote.
-                    pos = seed;
+                InnerExit::Tail => {
+                    // Out of safe scan room without committing.
                     break 'outer;
                 }
             }
         }
 
-        $self.seed_remaining_hashable_starts($current_abs_start, $current_len, pos);
+        $self.seed_remaining_hashable_starts($current_abs_start, $current_len);
         $self.emit_trailing_literals($current_abs_start, literals_start, $handle_sequence);
     }};
 }
@@ -3991,7 +3981,7 @@ macro_rules! start_matching_dict_loop_body {
                         if dfast_branch!(10, cand4 != v4) {
                             ip += ((ip - literals_start) >> DFAST_SKIP_STEP_SHIFT) + 1;
                             if dfast_branch!(24, ip > scan_limit) {
-                                break 'inner DfastInnerExit::Tail(ip);
+                                break 'inner DfastInnerExit::Tail;
                             }
                             continue 'inner;
                         }
@@ -4024,7 +4014,7 @@ macro_rules! start_matching_dict_loop_body {
                         if dfast_branch!(14, found == usize::MAX) {
                             ip += ((ip - literals_start) >> DFAST_SKIP_STEP_SHIFT) + 1;
                             if dfast_branch!(24, ip > scan_limit) {
-                                break 'inner DfastInnerExit::Tail(ip);
+                                break 'inner DfastInnerExit::Tail;
                             }
                             continue 'inner;
                         }
@@ -4147,7 +4137,7 @@ macro_rules! start_matching_dict_loop_body {
                     // A below-floor short hit with no upgrade: keep scanning.
                     ip += ((ip - literals_start) >> DFAST_SKIP_STEP_SHIFT) + 1;
                     if dfast_branch!(24, ip > scan_limit) {
-                        break 'inner DfastInnerExit::Tail(ip);
+                        break 'inner DfastInnerExit::Tail;
                     }
                 };
 
@@ -4280,27 +4270,23 @@ macro_rules! start_matching_dict_loop_body {
                             }
                         }
                     }
-                    DfastInnerExit::Tail(seed) => {
-                        pos = seed;
-                        break 'outer;
-                    }
+                    DfastInnerExit::Tail => break 'outer,
                 }
             }
             $self.offset_hist = reps;
         }
 
-        $self.seed_remaining_hashable_starts($current_abs_start, $current_len, pos);
+        $self.seed_remaining_hashable_starts($current_abs_start, $current_len);
         $self.emit_trailing_literals($current_abs_start, literals_start, $handle_sequence);
     }};
 }
 
 /// How the dictionary scan loop left its inner loop: with a match to emit
 /// (candidate, a path tag the `DFTRACE` gate prints, and the SCAN position the
-/// complementary insertion anchors on), or out of scan room at the first
-/// position it did not pack into the tables.
+/// complementary insertion anchors on), or out of scan room.
 enum DfastInnerExit {
     Committed(MatchCandidate, u8, usize),
-    Tail(usize),
+    Tail,
 }
 
 /// [`extend_backwards_shared`] over a window held in two buffers: the
diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs
index 95ae6628..f8bf6339 100644
--- a/zstd/src/encoding/match_generator/tests.rs
+++ b/zstd/src/encoding/match_generator/tests.rs
@@ -5083,8 +5083,12 @@ fn dfast_skip_matching_dense_backfills_newly_hashable_long_tail_positions() {
     );
 }
 
+/// The tail seed indexes the block's dense tail, the last starts the scan
+/// cannot reach, so the next block can match into them, and nothing before it:
+/// the gap a stepping scan skipped stays unindexed, as upstream leaves it.
+/// Seeding that gap cost 11% of a 100 KiB random frame's cycles at level 3.
 #[test]
-fn dfast_seed_remaining_hashable_starts_seeds_last_short_hash_positions() {
+fn dfast_seed_remaining_hashable_starts_seeds_the_dense_tail_only() {
     let mut matcher = DfastMatchGenerator::new(1 << 20);
     let block = deterministic_high_entropy_bytes(0x13F0_9A6D_55CE_7B21, 64);
     matcher.commit_input(&block);
@@ -5092,57 +5096,33 @@ fn dfast_seed_remaining_hashable_starts_seeds_last_short_hash_positions() {
 
     let current_len = matcher.window_blocks.back().copied().unwrap_or(0);
     let current_abs_start = matcher.history_abs_start + matcher.window_size - current_len;
-    let seed_start = current_len - DFAST_MIN_MATCH_LEN;
-    matcher.seed_remaining_hashable_starts(current_abs_start, current_len, seed_start);
+    matcher.seed_remaining_hashable_starts(current_abs_start, current_len);
 
-    let target_abs_pos = current_abs_start + current_len - 5;
-    let target_rel = target_abs_pos - matcher.history_abs_start;
-    let live = matcher.live_history();
-    assert!(
-        target_rel + 5 <= live.len(),
-        "fixture must leave the last short-hash start valid"
-    );
-    let short_hash = matcher.short_hash_index(&live[target_rel..]);
-    let target_slot = matcher.pack_slot(target_abs_pos);
+    let short_slot_of = |matcher: &DfastMatchGenerator, abs_pos: usize| {
+        let rel = abs_pos - matcher.history_abs_start;
+        let live = matcher.live_history();
+        assert!(
+            rel + 5 <= live.len(),
+            "fixture must leave the start short-hashable"
+        );
+        matcher.tables[matcher.long_len() + matcher.short_hash_index(&live[rel..])]
+    };
+    let last = current_abs_start + current_len - 5;
+    let last_slot = matcher.pack_slot(last);
     assert_ne!(
-        target_slot, DFAST_EMPTY_SLOT,
+        last_slot, DFAST_EMPTY_SLOT,
         "pack_slot must never return the empty-slot sentinel for a real position"
     );
     assert_eq!(
-        matcher.tables[matcher.long_len() + short_hash],
-        target_slot,
+        short_slot_of(&matcher, last),
+        last_slot,
         "tail seeding must include the last 5-byte-hashable start"
     );
-}
-
-#[test]
-fn dfast_seed_remaining_hashable_starts_handles_pos_at_block_end() {
-    let mut matcher = DfastMatchGenerator::new(1 << 20);
-    let block = deterministic_high_entropy_bytes(0x7BB2_DA91_441E_C0EF, 64);
-    matcher.commit_input(&block);
-    matcher.ensure_hash_tables();
-
-    let current_len = matcher.window_blocks.back().copied().unwrap_or(0);
-    let current_abs_start = matcher.history_abs_start + matcher.window_size - current_len;
-    matcher.seed_remaining_hashable_starts(current_abs_start, current_len, current_len);
-
-    let target_abs_pos = current_abs_start + current_len - 5;
-    let target_rel = target_abs_pos - matcher.history_abs_start;
-    let live = matcher.live_history();
-    assert!(
-        target_rel + 5 <= live.len(),
-        "fixture must leave the last short-hash start valid"
-    );
-    let short_hash = matcher.short_hash_index(&live[target_rel..]);
-    let target_slot = matcher.pack_slot(target_abs_pos);
+    let gap = current_abs_start + current_len - DfastMatchGenerator::BOUNDARY_DENSE_TAIL_LEN - 20;
     assert_ne!(
-        target_slot, DFAST_EMPTY_SLOT,
-        "pack_slot must never return the empty-slot sentinel for a real position"
-    );
-    assert_eq!(
-        matcher.tables[matcher.long_len() + short_hash],
-        target_slot,
-        "tail seeding must still include the last 5-byte-hashable start when pos is at block end"
+        short_slot_of(&matcher, gap),
+        matcher.pack_slot(gap),
+        "tail seeding must leave the gap before the dense tail unindexed"
     );
 }
 

From 283edd6bb2e194c1e8719d26b65147040ca6f410 Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Wed, 11 Nov 2026 03:09:24 +0200
Subject: [PATCH 2/7] perf(encode): seed only the unscanned dense tail after a
 dfast scan

After its scan the dfast matcher indexed every position from where the
scan stopped to the block's end, or the whole dense tail when it stopped
past it. Upstream's loops end without either, and on incompressible
input the scan stops a step short of the end, the step having grown to
hundreds of bytes by then: hundreds of inserts a frame that nothing
searches when the frame is one block. The seed now indexes only the
dense-tail starts the scan did not pass.

The previous commit seeded the whole tail and dropped the stop position
from the loop's exit; that changed the scan's code and cost z000033 at
level 3 +4.2% cycles (+3.1% instructions), so it is reverted here in
favour of the measured form, which keeps the scan's code as it was.

100 KiB random at level 3: -11.0% cycles, -23.8% instructions, 1.21x ->
1.07x of libzstd; z000033 whole -0.7%, in 10 KiB frames +0.5%, the L1
control flat. Bytes over the ten largest decodecorpus files: levels 1, 2
and 5 identical, levels 3 and 4 +43 / +44 of 3.1 MB.

Part of #653
---
 zstd/src/encoding/dfast/mod.rs             | 68 +++++++++++++-------
 zstd/src/encoding/match_generator/tests.rs | 72 +++++++++++++++++-----
 2 files changed, 101 insertions(+), 39 deletions(-)

diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs
index c16e0fdc..a7d8eaea 100644
--- a/zstd/src/encoding/dfast/mod.rs
+++ b/zstd/src/encoding/dfast/mod.rs
@@ -1967,19 +1967,22 @@ impl DfastMatchGenerator {
         pos
     }
 
-    /// Index the block's dense tail, the last starts the scan cannot reach,
-    /// so the next block can match into them.
+    /// Index the starts of the block's dense tail that the scan did not pass,
+    /// `pos` being where it stopped, so the next block can match into them.
     pub(crate) fn seed_remaining_hashable_starts(
         &mut self,
         current_abs_start: usize,
         current_len: usize,
+        pos: usize,
     ) {
-        // The tail only, never the gap the scan's step skipped before it
-        // stopped: upstream's loops index neither, and on incompressible input
-        // that gap is the step, hundreds of bytes by a block's end. Seeding it
-        // cost 11% of the cycles of a 100 KiB random frame at level 3 for 43
-        // bytes over 3.1 MB of corpus at levels 3-4.
-        let mut seed_pos = current_len.saturating_sub(Self::BOUNDARY_DENSE_TAIL_LEN);
+        let boundary_tail_start = current_len.saturating_sub(Self::BOUNDARY_DENSE_TAIL_LEN);
+        // Never the gap the scan's step skipped before it stopped, nor what a
+        // match ending in the tail covered: upstream's loops index neither,
+        // and on incompressible input that gap is the step, hundreds of bytes
+        // by a block's end. Seeding it cost 11% of the cycles of a 100 KiB
+        // random frame at level 3 for 43 bytes over 3.1 MB of corpus at
+        // levels 3-4.
+        let mut seed_pos = pos.min(current_len).max(boundary_tail_start);
         // Every start with a whole short key; past it nothing is hashable.
         while seed_pos + self.mls <= current_len {
             self.insert_position(current_abs_start + seed_pos);
@@ -2861,9 +2864,16 @@ macro_rules! start_matching_fast_loop_body {
             // load bound — using it here read up to 3 bytes past
             // `history.len()` on tiny blocks (CI fuzz `interop` crash,
             // 7-byte input).
-            // On a tiny block this fires on the first pass; the post-loop
-            // tail seed then indexes the block's dense tail, its own
-            // whole-key guard keeping every insert in bounds.
+            // NOTE: when this guard fires on the very first outer-iter
+            // (tiny block, `$current_len < 9`), we `break 'outer` BEFORE
+            // `tail_seed_anchor` is even declared. The post-loop
+            // `seed_remaining_hashable_starts(.., pos)` then runs with
+            // `pos` at its initial value (1 on first frame, or the
+            // post-match cursor from a previous outer iter); that's the
+            // correct tail seed for tiny blocks — `seed_pos = pos.min(
+            // $current_len).min(boundary_tail_start)` plus the
+            // `seed_pos + DFAST_SHORT_HASH_LOOKAHEAD <= $current_len`
+            // guard inside the seeder keeps every insert in-bounds.
             if pos + HASH_READ_SIZE > $current_len {
                 break 'outer;
             }
@@ -2994,7 +3004,7 @@ macro_rules! start_matching_fast_loop_body {
                 // `curr`, zstd_double_fast.c:184), NOT on the (rep1 `+1` / catch-up
                 // adjusted) match start.
                 Committed(MatchCandidate, u8),
-                Tail,
+                Tail(usize),
             }
 
             let inner_exit: InnerExit = 'inner: loop {
@@ -3470,7 +3480,11 @@ macro_rules! start_matching_fast_loop_body {
                 // leaving a remainder inside the build-to-build drift measured
                 // on this host. Kept for the op-reduction, not on a speed claim.
                 if ip1 > scan_limit {
-                    break 'inner InnerExit::Tail;
+                    // First position the fast loop did NOT pack into the
+                    // hash tables. `seed_remaining_hashable_starts` will
+                    // pick up from `ip0` instead of restarting at the
+                    // outer-entry `pos`.
+                    break 'inner InnerExit::Tail(ip0);
                 }
             };
 
@@ -3617,14 +3631,18 @@ macro_rules! start_matching_fast_loop_body {
                     }
                     }
                 }
-                InnerExit::Tail => {
-                    // Out of safe scan room without committing.
+                InnerExit::Tail(seed) => {
+                    // Inner loop ran out of safe scan room without
+                    // committing. Hand the tail seeder the first
+                    // un-scanned position so it does not redundantly
+                    // re-pack everything the fast loop already wrote.
+                    pos = seed;
                     break 'outer;
                 }
             }
         }
 
-        $self.seed_remaining_hashable_starts($current_abs_start, $current_len);
+        $self.seed_remaining_hashable_starts($current_abs_start, $current_len, pos);
         $self.emit_trailing_literals($current_abs_start, literals_start, $handle_sequence);
     }};
 }
@@ -3981,7 +3999,7 @@ macro_rules! start_matching_dict_loop_body {
                         if dfast_branch!(10, cand4 != v4) {
                             ip += ((ip - literals_start) >> DFAST_SKIP_STEP_SHIFT) + 1;
                             if dfast_branch!(24, ip > scan_limit) {
-                                break 'inner DfastInnerExit::Tail;
+                                break 'inner DfastInnerExit::Tail(ip);
                             }
                             continue 'inner;
                         }
@@ -4014,7 +4032,7 @@ macro_rules! start_matching_dict_loop_body {
                         if dfast_branch!(14, found == usize::MAX) {
                             ip += ((ip - literals_start) >> DFAST_SKIP_STEP_SHIFT) + 1;
                             if dfast_branch!(24, ip > scan_limit) {
-                                break 'inner DfastInnerExit::Tail;
+                                break 'inner DfastInnerExit::Tail(ip);
                             }
                             continue 'inner;
                         }
@@ -4137,7 +4155,7 @@ macro_rules! start_matching_dict_loop_body {
                     // A below-floor short hit with no upgrade: keep scanning.
                     ip += ((ip - literals_start) >> DFAST_SKIP_STEP_SHIFT) + 1;
                     if dfast_branch!(24, ip > scan_limit) {
-                        break 'inner DfastInnerExit::Tail;
+                        break 'inner DfastInnerExit::Tail(ip);
                     }
                 };
 
@@ -4270,23 +4288,27 @@ macro_rules! start_matching_dict_loop_body {
                             }
                         }
                     }
-                    DfastInnerExit::Tail => break 'outer,
+                    DfastInnerExit::Tail(seed) => {
+                        pos = seed;
+                        break 'outer;
+                    }
                 }
             }
             $self.offset_hist = reps;
         }
 
-        $self.seed_remaining_hashable_starts($current_abs_start, $current_len);
+        $self.seed_remaining_hashable_starts($current_abs_start, $current_len, pos);
         $self.emit_trailing_literals($current_abs_start, literals_start, $handle_sequence);
     }};
 }
 
 /// How the dictionary scan loop left its inner loop: with a match to emit
 /// (candidate, a path tag the `DFTRACE` gate prints, and the SCAN position the
-/// complementary insertion anchors on), or out of scan room.
+/// complementary insertion anchors on), or out of scan room at the first
+/// position it did not pack into the tables.
 enum DfastInnerExit {
     Committed(MatchCandidate, u8, usize),
-    Tail,
+    Tail(usize),
 }
 
 /// [`extend_backwards_shared`] over a window held in two buffers: the
diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs
index f8bf6339..1e4ad4c8 100644
--- a/zstd/src/encoding/match_generator/tests.rs
+++ b/zstd/src/encoding/match_generator/tests.rs
@@ -5083,12 +5083,8 @@ fn dfast_skip_matching_dense_backfills_newly_hashable_long_tail_positions() {
     );
 }
 
-/// The tail seed indexes the block's dense tail, the last starts the scan
-/// cannot reach, so the next block can match into them, and nothing before it:
-/// the gap a stepping scan skipped stays unindexed, as upstream leaves it.
-/// Seeding that gap cost 11% of a 100 KiB random frame's cycles at level 3.
 #[test]
-fn dfast_seed_remaining_hashable_starts_seeds_the_dense_tail_only() {
+fn dfast_seed_remaining_hashable_starts_seeds_last_short_hash_positions() {
     let mut matcher = DfastMatchGenerator::new(1 << 20);
     let block = deterministic_high_entropy_bytes(0x13F0_9A6D_55CE_7B21, 64);
     matcher.commit_input(&block);
@@ -5096,8 +5092,35 @@ fn dfast_seed_remaining_hashable_starts_seeds_the_dense_tail_only() {
 
     let current_len = matcher.window_blocks.back().copied().unwrap_or(0);
     let current_abs_start = matcher.history_abs_start + matcher.window_size - current_len;
-    matcher.seed_remaining_hashable_starts(current_abs_start, current_len);
+    let seed_start = current_len - DFAST_MIN_MATCH_LEN;
+    matcher.seed_remaining_hashable_starts(current_abs_start, current_len, seed_start);
 
+    let target_abs_pos = current_abs_start + current_len - 5;
+    let target_rel = target_abs_pos - matcher.history_abs_start;
+    let live = matcher.live_history();
+    assert!(
+        target_rel + 5 <= live.len(),
+        "fixture must leave the last short-hash start valid"
+    );
+    let short_hash = matcher.short_hash_index(&live[target_rel..]);
+    let target_slot = matcher.pack_slot(target_abs_pos);
+    assert_ne!(
+        target_slot, DFAST_EMPTY_SLOT,
+        "pack_slot must never return the empty-slot sentinel for a real position"
+    );
+    assert_eq!(
+        matcher.tables[matcher.long_len() + short_hash],
+        target_slot,
+        "tail seeding must include the last 5-byte-hashable start"
+    );
+}
+
+/// The tail seed indexes only the dense tail past where the scan stopped:
+/// neither the gap a stepping scan skipped before the tail, nor tail starts a
+/// match ending at the block's end already covered, as upstream's loops leave
+/// both. Seeding the gap cost 11% of a 100 KiB random frame's cycles at level 3.
+#[test]
+fn dfast_seed_remaining_hashable_starts_skips_the_gap_and_covered_starts() {
     let short_slot_of = |matcher: &DfastMatchGenerator, abs_pos: usize| {
         let rel = abs_pos - matcher.history_abs_start;
         let live = matcher.live_history();
@@ -5107,22 +5130,39 @@ fn dfast_seed_remaining_hashable_starts_seeds_the_dense_tail_only() {
         );
         matcher.tables[matcher.long_len() + matcher.short_hash_index(&live[rel..])]
     };
-    let last = current_abs_start + current_len - 5;
-    let last_slot = matcher.pack_slot(last);
-    assert_ne!(
-        last_slot, DFAST_EMPTY_SLOT,
-        "pack_slot must never return the empty-slot sentinel for a real position"
-    );
+    let fresh = || {
+        let mut matcher = DfastMatchGenerator::new(1 << 20);
+        let block = deterministic_high_entropy_bytes(0x7BB2_DA91_441E_C0EF, 64);
+        matcher.commit_input(&block);
+        matcher.ensure_hash_tables();
+        let current_len = matcher.window_blocks.back().copied().unwrap_or(0);
+        let current_abs_start = matcher.history_abs_start + matcher.window_size - current_len;
+        (matcher, current_abs_start, current_len)
+    };
+
+    // The scan stopped well before the tail: the tail is seeded, the gap is not.
+    let (mut matcher, start, len) = fresh();
+    matcher.seed_remaining_hashable_starts(start, len, 10);
+    let last = start + len - 5;
     assert_eq!(
         short_slot_of(&matcher, last),
-        last_slot,
-        "tail seeding must include the last 5-byte-hashable start"
+        matcher.pack_slot(last),
+        "the tail past an early stop is seeded"
     );
-    let gap = current_abs_start + current_len - DfastMatchGenerator::BOUNDARY_DENSE_TAIL_LEN - 20;
+    let gap = start + len - DfastMatchGenerator::BOUNDARY_DENSE_TAIL_LEN - 20;
     assert_ne!(
         short_slot_of(&matcher, gap),
         matcher.pack_slot(gap),
-        "tail seeding must leave the gap before the dense tail unindexed"
+        "the gap before the dense tail stays unindexed"
+    );
+
+    // A match ran to the block's end: nothing is left to seed.
+    let (mut matcher, start, len) = fresh();
+    matcher.seed_remaining_hashable_starts(start, len, len);
+    assert_ne!(
+        short_slot_of(&matcher, last),
+        matcher.pack_slot(last),
+        "starts a match covered are not seeded"
     );
 }
 

From 6e9d4bb1c5e63922784fb8ce78d9a6a4cc61f7d6 Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Wed, 11 Nov 2026 11:17:38 +0200
Subject: [PATCH 3/7] docs(encode): state the dfast tail seed's cost in time

The comment justifying the bytes the seed gives up quoted cycles. The
same paired run timed it: 100 KiB random at level 3, -11.0% wall-clock,
1.18x -> 1.05x of libzstd in that run; z000033 whole and in 10 KiB frames
and the level 1 control within 1%.

Part of #653
---
 zstd/src/encoding/dfast/mod.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/zstd/src/encoding/dfast/mod.rs b/zstd/src/encoding/dfast/mod.rs
index a7d8eaea..43725c06 100644
--- a/zstd/src/encoding/dfast/mod.rs
+++ b/zstd/src/encoding/dfast/mod.rs
@@ -1979,9 +1979,9 @@ impl DfastMatchGenerator {
         // Never the gap the scan's step skipped before it stopped, nor what a
         // match ending in the tail covered: upstream's loops index neither,
         // and on incompressible input that gap is the step, hundreds of bytes
-        // by a block's end. Seeding it cost 11% of the cycles of a 100 KiB
-        // random frame at level 3 for 43 bytes over 3.1 MB of corpus at
-        // levels 3-4.
+        // by a block's end. Seeding it cost 11% of the wall-clock time of a
+        // 100 KiB random frame at level 3 (1.18x -> 1.05x of libzstd in the
+        // same run) for 43 bytes over 3.1 MB of corpus at levels 3-4.
         let mut seed_pos = pos.min(current_len).max(boundary_tail_start);
         // Every start with a whole short key; past it nothing is hashable.
         while seed_pos + self.mls <= current_len {

From 5b18e5e8f46884272502ed7fb3e08a54caf2b8e3 Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Wed, 11 Nov 2026 11:53:42 +0200
Subject: [PATCH 4/7] docs(encode): state the tail seed test's cost in time too

The test's doc comment quoted the same cost in cycles; the paired run's
wall-clock figure is the same 11%.

Part of #653
---
 zstd/src/encoding/match_generator/tests.rs | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/zstd/src/encoding/match_generator/tests.rs b/zstd/src/encoding/match_generator/tests.rs
index 1e4ad4c8..47ee4354 100644
--- a/zstd/src/encoding/match_generator/tests.rs
+++ b/zstd/src/encoding/match_generator/tests.rs
@@ -5118,7 +5118,8 @@ fn dfast_seed_remaining_hashable_starts_seeds_last_short_hash_positions() {
 /// The tail seed indexes only the dense tail past where the scan stopped:
 /// neither the gap a stepping scan skipped before the tail, nor tail starts a
 /// match ending at the block's end already covered, as upstream's loops leave
-/// both. Seeding the gap cost 11% of a 100 KiB random frame's cycles at level 3.
+/// both. Seeding the gap cost 11% of a 100 KiB random frame's wall-clock time
+/// at level 3.
 #[test]
 fn dfast_seed_remaining_hashable_starts_skips_the_gap_and_covered_starts() {
     let short_slot_of = |matcher: &DfastMatchGenerator, abs_pos: usize| {

From 3da1164a31ab65c39d1af08710df4e7045091096 Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Wed, 11 Nov 2026 12:31:20 +0200
Subject: [PATCH 5/7] perf(encode): address a tree node by its stored index

The binary tree walk addressed a node's pair as `(stored + bt_bias) &
bt_mask`, a per-block bias the loop kept live across every node, and
tested the block end on every node. Upstream does neither: the slot is
`matchIndex & btMask` and the end is tested on a new best only.

- The slot is now `stored & bt_mask`. Within one set of coordinates the
  two differ by a constant rotation of the ring, so the tree keeps its
  shape; the coordinates move only where no node is live (the tables are
  cleared, replaced with them, or every node falls below the new window
  floor), which `bt_pair_index_for_stored` documents.
- The per-node `match_len >= tail_limit` test is a `debug_assert`: the
  walk enters below the limit and a new best that reaches it breaks
  inside the new-best block, so a node that is not a new best cannot.

Output is identical at every level from -7 to 22 on z000033, z000088 and
random input. Instructions fall 0.3-0.8% on every tree cell (z000033 in
10 KiB and 1 KiB frames at levels 13-19, whole at 16), branch misses with
them; the cycle deltas sit inside the +2.0% the level 1 control moved, so
no time change is established.

Part of #653
---
 zstd/src/encoding/hc/generator.rs             | 46 ++++++-------
 zstd/src/encoding/match_table/storage.rs      | 39 +++++------
 .../storage/bt_pair_index_wrap_tests.rs       | 69 +++++++------------
 3 files changed, 61 insertions(+), 93 deletions(-)

diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs
index 36ca50be..c0f985c0 100644
--- a/zstd/src/encoding/hc/generator.rs
+++ b/zstd/src/encoding/hc/generator.rs
@@ -103,13 +103,13 @@ macro_rules! bt_insert_range_body {
         // The walk carries one coordinate, the stored index, as the collect body
         // does and as upstream carries `matchIndex`: the window bound becomes one
         // unsigned range test on it (HC_EMPTY, 0, decodes below the window and
-        // ends the walk), and the absolute position, the history index and the
-        // pair slot are each one add of a bias taken here, instead of reloading
-        // `position_base` / `index_shift` / `history_abs_start` through the
-        // table on every node. See the collect body for the derivation.
+        // ends the walk), the absolute position and the history index are each
+        // one add of a bias taken here, instead of reloading `position_base` /
+        // `index_shift` / `history_abs_start` through the table on every node,
+        // and the pair slot is the index itself, masked. See the collect body
+        // for the derivation.
         let abs_bias = coords.abs_bias;
         let idx_bias = coords.idx_bias;
-        let bt_bias = coords.bt_bias;
         // The window floor in stored space, fixed for the run: a node is in the
         // window when its stored index is at or above it, one compare (upstream
         // `matchIndex >= windowLow`). It is at least 1, so the empty slot 0
@@ -218,10 +218,9 @@ macro_rules! bt_insert_range_body {
                 // floor points at `closed_side`, so the final stores need no test
                 // (upstream `dummy32`).
                 let mut closed_side: u32 = 0;
-                // SAFETY: `2*((stored + bt_bias) & bt_mask) + 1 < chain_table.len()`
-                // (`stored + bt_bias == abs_pos + index_shift`, the pair index of
-                // `MatchTable::bt_pair_index_for_abs`).
-                let pair = unsafe { chain_ptr.add(2 * (stored.wrapping_add(bt_bias) & bt_mask)) };
+                // SAFETY: `2*(stored & bt_mask) + 1 < chain_table.len()`, the pair
+                // index of `MatchTable::bt_pair_index_for_stored`.
+                let pair = unsafe { chain_ptr.add(2 * (stored & bt_mask)) };
                 let mut smaller_slot: *mut u32 = pair;
                 let mut larger_slot: *mut u32 = unsafe { pair.add(1) };
                 // SAFETY: `hash` is masked to `hash_log` bits and the table is
@@ -244,8 +243,7 @@ macro_rules! bt_insert_range_body {
                     // SAFETY: the pair index is below `chain_table.len() - 1`;
                     // `chain_ptr` is the hoisted live base, not reallocated during
                     // the walk.
-                    let next_pair =
-                        unsafe { chain_ptr.add(2 * (node_stored.wrapping_add(bt_bias) & bt_mask)) };
+                    let next_pair = unsafe { chain_ptr.add(2 * (node_stored & bt_mask)) };
                     let seed_len = common_length_smaller.min(common_length_larger);
                     // Each side's common length is a match the walk did not stop
                     // at, so it is below the tail limit.
@@ -908,12 +906,11 @@ macro_rules! bt_insert_and_collect_matches_body {
         // A node is reached through its stored index alone, as upstream walks
         // one `matchIndex` (`match = base + matchIndex`, slot
         // `2*(matchIndex & btMask)`): its bytes sit at `cand_base + stored`
-        // and its pair at `(stored + bt_bias) & bt_mask`, and nothing decodes
-        // it to an absolute position unless it joins the ladder.
+        // and its pair at `stored & bt_mask`, and nothing decodes it to an
+        // absolute position unless it joins the ladder.
         // `cand_base` may lie outside the history; only the candidate it is
         // offset to is ever read, and the window gate proves that one inside.
         debug_assert!(win_low_stored >= 1);
-        let bt_bias = coords.bt_bias;
         // SAFETY: `idx + 8 <= concat.len()`, which the search's entry checks.
         let cur_ptr = unsafe { cbase.add(idx) };
         let cand_base = cbase.wrapping_add(coords.idx_bias);
@@ -926,10 +923,8 @@ macro_rules! bt_insert_and_collect_matches_body {
         let mut compares_left = $search_depth;
         let mut common_length_smaller = 0usize;
         let mut common_length_larger = 0usize;
-        // `bt_pair_index_for_abs(abs_pos)`: `stored + bt_bias` is
-        // `abs_pos + index_shift`.
-        let pair_idx = 2 * ((stored as usize).wrapping_add(bt_bias) & bt_mask);
-        debug_assert_eq!(pair_idx, $table.bt_pair_index_for_abs($abs_pos));
+        let pair_idx = 2 * (stored as usize & bt_mask);
+        debug_assert_eq!(pair_idx, $table.bt_pair_index_for_stored(stored as usize));
         // The two open sides of the tree, as pointers into the pair table
         // (upstream `smallerPtr` / `largerPtr`). A side the walk closes at the
         // tree floor is pointed at `closed_side` instead, so the final store
@@ -968,12 +963,10 @@ macro_rules! bt_insert_and_collect_matches_body {
                 node_stored <= bt_low_stored,
                 node_stored.wrapping_add(coords.abs_bias) <= bt_low,
             );
-            // `2*(candidate_abs + index_shift & bt_mask)` with `index_shift`
-            // folded away: `candidate_abs + index_shift == stored + bt_bias`.
             // SAFETY: the pair index is below `chain_table.len() - 1`;
             // `chain_ptr` is the hoisted live base, not reallocated during the
             // walk.
-            let next_pair = unsafe { chain_ptr.add(2 * (node_stored.wrapping_add(bt_bias) & bt_mask)) };
+            let next_pair = unsafe { chain_ptr.add(2 * (node_stored & bt_mask)) };
             // Both children are read ahead of the compare, where upstream reads
             // only the side it descends, so the next node's load can overlap
             // the compare. Reading only the chosen side after it measured about
@@ -1034,12 +1027,15 @@ macro_rules! bt_insert_and_collect_matches_body {
                 }
             }
 
-            if match_len >= tail_limit {
-                break;
-            }
+            // The walk enters with `best_len < tail_limit` (a repeat or hash3
+            // match reaching the block end skips the walk), and a new best that
+            // reaches it breaks above, so a match that is not a new best is
+            // shorter than `best_len` and below the limit. Upstream tests the
+            // block end only on a new best (`ip+matchLength == iLimit`).
+            debug_assert!(match_len < tail_limit);
 
             // SAFETY: the first differing bytes after a `match_len`-long
-            // prefix; `match_len < tail_limit` (break above) keeps both in
+            // prefix; `match_len < tail_limit` (asserted above) keeps both in
             // range.
             if unsafe { *cand_ptr.add(match_len) < *cur_ptr.add(match_len) } {
                 // SAFETY: an open side points at a pair slot (initial pair or
diff --git a/zstd/src/encoding/match_table/storage.rs b/zstd/src/encoding/match_table/storage.rs
index 2694981a..59bc9e44 100644
--- a/zstd/src/encoding/match_table/storage.rs
+++ b/zstd/src/encoding/match_table/storage.rs
@@ -153,8 +153,9 @@ pub(crate) const HC_CHAIN_LOG: usize = 19;
 pub(crate) const HC3_HASH_LOG: usize = 17;
 
 /// The binary tree's coordinates: the biases that map a stored index to its
-/// absolute position (`abs_bias`), its history index (`idx_bias`) and its pair
-/// slot (`bt_bias`, then `& bt_mask`, doubled). They follow from the position
+/// absolute position (`abs_bias`) and its history index (`idx_bias`), and the
+/// mask that maps it to its pair slot (`stored & bt_mask`, doubled; see
+/// [`MatchTable::bt_pair_index_for_stored`]). They follow from the position
 /// base, the index shift, the history start and the chain log, none of which
 /// moves while an armed block is parsed, so the parser takes them once per
 /// block, as upstream resolves `base` and `btMask` once per call.
@@ -162,7 +163,6 @@ pub(crate) const HC3_HASH_LOG: usize = 17;
 pub(crate) struct BtCoords {
     pub(crate) abs_bias: usize,
     pub(crate) idx_bias: usize,
-    pub(crate) bt_bias: usize,
     pub(crate) bt_mask: usize,
 }
 
@@ -1237,7 +1237,6 @@ impl MatchTable {
         BtCoords {
             abs_bias,
             idx_bias: abs_bias.wrapping_sub(self.history_abs_start),
-            bt_bias: self.position_base.wrapping_sub(1),
             bt_mask: self.bt_mask(),
         }
     }
@@ -1272,26 +1271,20 @@ impl MatchTable {
         unsafe { core::slice::from_raw_parts(self.block_live.0, self.block_live.1) }
     }
 
-    /// Convert an absolute position into a BT pair index in
-    /// `chain_table`. Each node occupies two consecutive slots
-    /// (smaller, larger) so the result is doubled. Upstream zstd parity:
-    /// `2 * (curr & btMask)` from `ZSTD_insertBt1`.
+    /// The pair index in `chain_table` of the node whose stored index is
+    /// `stored`: its low `bt_log` bits, doubled because each node holds two
+    /// slots (smaller, larger). Upstream zstd: `2 * (matchIndex & btMask)` in
+    /// `ZSTD_insertBt1` and `ZSTD_insertBtAndGetAllMatches`.
+    ///
+    /// The slot is keyed by the stored index alone, so it moves whenever
+    /// `position_base` or `index_shift` does. That is sound because neither
+    /// moves while the tree holds a live node: every change either clears the
+    /// tables, replaces them together with the coordinates, or (the btultra2
+    /// seed pass, the floor-advancing reset) leaves every stored node below
+    /// the new window floor, where no walk reads it.
     #[inline(always)]
-    pub(crate) fn bt_pair_index_for_abs(&self, abs_pos: usize) -> usize {
-        // Hot per-iteration BT walker entry. `abs_pos` is a
-        // frame-lifetime absolute stream cursor (capped by
-        // `check_stream_abs_headroom`); `index_shift` is the offset the
-        // btultra2 seed pass leaves behind, which a reused compressor carries
-        // into later frames until a rebase clears it. The result is immediately masked down to the BT ring width
-        // by `& bt_mask()`, so what matters here is only the modulo-
-        // ring identity `(a + b) mod m == ((a mod 2^bits) + (b mod
-        // 2^bits)) mod m` for `m | 2^bits`: `wrapping_add` preserves
-        // that identity even on the rare i686 streams where the raw
-        // `usize` sum overflows. Using `wrapping_add` instead of `+`
-        // (or `saturating_add`) also keeps the release-mode overflow
-        // branch and the debug-mode overflow panic off this hot path.
-        let bt_pos = abs_pos.wrapping_add(self.index_shift);
-        2 * (bt_pos & self.bt_mask())
+    pub(crate) fn bt_pair_index_for_stored(&self, stored: usize) -> usize {
+        2 * (stored & self.bt_mask())
     }
 
     /// Decode a stored hash / chain table entry back into its absolute
diff --git a/zstd/src/encoding/match_table/storage/bt_pair_index_wrap_tests.rs b/zstd/src/encoding/match_table/storage/bt_pair_index_wrap_tests.rs
index 6002fa36..51506ed9 100644
--- a/zstd/src/encoding/match_table/storage/bt_pair_index_wrap_tests.rs
+++ b/zstd/src/encoding/match_table/storage/bt_pair_index_wrap_tests.rs
@@ -1,55 +1,34 @@
 use super::MatchTable;
 
-/// `bt_pair_index_for_abs` switched from `+` to `wrapping_add` so the
-/// release-mode overflow branch and the debug-mode panic stay off
-/// the hot path on rare 32-bit streams where `abs_pos + index_shift`
-/// overflows `usize`. This test forces that overflow and verifies
-/// the returned BT slot still equals what the modulo-ring identity
-/// promises: `(abs_pos + index_shift) mod 2^bt_log`, doubled
-/// because the table stores pointer pairs.
+/// A node's pair sits at its stored index's low `bt_log` bits, doubled, so
+/// any `bt_mask + 1` consecutive stored indices (the tree's reach behind a
+/// position) occupy distinct pairs, whatever the coordinates. A slot formula
+/// that let two of them share a pair would overwrite a live node's children.
 #[test]
-fn bt_pair_index_matches_modulo_ring_after_wraparound() {
+fn bt_pair_index_gives_a_window_of_stored_indices_distinct_pairs() {
     let mut table = MatchTable::new(1 << 20);
-    // Small BT ring (`bt_log = chain_log - 1 = 3` → mask = 0b0111) so
-    // the modular identity is easy to read at a glance.
+    // `bt_log = chain_log - 1 = 3`, mask 0b0111: eight pairs.
     table.chain_log = 4;
-    // Use a wide `index_shift` so the addition wraps for many of
-    // the `abs_pos` values we probe below.
-    table.index_shift = usize::MAX - 5;
-
+    // Coordinates that must not enter the slot.
+    table.position_base = 12_345;
+    table.index_shift = 678;
     let bt_mask = table.bt_mask();
     assert_eq!(bt_mask, 0b0111);
 
-    for abs_pos in [0usize, 1, 4, 5, 6, 7, 8, 12, 17] {
-        let got = table.bt_pair_index_for_abs(abs_pos);
-        let expected = 2 * (abs_pos.wrapping_add(table.index_shift) & bt_mask);
-        assert_eq!(
-            got, expected,
-            "abs_pos={abs_pos}: wrapping_add ring slot must match the masked sum"
-        );
-    }
-
-    // Spot-check one value where overflow is certain (abs_pos > 5)
-    // and the ring index has a stable closed form.
-    // abs_pos=7, index_shift=usize::MAX-5: sum wraps to 1, mask -> 1, doubled -> 2.
-    assert_eq!(table.bt_pair_index_for_abs(7), 2);
-    // abs_pos=14: sum wraps to 8, masked -> 0, doubled -> 0.
-    assert_eq!(table.bt_pair_index_for_abs(14), 0);
-}
-
-/// Sanity check the non-overflow path keeps the same identity so a
-/// future refactor cannot regress the common case while leaving the
-/// overflow case green.
-#[test]
-fn bt_pair_index_matches_modulo_ring_without_overflow() {
-    let mut table = MatchTable::new(1 << 20);
-    table.chain_log = 8; // bt_log = 7, mask = 0x7f
-    table.index_shift = 17;
-
-    let bt_mask = table.bt_mask();
-    for abs_pos in [0usize, 1, 16, 32, 64, 127, 128, 255, 1 << 20] {
-        let got = table.bt_pair_index_for_abs(abs_pos);
-        let expected = 2 * ((abs_pos + table.index_shift) & bt_mask);
-        assert_eq!(got, expected, "abs_pos={abs_pos}");
+    for first in [1usize, 5, 8, 1 << 20, u32::MAX as usize - 8] {
+        let mut seen = [false; 8];
+        for stored in first..=first + bt_mask {
+            let pair = table.bt_pair_index_for_stored(stored);
+            assert_eq!(pair, 2 * (stored & bt_mask), "stored={stored}");
+            assert!(
+                pair + 1 < 2 << table.bt_log(),
+                "stored={stored}: pair in range"
+            );
+            assert!(
+                !seen[pair / 2],
+                "stored={stored}: pair already taken in this window"
+            );
+            seen[pair / 2] = true;
+        }
     }
 }

From 3456ebbea0cd26ec8f4c5a9df3479caa2050df3e Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Wed, 11 Nov 2026 12:46:56 +0200
Subject: [PATCH 6/7] perf(encode): read only the descended child in the tree
 search

The search body loaded both children of each node right after the
prefix count and kept them live through the compare and the new-best
block. Upstream reads the side it descends after the compare
(`matchIndex = nextPtr[1]`), as our insert body already does; the search
now does the same, reading the same slot later, so output is unchanged.

z000033 in small frames, five interleaved rounds: instructions -1.2 to
-1.3% on every tree cell; cycles -3.9% at level 19 in 10 KiB frames and
-3.0% at level 16 in 1 KiB frames, where the rounds of the two builds do
not overlap, against a level 1 control that moved -2.0% with identical
instructions. Levels 16 and 17 in 10 KiB frames -3.1% / -3.5% and level
13 +1.0% sit in overlapping rounds. Level 17 and 19 frames now encode
faster than libzstd in the same run (0.94x, 0.93x).

Part of #653
---
 zstd/src/encoding/hc/generator.rs | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/zstd/src/encoding/hc/generator.rs b/zstd/src/encoding/hc/generator.rs
index c0f985c0..321a37fa 100644
--- a/zstd/src/encoding/hc/generator.rs
+++ b/zstd/src/encoding/hc/generator.rs
@@ -967,16 +967,6 @@ macro_rules! bt_insert_and_collect_matches_body {
             // `chain_ptr` is the hoisted live base, not reallocated during the
             // walk.
             let next_pair = unsafe { chain_ptr.add(2 * (node_stored & bt_mask)) };
-            // Both children are read ahead of the compare, where upstream reads
-            // only the side it descends, so the next node's load can overlap
-            // the compare. Reading only the chosen side after it measured about
-            // 1% slower on z000033 10 KiB at level 19 with output unchanged,
-            // against a flat level 3 control; that is below what two builds
-            // resolve, so neither order is shown faster and the walk keeps its
-            // existing one.
-            // SAFETY: as for `next_pair`.
-            let next_smaller = unsafe { *next_pair };
-            let next_larger = unsafe { *next_pair.add(1) };
             let seed_len = common_length_smaller.min(common_length_larger);
             // Every side's common length is a match the walk did not stop at,
             // so it is below the tail limit and the compare below has bytes
@@ -1046,9 +1036,11 @@ macro_rules! bt_insert_and_collect_matches_body {
                     smaller_slot = &raw mut closed_side;
                     break;
                 }
+                // Only the side descended is read, after the compare, as
+                // upstream does (`matchIndex = nextPtr[1]`).
                 // SAFETY: as for `next_pair`.
                 smaller_slot = unsafe { next_pair.add(1) };
-                match_stored = next_larger;
+                match_stored = unsafe { *smaller_slot };
             } else {
                 // SAFETY: as above for `larger_slot`.
                 unsafe { *larger_slot = match_stored };
@@ -1058,7 +1050,8 @@ macro_rules! bt_insert_and_collect_matches_body {
                     break;
                 }
                 larger_slot = next_pair;
-                match_stored = next_smaller;
+                // SAFETY: as for `next_pair`.
+                match_stored = unsafe { *larger_slot };
             }
         }
 

From 6d28db0e7c626114ad40eb970191f6ab0eb0369e Mon Sep 17 00:00:00 2001
From: Dmitry Prudnikov <mail@polaz.com>
Date: Wed, 11 Nov 2026 15:16:50 +0200
Subject: [PATCH 7/7] perf(encode): select the parser's repeat code without
 branching

The optimal parser asks, for every candidate the search returns, which
repeat code its offset takes against the position's history. It did so
with a compare chain over three repeats, an outcome the predictor cannot
learn: the last-branch record put about 110 mispredicts per 300 frames
there at level 16 over the two btultra2 passes, the parser's whole
excess over libzstd's opt2, which receives offBase ready-coded.

`encode_offset_base_with_reps` now picks the three repeat values by the
literal length and folds them with `select_unpredictable`, last to first
so the first match still wins. A test checks it against the compare
chain over every combination of nine edge values.

z000033, five interleaved rounds against the parent: branch misses
-4.2 to -6.8% on every tree cell (10 KiB frames at levels 13-19, 1 KiB
at 16, whole at 16), instructions -0.1 to -0.4%; time -1.5 to -3.7% in
10 KiB frames against a level 1 control that moved +2.7% the other way,
inside that host's spread, so no time change is established.

Part of #653
---
 zstd/src/encoding/bt/mod.rs   | 39 ++++++++++++++---------------
 zstd/src/encoding/bt/tests.rs | 46 +++++++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+), 19 deletions(-)

diff --git a/zstd/src/encoding/bt/mod.rs b/zstd/src/encoding/bt/mod.rs
index cc7a5b2e..fe1bba68 100644
--- a/zstd/src/encoding/bt/mod.rs
+++ b/zstd/src/encoding/bt/mod.rs
@@ -431,31 +431,32 @@ impl BtMatcher {
     /// `encode_offset_with_reps` minus the rep-history update — used in
     /// the optimal parser's per-candidate price probe where the rep
     /// window hasn't been committed yet.
+    ///
+    /// Selects rather than branches: it runs for every candidate the search
+    /// returns, and whether an offset repeats one of three histories is data
+    /// the predictor cannot learn, so a compare chain mispredicted on a large
+    /// share of candidates. The three repeat values are picked by `lit_len`
+    /// (RFC 8878 3.1.2.5: with no literals the codes shift to `rep[1]`,
+    /// `rep[2]`, `rep[0] - 1`), then folded last to first so the first match
+    /// wins, as the chain did.
     #[inline(always)]
     pub(crate) fn encode_offset_base_with_reps(
         actual_offset: u32,
         lit_len: usize,
         reps: [u32; 3],
     ) -> u32 {
-        if lit_len > 0 {
-            if actual_offset == reps[0] {
-                1
-            } else if actual_offset == reps[1] {
-                2
-            } else if actual_offset == reps[2] {
-                3
-            } else {
-                actual_offset.saturating_add(3)
-            }
-        } else if actual_offset == reps[1] {
-            1
-        } else if actual_offset == reps[2] {
-            2
-        } else if reps[0] > 1 && actual_offset == reps[0] - 1 {
-            3
-        } else {
-            actual_offset.saturating_add(3)
-        }
+        use core::hint::select_unpredictable;
+        let ll0 = lit_len == 0;
+        // `rep[0] - 1` is a repeat code only when it is a real offset; 0
+        // never equals a match offset, which is at least 1.
+        let rep0_minus_one = select_unpredictable(reps[0] > 1, reps[0].wrapping_sub(1), 0);
+        let first = select_unpredictable(ll0, reps[1], reps[0]);
+        let second = select_unpredictable(ll0, reps[2], reps[1]);
+        let third = select_unpredictable(ll0, rep0_minus_one, reps[2]);
+        let code = actual_offset.saturating_add(3);
+        let code = select_unpredictable(actual_offset == third, 3, code);
+        let code = select_unpredictable(actual_offset == second, 2, code);
+        select_unpredictable(actual_offset == first, 1, code)
     }
 
     /// Upstream zstd parity: `ZSTD_updateStats` for one sequence the parser
diff --git a/zstd/src/encoding/bt/tests.rs b/zstd/src/encoding/bt/tests.rs
index 053117ee..c5bef4fd 100644
--- a/zstd/src/encoding/bt/tests.rs
+++ b/zstd/src/encoding/bt/tests.rs
@@ -23,3 +23,49 @@ fn estimated_workspace_bytes_matches_retained_scratch_at_growth_bounds() {
         core::mem::size_of::<BtMatcher>() + bt.heap_size()
     );
 }
+
+/// The repeat-code probe selects instead of branching; it must return what
+/// the RFC 8878 3.1.2.5 compare chain returns for every offset and history,
+/// including colliding repeats (the first match wins), `rep[0] <= 1` with no
+/// literals (no `rep[0] - 1` code), and an offset at the saturation edge.
+#[test]
+fn offset_base_probe_matches_the_repeat_compare_chain() {
+    fn chain(offset: u32, lit_len: usize, reps: [u32; 3]) -> u32 {
+        if lit_len > 0 {
+            if offset == reps[0] {
+                1
+            } else if offset == reps[1] {
+                2
+            } else if offset == reps[2] {
+                3
+            } else {
+                offset.saturating_add(3)
+            }
+        } else if offset == reps[1] {
+            1
+        } else if offset == reps[2] {
+            2
+        } else if reps[0] > 1 && offset == reps[0] - 1 {
+            3
+        } else {
+            offset.saturating_add(3)
+        }
+    }
+    let values = [0u32, 1, 2, 3, 4, 5, u32::MAX - 3, u32::MAX - 2, u32::MAX];
+    for &r0 in &values {
+        for &r1 in &values {
+            for &r2 in &values {
+                for &offset in &values {
+                    for lit_len in [0usize, 1, 7] {
+                        let reps = [r0, r1, r2];
+                        assert_eq!(
+                            BtMatcher::encode_offset_base_with_reps(offset, lit_len, reps),
+                            chain(offset, lit_len, reps),
+                            "offset={offset} lit_len={lit_len} reps={reps:?}"
+                        );
+                    }
+                }
+            }
+        }
+    }
+}