From c8140c9ab9edab01702b1091302b54fc03539d41 Mon Sep 17 00:00:00 2001 From: haochengxia Date: Sat, 7 Nov 2026 14:42:24 -0600 Subject: [PATCH 1/3] perf: FlatIdSet for the simulator's per-request block sets The pin predicate, alpha_is_pinned, runs for every pinned object that cache_skip_pinned walks past on an eviction: on freeinference, 6.0e9 calls with LRU at 64k blocks (9 per block accessed) and 2.2e11 at 16k. Each was an std::unordered_set::count, a 64-bit division and a node chase (about 6.5 ns), and the request's sets were rebuilt node by node (about 114 ns per insert). alpha_ and missing_ become a FlatIdSet: open addressing with a multiplicative hash, cleared in O(1) by bumping an epoch, so one instance serves every request without allocating (about 1 ns per lookup, 9 ns per insert). alpha_resident_ is built only on the path that reads it, for algorithms that do not report their victims. Results are unchanged: the RESULT lines are identical. Run time drops by 12% to 41% (7% to 29% when built with tcmalloc) across LRU, FIFO, CLOCK, ARC, LFU, S3-FIFO and RandomCompute (Xeon Gold 6142). Co-Authored-By: Claude Opus 5.5 --- src/flat_id_set.hpp | 67 +++++++++++++++++++++++++++++++++++++++++++++ src/simulator.cpp | 24 +++++++++------- src/simulator.hpp | 4 ++- 3 files changed, 84 insertions(+), 11 deletions(-) create mode 100644 src/flat_id_set.hpp diff --git a/src/flat_id_set.hpp b/src/flat_id_set.hpp new file mode 100644 index 0000000..d728de4 --- /dev/null +++ b/src/flat_id_set.hpp @@ -0,0 +1,67 @@ +// A set of block ids for the Simulator's per-request scratch sets. Open +// addressing, cleared in O(1) by bumping an epoch, so one instance serves every +// request without allocating. count() is a multiply and an array read: the pin +// predicate calls it for every pinned object cache_skip_pinned walks past (~9 +// per block accessed, LRU at 64k blocks), which made std::unordered_set's modulo +// and node chase the hot spot. Iterates in insertion order. + +#pragma once + +#include +#include +#include + +namespace prefixsim { + +class FlatIdSet { + public: + void clear() { + ids_.clear(); + if (++epoch_ == 0) { // wrapped: old stamps would alias the new epoch + std::fill(stamps_.begin(), stamps_.end(), 0u); + epoch_ = 1; + } + } + bool insert(uint64_t id) { + if (2 * (ids_.size() + 1) > stamps_.size()) grow(); + size_t h = slot(id); + for (; stamps_[h] == epoch_; h = (h + 1) & mask_) { + if (keys_[h] == id) return false; + } + stamps_[h] = epoch_; + keys_[h] = id; + ids_.push_back(id); + return true; + } + size_t count(uint64_t id) const { + if (stamps_.empty()) return 0; + for (size_t h = slot(id); stamps_[h] == epoch_; h = (h + 1) & mask_) { + if (keys_[h] == id) return 1; + } + return 0; + } + size_t size() const { return ids_.size(); } + std::vector::const_iterator begin() const { return ids_.begin(); } + std::vector::const_iterator end() const { return ids_.end(); } + + private: + size_t slot(uint64_t id) const { return (id * 0x9E3779B97F4A7C15ull >> 32) & mask_; } + void grow() { // to twice the size, keeping the current ids + const size_t cap = std::max(64, 2 * stamps_.size()); + keys_.assign(cap, 0); + stamps_.assign(cap, 0); + mask_ = cap - 1; + epoch_ = 1; + std::vector ids; + ids.swap(ids_); + for (const uint64_t id : ids) insert(id); + } + + std::vector keys_; + std::vector stamps_; + std::vector ids_; + uint32_t epoch_ = 1; + size_t mask_ = 0; +}; + +} // namespace prefixsim diff --git a/src/simulator.cpp b/src/simulator.cpp index 342aa97..71b43ac 100644 --- a/src/simulator.cpp +++ b/src/simulator.cpp @@ -153,7 +153,7 @@ const AlgoEntry kAlgos[] = { /// the arriving request's block set (Simulator::alpha_). extern "C" because it /// is stored in a C function pointer declared inside libCacheSim's extern "C". extern "C" bool alpha_is_pinned(void *ctx, obj_id_t id) { - return static_cast *>(ctx)->count(id) != 0; + return static_cast(ctx)->count(id) != 0; } } // namespace @@ -303,7 +303,7 @@ bool Simulator::serve(const Request &request, std::string &error) { // Every statistic this simulator reports is computed here, against the cache // exactly as the request found it. Nothing in phases 2 and 3 can change it. resident_.assign(n_pos, 0); - std::unordered_set missing; + missing_.clear(); for (size_t i = 0; i < n_pos; ++i) { const int64_t cost = config_.cost_model.cost(static_cast(i)); @@ -324,7 +324,7 @@ bool Simulator::serve(const Request &request, std::string &error) { ++stats_.n_block_hits; stats_.saved_cost += cost; } else { - missing.insert(request.blocks[i]); + missing_.insert(request.blocks[i]); } } @@ -333,7 +333,7 @@ bool Simulator::serve(const Request &request, std::string &error) { // ---------------- phase 2: allocate ---------------- const int64_t occupied = cache_->get_occupied_byte(cache_); const int64_t free_slots = config_.cache_size_blocks - occupied; - const int64_t needed = static_cast(missing.size()) - free_slots; + const int64_t needed = static_cast(missing_.size()) - free_slots; if (needed > 0) { // Pins cover phase 2 only. Phase 3 must not evict at all (phase 2 reserved // exactly enough room, and n_unexpected_evictions asserts it), so leaving @@ -398,12 +398,9 @@ bool Simulator::allocate(const Request &request, int64_t needed, int64_t unobserved_freed = 0; // Which of Alpha's blocks are resident right now. Only consulted on the - // no-victim-reported path, so building it costs nothing for the algorithms - // that do report their victims. - alpha_resident_.clear(); - for (size_t i = 0; i < request.blocks.size(); ++i) { - if (resident_[i] != 0) alpha_resident_.insert(request.blocks[i]); - } + // no-victim-reported path, so it is built there, on first use, and costs + // nothing for the algorithms that do report their victims. + bool alpha_resident_built = false; // Termination: the request needs at most `cache_size` distinct blocks (larger // ones were skipped), so the cache always holds at least `needed` blocks the @@ -477,6 +474,13 @@ bool Simulator::allocate(const Request &request, int64_t needed, unobserved_freed += occupied_before - occupied_now; if (unobserved_freed < needed) continue; + if (!alpha_resident_built) { + alpha_resident_.clear(); + for (size_t i = 0; i < request.blocks.size(); ++i) { + if (resident_[i] != 0) alpha_resident_.insert(request.blocks[i]); + } + alpha_resident_built = true; + } int64_t missing = 0; for (const obj_id_t id : alpha_) { if (probe(id)) continue; diff --git a/src/simulator.hpp b/src/simulator.hpp index fdac343..a8798e5 100644 --- a/src/simulator.hpp +++ b/src/simulator.hpp @@ -20,6 +20,7 @@ #include #include "cost_model.hpp" +#include "flat_id_set.hpp" #include "trace.hpp" extern "C" { @@ -158,7 +159,8 @@ class Simulator { // Scratch reused across requests so the hot loop does not allocate. std::vector resident_; - std::unordered_set alpha_; + FlatIdSet alpha_; + FlatIdSet missing_; std::unordered_set alpha_resident_; std::vector victims_; }; From d923c8a3806f6faf1daf8acbcd7c2c9c69b4836b Mon Sep 17 00:00:00 2001 From: Yiyu Liu Date: Mon, 9 Nov 2026 16:29:44 -0500 Subject: [PATCH 2/3] style: clang-format flat_id_set.hpp --- src/flat_id_set.hpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/flat_id_set.hpp b/src/flat_id_set.hpp index d728de4..514753a 100644 --- a/src/flat_id_set.hpp +++ b/src/flat_id_set.hpp @@ -2,8 +2,8 @@ // addressing, cleared in O(1) by bumping an epoch, so one instance serves every // request without allocating. count() is a multiply and an array read: the pin // predicate calls it for every pinned object cache_skip_pinned walks past (~9 -// per block accessed, LRU at 64k blocks), which made std::unordered_set's modulo -// and node chase the hot spot. Iterates in insertion order. +// per block accessed, LRU at 64k blocks), which made std::unordered_set's +// modulo and node chase the hot spot. Iterates in insertion order. #pragma once @@ -45,7 +45,9 @@ class FlatIdSet { std::vector::const_iterator end() const { return ids_.end(); } private: - size_t slot(uint64_t id) const { return (id * 0x9E3779B97F4A7C15ull >> 32) & mask_; } + size_t slot(uint64_t id) const { + return (id * 0x9E3779B97F4A7C15ull >> 32) & mask_; + } void grow() { // to twice the size, keeping the current ids const size_t cap = std::max(64, 2 * stamps_.size()); keys_.assign(cap, 0); From ab9f5bde1cfc70a65657312b07a5dd90980321c8 Mon Sep 17 00:00:00 2001 From: Yiyu Liu Date: Mon, 9 Nov 2026 18:02:53 -0500 Subject: [PATCH 3/3] perf: hash block ids in groups of 8 so consecutive ids share a cache line --- src/flat_id_set.hpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/flat_id_set.hpp b/src/flat_id_set.hpp index 514753a..75285ff 100644 --- a/src/flat_id_set.hpp +++ b/src/flat_id_set.hpp @@ -45,8 +45,20 @@ class FlatIdSet { std::vector::const_iterator end() const { return ids_.end(); } private: + // Hashes ids in groups of 8: the group picks a run of 8 slots and the low 3 + // bits the slot within it. lcp ids are dense and given out in first-access + // order, so a request's blocks mostly have consecutive ids, and the pin + // predicate is asked about them run after run as cache_skip_pinned walks the + // same pinned blocks for every eviction. Keeping neighbours in one cache line + // is what made std::unordered_set (id % prime) fast there. Hashing each id + // on its own scattered them: LRU at 16k blocks on the first 100k + // freeinference requests took 12% longer than with std::unordered_set on an + // AMD Zen 4, and hashing groups takes 8% less. The groups themselves are + // still scattered, so ids that are not consecutive (raw hashes) hash as well + // as before. size_t slot(uint64_t id) const { - return (id * 0x9E3779B97F4A7C15ull >> 32) & mask_; + const uint64_t group = (id >> 3) * 0x9E3779B97F4A7C15ull >> 32; + return ((group << 3) | (id & 7)) & mask_; } void grow() { // to twice the size, keeping the current ids const size_t cap = std::max(64, 2 * stamps_.size());