From 8f63d6876842050704359a458ff9f9f117f588a5 Mon Sep 17 00:00:00 2001 From: Yiyu Liu Date: Mon, 9 Nov 2026 23:45:07 -0500 Subject: [PATCH 1/3] feat: report both ratios every --report-interval seconds of trace time, as cachesim does --- src/main.cpp | 43 +++++++++++++++++++++++++++++++++++++++++ src/simulator.cpp | 33 +++++++++++++++++++++++++++++++ src/simulator.hpp | 49 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 125 insertions(+) diff --git a/src/main.cpp b/src/main.cpp index 5e396bc..026a411 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -29,6 +29,7 @@ struct Options { std::string dump_holes_dir; std::vector algorithms{"lru"}; int64_t cache_size = 0; + int64_t report_interval = 0; ///< seconds of trace time; 0 = off prefixsim::TraceFormat format = prefixsim::TraceFormat::kQwenJsonl; bool format_given = false; ///< else guessed from the file name prefixsim::CostModel cost_model; // uniform @@ -106,6 +107,10 @@ void print_usage(const char *program) { printf( " its recompute holes (runs of missed " "blocks).\n"); + printf( + " --report-interval Every seconds of trace time, print the\n" + " ratios so far and over the last interval,\n" + " as cachesim does (default: 0, off).\n"); printf( " --output Also append the RESULT lines to this file.\n"); printf(" --list-algos Print supported algorithms and exit.\n"); @@ -214,6 +219,15 @@ bool parse_args(int argc, char **argv, Options &opts, bool &should_exit) { opts.dump_holes_dir = value; } else if (strcmp(arg, "--output") == 0) { opts.output_path = value; + } else if (strcmp(arg, "--report-interval") == 0) { + char *end = nullptr; + errno = 0; + const long long seconds = strtoll(value, &end, 10); + if (end == value || *end != '\0' || seconds < 0 || errno != 0) { + fprintf(stderr, "error: bad --report-interval '%s' (seconds)\n", value); + return false; + } + opts.report_interval = static_cast(seconds); } else if (strcmp(arg, "--algo") == 0) { opts.algorithms = split_commas(value); } else if (strcmp(arg, "--cache-size") == 0) { @@ -288,6 +302,28 @@ std::string holes_path_for(const Options &opts, const std::string &algorithm) { std::to_string(opts.cache_size) + "_" + algorithm + ".txt"; } +std::string base_name(const std::string &path) { + const size_t slash = path.find_last_of('/'); + return slash == std::string::npos ? path : path.substr(slash + 1); +} + +/// One interval report, worded like cachesim's "interval miss ratio" line so +/// that scripts reading one read the other with a changed regex. +void print_interval(const std::string &trace, const std::string &algorithm, + const prefixsim::IntervalReport &report) { + printf( + "%s %s %.2f hour: %lld requests, block hit ratio %.4f, interval block " + "hit ratio %.4f, compute saving ratio %.4f, interval compute saving " + "ratio %.4f\n", + trace.c_str(), algorithm.c_str(), report.time_sec / 3600, + static_cast(report.total.n_requests), + report.total.block_hit_ratio(), report.interval.block_hit_ratio(), + report.total.compute_saving_ratio(), + report.interval.compute_saving_ratio()); + // Shown as the run goes, even when stdout is a pipe or a file. + fflush(stdout); +} + /// Fail before the trace is read rather than after. Loading a multi-GB trace /// takes real time, and discovering a missing output directory on the far side /// of it wastes the whole run. @@ -446,6 +482,13 @@ int main(int argc, char **argv) { } config.replay_deepest_first = opts.replay_deepest_first; config.dump_holes_path = holes_path; + if (opts.report_interval > 0) { + config.report_interval_sec = opts.report_interval; + config.on_report = [trace = base_name(opts.trace_path), + algorithm](const prefixsim::IntervalReport &report) { + print_interval(trace, algorithm, report); + }; + } // A stream is consumed by one algorithm; the next one reopens the file. if (streaming && reader == nullptr) { diff --git a/src/simulator.cpp b/src/simulator.cpp index 71b43ac..50ce662 100644 --- a/src/simulator.cpp +++ b/src/simulator.cpp @@ -228,6 +228,7 @@ bool Simulator::run(const std::vector &requests, std::string &error) { for (const Request &request : requests) { if (!serve(request, error)) return false; } + report_rest(); return true; } @@ -251,12 +252,19 @@ bool Simulator::run(TraceReader &reader, std::string &error) { while (reader.next(request)) { if (!serve(request, error)) return false; } + report_rest(); return true; } bool Simulator::serve(const Request &request, std::string &error) { const size_t n_pos = request.blocks.size(); + // Trace time starts at the first request, served or not, as in cachesim. + if (!clock_started_) { + first_timestamp_ = request.timestamp; + clock_started_ = true; + } + alpha_.clear(); for (const obj_id_t id : request.blocks) alpha_.insert(id); @@ -375,6 +383,14 @@ bool Simulator::serve(const Request &request, std::string &error) { } } } + + if (config_.report_interval_sec > 0) { + last_served_time_ = request.timestamp - first_timestamp_; + if (last_served_time_ - last_report_time_ >= + static_cast(config_.report_interval_sec)) { + report_interval(last_served_time_); + } + } return true; } @@ -545,6 +561,23 @@ void Simulator::dump_holes(const Request &request) { fprintf(holes_file_, "\n"); } +void Simulator::report_interval(double time_sec) { + IntervalReport report; + report.time_sec = time_sec; + report.total = stats_; + report.interval = stats_.since(last_report_stats_); + last_report_time_ = time_sec; + last_report_stats_ = stats_; + if (config_.on_report) config_.on_report(report); +} + +void Simulator::report_rest() { + if (config_.report_interval_sec > 0 && + stats_.n_requests > last_report_stats_.n_requests) { + report_interval(last_served_time_); + } +} + void Simulator::access(const Request &request) { // Reverse order: the deepest (most private) block is touched first and the // prefix root last, so recency-ordered algorithms leave the root at the MRU diff --git a/src/simulator.hpp b/src/simulator.hpp index a8798e5..ee93bbc 100644 --- a/src/simulator.hpp +++ b/src/simulator.hpp @@ -15,6 +15,7 @@ #include #include +#include #include #include #include @@ -66,6 +67,33 @@ struct Stats { double compute_saving_ratio() const { return total_cost > 0.0 ? saved_cost / total_cost : 0.0; } + + /// What the run did between `earlier`, a snapshot of the same run, and now. + Stats since(const Stats &earlier) const { + Stats d; + d.n_requests = n_requests - earlier.n_requests; + d.n_requests_skipped = n_requests_skipped - earlier.n_requests_skipped; + d.n_blocks = n_blocks - earlier.n_blocks; + d.n_block_hits = n_block_hits - earlier.n_block_hits; + d.total_cost = total_cost - earlier.total_cost; + d.saved_cost = saved_cost - earlier.saved_cost; + d.n_evictions = n_evictions - earlier.n_evictions; + d.n_self_evictions = n_self_evictions - earlier.n_self_evictions; + d.n_unexpected_evictions = + n_unexpected_evictions - earlier.n_unexpected_evictions; + d.n_unobserved_eviction_rounds = + n_unobserved_eviction_rounds - earlier.n_unobserved_eviction_rounds; + return d; + } +}; + +/// One interval report; see SimulatorConfig::report_interval_sec. +struct IntervalReport { + /// Trace time of the request that closed the interval, in seconds since the + /// trace's first request. + double time_sec = 0.0; + Stats total; ///< The run so far. + Stats interval; ///< Since the previous report. }; struct SimulatorConfig { @@ -99,6 +127,14 @@ struct SimulatorConfig { /// holes -- the contiguous runs of blocks it had to recompute. Empty /// disables it. See Simulator::dump_holes. std::string dump_holes_path; + /// Trace time, in seconds, between interval reports to `on_report`; 0 + /// disables them. The rule is cachesim's --report-interval: time is a + /// request's `timestamp` less the trace's first, and a served request that + /// lands at least this long after the previous report closes an interval + /// that includes it. What follows the last such request is reported when the + /// trace ends, which cachesim does not do. Skipped requests close nothing. + int64_t report_interval_sec = 0; + std::function on_report; }; /// Whether the phase-2 pin predicate (SimulatorConfig::pin_request_blocks) is @@ -149,10 +185,23 @@ class Simulator { /// Write this request's holes, one line per request (see dump_holes()). void dump_holes(const Request &request); + /// Close an interval at trace time `time_sec` (see report_interval_sec). + void report_interval(double time_sec); + + /// Report what the last interval left unreported, at the end of a run. + void report_rest(); + cache_t *cache_ = nullptr; SimulatorConfig config_; Stats stats_; + // Interval reports, in seconds since the first request's timestamp. + bool clock_started_ = false; + double first_timestamp_ = 0.0; + double last_served_time_ = 0.0; + double last_report_time_ = 0.0; + Stats last_report_stats_; + request_t *req_buf_ = nullptr; prefetcher_t *recorder_ = nullptr; FILE *holes_file_ = nullptr; From 7e4b39dc6c26a42e6f78050a57ad5bed4be4f4ea Mon Sep 17 00:00:00 2001 From: Yiyu Liu Date: Mon, 9 Nov 2026 23:45:07 -0500 Subject: [PATCH 2/3] test: check that interval reports add up to the final result --- test/test_prefixsim.cpp | 127 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 125 insertions(+), 2 deletions(-) diff --git a/test/test_prefixsim.cpp b/test/test_prefixsim.cpp index c49b64c..9850b94 100644 --- a/test/test_prefixsim.cpp +++ b/test/test_prefixsim.cpp @@ -1,8 +1,9 @@ // Sanity checks for the prefixsim three-phase serve loop, on LRU; a check that // lhd_compute reduces to lhd under the uniform cost model; that a skipped // request does not mislead the oracles; that every algorithm's read-only lookup -// really is read-only; and that the cost / recency policies stamp a request's -// blocks with one last-access time. +// really is read-only; that the cost / recency policies stamp a request's +// blocks with one last-access time; and that interval reports add up to the +// final result. // // Each case is a hand-traceable trace where the correct number of block hits // can be derived on paper. Requests are built directly from block ids rather @@ -430,6 +431,124 @@ bool request_blocks_share_one_stamp() { return ok; } +/// Run LRU over `requests`, collecting interval reports every `interval_sec`. +prefixsim::Stats run_with_reports( + const std::vector &requests, int64_t cache_size, + int64_t interval_sec, std::vector &reports) { + cache_t *cache = prefixsim::create_cache_by_name("lru", cache_size); + prefixsim::SimulatorConfig config; + config.cache_size_blocks = cache_size; + std::string error; + prefixsim::CostModel::parse("position", config.cost_model, error); + config.report_interval_sec = interval_sec; + config.on_report = [&reports](const prefixsim::IntervalReport &report) { + reports.push_back(report); + }; + prefixsim::Stats stats; + { + prefixsim::Simulator sim(cache, config); + if (!sim.run(requests, error)) printf(" FAIL: %s\n", error.c_str()); + stats = sim.stats(); + } + cache->cache_free(cache); + return stats; +} + +/// Interval reports follow cachesim's rule, add up to the final result, and +/// change nothing about it. +/// +/// Request i arrives at second i, and request 5 is larger than the cache, so +/// it is skipped. Every 3 s: request 3 closes [0..3], request 6 closes [4..6] +/// (with the skip, which closes nothing itself), request 9 closes [7..9], and +/// request 10 is left over for the report at the end of the trace. +bool interval_reports_add_up() { + const std::vector> raw = { + {1, 2, 3}, {1, 2, 4}, {5, 6}, + {1, 2, 3}, {7, 8, 9}, {10, 11, 12, 13, 14}, + {1, 2, 3}, {7, 8}, {1, 2, 4}, + {5, 6, 15}, {1, 2}}; + const std::vector requests = build(raw); + const int64_t cache_size = 4; + const std::vector expected_time = {3, 6, 9, 10}; + const std::vector expected_requests = {4, 2, 3, 1}; + const std::vector expected_skipped = {0, 1, 0, 0}; + + bool ok = true; + std::vector reports; + const prefixsim::Stats stats = + run_with_reports(requests, cache_size, 3, reports); + if (reports.size() != expected_time.size()) { + printf(" FAIL: %zu reports, expected %zu\n", reports.size(), + expected_time.size()); + return false; + } + + prefixsim::Stats sum; + for (size_t i = 0; i < reports.size(); ++i) { + const prefixsim::IntervalReport &r = reports[i]; + printf( + " %.0f s: %lld requests (%lld skipped), interval hit ratio %.4f, " + "compute saving %.4f\n", + r.time_sec, static_cast(r.interval.n_requests), + static_cast(r.interval.n_requests_skipped), + r.interval.block_hit_ratio(), r.interval.compute_saving_ratio()); + if (r.time_sec != expected_time[i] || + r.interval.n_requests != expected_requests[i] || + r.interval.n_requests_skipped != expected_skipped[i]) { + printf( + " FAIL: report %zu at %.0f s with %lld requests (%lld skipped), " + "expected %.0f s with %lld (%lld)\n", + i, r.time_sec, static_cast(r.interval.n_requests), + static_cast(r.interval.n_requests_skipped), + expected_time[i], static_cast(expected_requests[i]), + static_cast(expected_skipped[i])); + ok = false; + } + sum.n_requests += r.interval.n_requests; + sum.n_requests_skipped += r.interval.n_requests_skipped; + sum.n_blocks += r.interval.n_blocks; + sum.n_block_hits += r.interval.n_block_hits; + sum.total_cost += r.interval.total_cost; + sum.saved_cost += r.interval.saved_cost; + sum.n_evictions += r.interval.n_evictions; + } + + // Costs are small integers, so the sums are exact. + auto is_final = [&stats](const prefixsim::Stats &s) { + return s.n_requests == stats.n_requests && + s.n_requests_skipped == stats.n_requests_skipped && + s.n_blocks == stats.n_blocks && + s.n_block_hits == stats.n_block_hits && + s.total_cost == stats.total_cost && + s.saved_cost == stats.saved_cost && + s.n_evictions == stats.n_evictions; + }; + if (!is_final(sum)) { + printf(" FAIL: the intervals do not add up to the final result\n"); + ok = false; + } + if (!is_final(reports.back().total)) { + printf(" FAIL: the last report's totals are not the final result\n"); + ok = false; + } + + // Off by default, and the result is the same either way. + std::vector none; + const prefixsim::Stats plain = + run_with_reports(requests, cache_size, 0, none); + if (!none.empty()) { + printf(" FAIL: %zu reports with reporting off\n", none.size()); + ok = false; + } + if (plain.n_block_hits != stats.n_block_hits || + plain.saved_cost != stats.saved_cost || + plain.n_evictions != stats.n_evictions) { + printf(" FAIL: reporting changed the result\n"); + ok = false; + } + return ok; +} + } // namespace int main(void) { @@ -499,6 +618,10 @@ int main(void) { ++n_run; if (!request_blocks_share_one_stamp()) ++n_failed; + printf("interval reports\n"); + ++n_run; + if (!interval_reports_add_up()) ++n_failed; + if (n_failed != 0) { printf("%d/%zu case(s) FAILED\n", n_failed, n_run); return 1; From d845c3bb51608add1892f6c5891fb0de4c9d0e9e Mon Sep 17 00:00:00 2001 From: Yiyu Liu Date: Mon, 9 Nov 2026 23:45:07 -0500 Subject: [PATCH 3/3] docs: describe --report-interval --- doc/design.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ doc/quickstart.md | 1 + 2 files changed, 46 insertions(+) diff --git a/doc/design.md b/doc/design.md index 402f9af..379466f 100644 --- a/doc/design.md +++ b/doc/design.md @@ -794,6 +794,51 @@ partial-node policy occasionally punches a gap mid-prefix instead. That is the contiguity difference that the removed `prefix_hit_ratio` metric used to summarise in one number, now visible in full rather than averaged away. +## Interval reports (`--report-interval`) + +The `RESULT` line averages over the whole trace. To see how the ratios move +through it, `--report-interval ` prints a line every `s` seconds of trace +time, as cachesim's option of the same name does: + +```bash +prefixsim --trace freeinference.lcp.zst --cache-size 64k --algo lru \ + --cost-model qwen3coder30b_blksz_16 --report-interval 86400 +``` +``` +freeinference.lcp.zst lru 24.00 hour: 15761 requests, block hit ratio 0.9439, interval block hit ratio 0.9439, compute saving ratio 0.9391, interval compute saving ratio 0.9391 +freeinference.lcp.zst lru 48.00 hour: 33679 requests, block hit ratio 0.9434, interval block hit ratio 0.9428, compute saving ratio 0.9390, interval compute saving ratio 0.9389 +freeinference.lcp.zst lru 72.00 hour: 73808 requests, block hit ratio 0.9428, interval block hit ratio 0.9417, compute saving ratio 0.9385, interval compute saving ratio 0.9376 +freeinference.lcp.zst lru 96.00 hour: 179312 requests, block hit ratio 0.9293, interval block hit ratio 0.8981, compute saving ratio 0.9349, interval compute saving ratio 0.9260 +freeinference.lcp.zst lru 120.00 hour: 244833 requests, block hit ratio 0.9252, interval block hit ratio 0.9127, compute saving ratio 0.9297, interval compute saving ratio 0.9141 +freeinference.lcp.zst lru 144.00 hour: 281994 requests, block hit ratio 0.9239, interval block hit ratio 0.9201, compute saving ratio 0.9307, interval compute saving ratio 0.9335 +freeinference.lcp.zst lru 168.00 hour: 327548 requests, block hit ratio 0.9175, interval block hit ratio 0.8967, compute saving ratio 0.9259, interval compute saving ratio 0.9118 +lru block hit ratio 0.9175 compute saving 0.9259 +RESULT trace=freeinference.lcp.zst algo=lru cache_size=65536 ... +``` + +Each line gives the trace time in hours, the requests served so far, and both +ratios over the run so far and over the interval just closed. The wording +follows cachesim's `interval miss ratio` lines, so a script that parses one +parses the other with a changed regex. The lines go to stdout as they happen, +ahead of each algorithm's summary and `RESULT` lines. + +An interval is cut as cachesim cuts it: + +- **Trace time** is a request's `timestamp` less the first request's: the + completion time in fi-new and chutes, the arrival time in the Qwen traces. +- **An interval closes** at the first served request at least `s` seconds after + the previous report, and includes that request. The times on the lines are + those requests' times, so they drift past multiples of `s`, and a quiet + stretch of the trace makes one long interval rather than several empty ones. +- **The end of the trace** closes one more interval if anything was served + after the last report, so the intervals always add up to the `RESULT` line. + cachesim stops at its last full interval. +- **Skipped requests** (larger than the cache) are left out of every count, as + everywhere else, and do not close an interval. + +The reports are read off the phase-1 statistics and change nothing else: the +`RESULT` line is the same with or without the option. It is off by default. + ## Adding an input format Implement `TraceReader` in `src/trace.cpp` and add one diff --git a/doc/quickstart.md b/doc/quickstart.md index c1a8b4f..e498f7b 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -148,6 +148,7 @@ The other diagnostics are described in | `--no-pin` | let eviction take blocks the arriving request already holds | | `--no-verify` | skip the check that every block is resident after each request | | `--dump-holes ` | write each request's runs of missed blocks, per algorithm | +| `--report-interval ` | every `s` seconds of trace time, print both ratios so far and over the last interval, like cachesim's option ([how](design.md#interval-reports---report-interval)) | | `--block-id raw` | JSONL only: use the trace's hashes as block ids without positional encoding ([when](design.md#block-identity---block-id)) | ---