diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index 42c6a658..8e816355 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -29,3 +29,11 @@ {"ts": "2026-10-24T05:13:10.885088+00:00", "task": "aicomp #265 / #267", "result": "KONI whole diagnostic264 responses pass;8K/128 native7.7847->whole9.3113tok/s; native/whole token parity6/8. Stream pending, different physical ATOM allocation.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-24-stream265-atom-http-progress.md"], "runtime_commit": "33d6cb08ec806f693dbecfd7d630331a3bd1bb2f"} {"ts": "2026-10-24T05:56:15.922056+00:00", "task": "aicomp #265 / #267", "result": "KONI all3 routes264 responses each audited; whole/stream8/8 exact.8K128 throughput7.7847/9.3113/10.8298tok/s. Native parity6/8; single batch and different physical card limits retained.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-24-stream265-atom-http-progress.md"], "runtime_commit": "33d6cb08ec806f693dbecfd7d630331a3bd1bb2f"} {"ts": "2026-10-30T04:24:12.257264+00:00", "task": "aicomp #201 / #265", "result": "ATOM short/long final-HTTP sweep:12 arms96conditions528responses audited;whole/stream16/16 references exact. Native/PD Qwen0.6/Qwen4=4/4,Gemma2/4(other2afterEOS),KONI2/4(preEOS). Eleven selfchecks pass;coordinator/cleanup0,fresh owned-Pod absence verified.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-atom-edge201-final.md"], "runtime_scope": "historical stream265 B1 engine,final HTTP SHA256e3449979b88d03edb6d5bf0d402a708e5d98054fdf203e22b480f3d7e6fd759a"} +{"ts": "2026-10-30T05:14:21.740281+00:00", "task": "aicomp #267 / #201", "result": "HTTP backlog128 candidate:15CPUtests pass;Qwen4 whole/cacheoff24conditions344responses exact in one retry after originalreset. Remaining hardware verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T05:26:15.716445+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 native ATOM cache-on retry:24 conditions,344 responses exact;13 auditor selfchecks pass;host listen drops +82 original/+0 retry. Wider hardware verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T05:36:34.408243+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 RNGD-GPU whole KV paired cache off/on:48 conditions,688 responses exact. Across24 conditions throughput improves11,range -3.72% to +15.26%;single ordered pair. Full paired evidence retained;remaining hardware runs pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T05:48:16.366880+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 GPU stream/off24conditions344responses exact;whole/stream references8/8equal. Same candidate,only stream flag differs. Single ordered pair all24 throughput increases(+4.33..30.38%);8K128c4 31.97->37.54tok/s. Remaining verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T06:03:17.316059+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 GPU PD four configurations complete96conditions1376responses exact;all4reference sets equal,listen deltas0. Full grid provenance/timings retained;single ordered samples only. Gemma GPU and Qwen4 ATOM pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T06:38:39.539907+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 ATOM whole/on24conditions344responses same-route exact,listen deltas0. Native/PD6/8refs equal;2preEOS differences also present in historical cacheoff. At8K128c4 native23.07vsPD15.04tok/s. Streaming pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T06:45:47.350695+00:00", "task": "aicomp #267 / #201", "result": "Gemma GPU PD four configurations96conditions1376responses exact,all4reference sets equal,listen deltas0. Full timing/provenance retained;streamcache gain10/24 only(-4.93..+2.58%),single ordered samples. Remaining model tests pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T06:50:26.376967+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 ATOM native/whole/stream72conditions1032responses same-route exact;whole/stream8/8refs,nativePD6/8.8K128c4 native23.07/whole15.04/stream16.52tok/s. Owned cleanup and freshabsence verified. KONI GPU/TP2 pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index d2880622..15237318 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -146,3 +146,21 @@ Four models, whole/stream, input128/8192, output32/128, c1/c4:64 conditions352 r ## ATOM short/long controls complete (2026-10-30) All four models complete native/whole/stream at128/8192 input,32/128 output,c1/c4:96conditions528responses. Whole/stream16/16 reference pairs match. Native/PD equality is Qwen0.6/Qwen4=4/4,Gemma2/4(other2equal through EOS),KONI2/4(pre-EOS differences retained). Single-batch8K/128/c4 stream throughput44.24/14.47/43.87/10.44tok/s; one ATOM lane,extra RNGD for PD. Eleven independent auditor selfchecks pass;all exits0 and owned Pod absence verified. Cache and broader goal work remain active. See experiments/2026-10-30-atom-edge201-final.md. + +## HTTP listener backlog candidate (#267) + +Backlog128 handles a16connection accept-pause regression;15HTTPtests pass. Qwen4 RNGD-GPU whole/cacheoff retry24conditions344responses exact after originalc16 connection reset;listen drops+36before/+0retry(host-wide). One retry only;ATOM/remaining GPU verification pending. See experiments/2026-10-30-http-backlog267.md. + +PR289 follow-up: Qwen4 native ATOM cache-on retry also passes 24 conditions/344 responses; listen drops +82 original/+0 retry, no OOM in original. Both GPU and ATOM have one successful retry; other routes/models remain pending. + +PR289 paired-cache follow-up: Qwen4 RNGD-GPU whole KV paired cache off/on:48 conditions,688 responses exact. Across24 conditions throughput improves11,range -3.72% to +15.26%;single ordered pair. Full paired evidence retained;remaining hardware runs pending. + +PR289 streamed-control follow-up: Qwen4 GPU stream/off24conditions344responses exact;whole/stream references8/8equal. Same candidate,only stream flag differs. Single ordered pair all24 throughput increases(+4.33..30.38%);8K128c4 31.97->37.54tok/s. Remaining verification pending. + +PR289 Qwen4 GPU subset: Qwen4 GPU PD four configurations complete96conditions1376responses exact;all4reference sets equal,listen deltas0. Full grid provenance/timings retained;single ordered samples only. Gemma GPU and Qwen4 ATOM pending. + +PR289 ATOM PD follow-up: Qwen4 ATOM whole/on24conditions344responses same-route exact,listen deltas0. Native/PD6/8refs equal;2preEOS differences also present in historical cacheoff. At8K128c4 native23.07vsPD15.04tok/s. Streaming pending. + +PR289 Gemma GPU subset: Gemma GPU PD four configurations96conditions1376responses exact,all4reference sets equal,listen deltas0. Full timing/provenance retained;streamcache gain10/24 only(-4.93..+2.58%),single ordered samples. Remaining model tests pending. + +PR289 Qwen4 ATOM subset: Qwen4 ATOM native/whole/stream72conditions1032responses same-route exact;whole/stream8/8refs,nativePD6/8.8K128c4 native23.07/whole15.04/stream16.52tok/s. Owned cleanup and freshabsence verified. KONI GPU/TP2 pending. diff --git a/opt_prime/examples/pd/http_matrix/evidence267/gemma-full-pd-cache-summary.json b/opt_prime/examples/pd/http_matrix/evidence267/gemma-full-pd-cache-summary.json new file mode 100644 index 00000000..f427c1a5 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/gemma-full-pd-cache-summary.json @@ -0,0 +1,1388 @@ +{ + "model": "unsloth/gemma-3-1b-it", + "runtime_base": "f52c41305f6bb2ee65d3c0fe8166acbca51dcc72", + "source_sha256": { + "benchmark201_native_http.py": "e1ea87a51343d1af83c40aaa9c41ae2a20e9864aa8b53bd475d047d175c41433", + "llmd.py": "9f3aaf0902f7b1958f32be12b1f276384f20cc3e0070571af10bb7bce148c1f0", + "serve_batched.py": "cf509e2c392a39aab7b47d7296447ff62cc8237e193e42d7d9cd76fdc47d9c30", + "streaming_factories.py": "77745350973cc54bf508e0c8b473efc442afef20b1c64f7c4edc179ea1471389", + "stream265-python.sh": "1a417cc6dde8ec6837362153406e618b006169513851131f8d324ff46aa96d9a", + "fsdk.py": "2cf093e93fbc2a98c1ce242563e65d7feaaabff468a91654392412c74df98bb6" + }, + "arms": { + "whole/off": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 0.0, + "transferred_tokens": 935680.0 + }, + "raw_sha256": "5f7c0812f6492494248c64ce13100ab298baafe3754364c6d60f3ca781ca09b7", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.3279264480806887, + "output_tok_s": 13.746138769282474, + "median_first_sse_s": 1.5195072460919619, + "median_tpot_est_s": 0.026071898319998815, + "median_e2e_s": 2.327736094011925, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.743163365055807, + "output_tok_s": 18.98218878446832, + "median_first_sse_s": 3.7519505819655024, + "median_tpot_est_s": 0.024586322999018577, + "median_e2e_s": 4.534886106965132, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.738074910943396, + "output_tok_s": 20.696840875581078, + "median_first_sse_s": 12.740061033982784, + "median_tpot_est_s": 0.025022625419942122, + "median_e2e_s": 13.519588774011936, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.590997903025709, + "output_tok_s": 27.88064876170849, + "median_first_sse_s": 1.5005308070685714, + "median_tpot_est_s": 0.02433352762976236, + "median_e2e_s": 4.590888816048391, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.130475593032315, + "output_tok_s": 56.07593983282968, + "median_first_sse_s": 3.7736012175446376, + "median_tpot_est_s": 0.02480497963746687, + "median_e2e_s": 6.946635517524555, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.1226946109673, + "output_tok_s": 75.508721731943, + "median_first_sse_s": 12.762706937966868, + "median_tpot_est_s": 0.025122510413199253, + "median_e2e_s": 16.032839732943103, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.3080955729819834, + "output_tok_s": 13.8642439137202, + "median_first_sse_s": 1.5271887209964916, + "median_tpot_est_s": 0.025187086708272896, + "median_e2e_s": 2.3079884089529514, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.8409073810325935, + "output_tok_s": 18.71096813193211, + "median_first_sse_s": 3.8035742895444855, + "median_tpot_est_s": 0.025399381223495207, + "median_e2e_s": 4.5866546424804255, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.83240737800952, + "output_tok_s": 20.618218451643337, + "median_first_sse_s": 12.765676048526075, + "median_tpot_est_s": 0.024825153613252743, + "median_e2e_s": 13.52388443046948, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.821606518002227, + "output_tok_s": 26.547168360191122, + "median_first_sse_s": 1.5025401849998161, + "median_tpot_est_s": 0.026132927086446288, + "median_e2e_s": 4.821421924978495, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.118290200014599, + "output_tok_s": 56.15087793533707, + "median_first_sse_s": 3.751231240981724, + "median_tpot_est_s": 0.024738553586492944, + "median_e2e_s": 6.915239866473712, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.99413980008103, + "output_tok_s": 75.8683186486962, + "median_first_sse_s": 12.75357724505011, + "median_tpot_est_s": 0.024844022570856238, + "median_e2e_s": 15.9701726125204, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.2997427120571956, + "output_tok_s": 13.914600025571968, + "median_first_sse_s": 1.5408305689925328, + "median_tpot_est_s": 0.024475289095613743, + "median_e2e_s": 2.299564530956559, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.828186889062636, + "output_tok_s": 18.7458255140951, + "median_first_sse_s": 3.8055082780192606, + "median_tpot_est_s": 0.02437940348247667, + "median_e2e_s": 4.566361733479425, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 25.028537103906274, + "output_tok_s": 20.456649059208928, + "median_first_sse_s": 12.85288890299853, + "median_tpot_est_s": 0.023986310467139005, + "median_e2e_s": 13.629948533489369, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.6510499940486625, + "output_tok_s": 27.520667411398453, + "median_first_sse_s": 1.5235844729468226, + "median_tpot_est_s": 0.024624736181196323, + "median_e2e_s": 4.650925967958756, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.157141407951713, + "output_tok_s": 55.91264535408383, + "median_first_sse_s": 3.9284273515222594, + "median_tpot_est_s": 0.023716124732094987, + "median_e2e_s": 6.974470154498704, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.226294361054897, + "output_tok_s": 75.22140078414436, + "median_first_sse_s": 12.845481677446514, + "median_tpot_est_s": 0.02481556302404002, + "median_e2e_s": 16.034130166459363, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.3383831130340695, + "output_tok_s": 13.684669471667439, + "median_first_sse_s": 1.5876917759887874, + "median_tpot_est_s": 0.024212286161679412, + "median_e2e_s": 2.3382726470008492, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.9451680039055645, + "output_tok_s": 18.430079722768426, + "median_first_sse_s": 3.9202036120113917, + "median_tpot_est_s": 0.02514317951675865, + "median_e2e_s": 4.689277389028575, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 25.149196759914048, + "output_tok_s": 20.35850309207847, + "median_first_sse_s": 13.821421672473662, + "median_tpot_est_s": 0.02708717667865717, + "median_e2e_s": 14.623511960497126, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.719465148984455, + "output_tok_s": 27.12171738942565, + "median_first_sse_s": 1.5936326179653406, + "median_tpot_est_s": 0.02461201841777968, + "median_e2e_s": 4.71935895702336, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.28263967903331, + "output_tok_s": 55.15672456364476, + "median_first_sse_s": 3.9311249460442923, + "median_tpot_est_s": 0.0258020139248917, + "median_e2e_s": 7.207980714505538, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.833847583038732, + "output_tok_s": 73.5794788661558, + "median_first_sse_s": 13.436637023987714, + "median_tpot_est_s": 0.025422491287211264, + "median_e2e_s": 16.623036271485034, + "all_text_equal": true + } + ], + "config_sha256": "429f861781991f63e9d31c672c1d612aff45e15e49058377ddd15d2c34029330", + "packages": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "whole/on": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 850688.0, + "transferred_tokens": 84992.0 + }, + "raw_sha256": "44592b0e9d312d2dd5fde84db5ac1f63e51d1ce84980932af6c95b66cfe035d3", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.294777803006582, + "output_tok_s": 13.944705216371755, + "median_first_sse_s": 1.5151921110227704, + "median_tpot_est_s": 0.025143629416913515, + "median_e2e_s": 2.2946446229470894, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.7410926590673625, + "output_tok_s": 18.98801966886314, + "median_first_sse_s": 3.753810322494246, + "median_tpot_est_s": 0.02379378430815714, + "median_e2e_s": 4.495097044564318, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.815822748001665, + "output_tok_s": 20.63199778622007, + "median_first_sse_s": 12.720985402003862, + "median_tpot_est_s": 0.02436743817675198, + "median_e2e_s": 13.510060362983495, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.499732009018771, + "output_tok_s": 28.44613851301606, + "median_first_sse_s": 1.4961800730088726, + "median_tpot_est_s": 0.023648921007979808, + "median_e2e_s": 4.499593041022308, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.146101497928612, + "output_tok_s": 55.980135374176264, + "median_first_sse_s": 3.760683643457014, + "median_tpot_est_s": 0.02449713394505991, + "median_e2e_s": 6.871819654479623, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.23556305700913, + "output_tok_s": 75.19580174322641, + "median_first_sse_s": 12.845750043052249, + "median_tpot_est_s": 0.025506295956300908, + "median_e2e_s": 16.08926992106717, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.284619624959305, + "output_tok_s": 14.006708009684546, + "median_first_sse_s": 1.5479519880609587, + "median_tpot_est_s": 0.023759875837112625, + "median_e2e_s": 2.28450813901145, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.722448652959429, + "output_tok_s": 19.040680949441015, + "median_first_sse_s": 3.7428368200198747, + "median_tpot_est_s": 0.02400586399955735, + "median_e2e_s": 4.482454030017834, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.702370829996653, + "output_tok_s": 20.726755481229628, + "median_first_sse_s": 12.781110248004552, + "median_tpot_est_s": 0.024122127290484643, + "median_e2e_s": 13.536579754494596, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.564453889033757, + "output_tok_s": 28.042785207563163, + "median_first_sse_s": 1.498603621032089, + "median_tpot_est_s": 0.02413911622028622, + "median_e2e_s": 4.564271381008439, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.206287367036566, + "output_tok_s": 55.61416666540672, + "median_first_sse_s": 3.8192548800143413, + "median_tpot_est_s": 0.02545566732654216, + "median_e2e_s": 7.061408449488226, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.19484285300132, + "output_tok_s": 75.30839619372816, + "median_first_sse_s": 12.779843088472262, + "median_tpot_est_s": 0.024819888452738963, + "median_e2e_s": 15.918898725474719, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.294593045953661, + "output_tok_s": 13.945828022284624, + "median_first_sse_s": 1.5489279510220513, + "median_tpot_est_s": 0.024049640159993883, + "median_e2e_s": 2.2944667959818617, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.811993209994398, + "output_tok_s": 18.79038866512688, + "median_first_sse_s": 3.810922817501705, + "median_tpot_est_s": 0.023920489257321723, + "median_e2e_s": 4.5524579844786786, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.577400514041074, + "output_tok_s": 20.832146170523377, + "median_first_sse_s": 12.677245650964323, + "median_tpot_est_s": 0.023498049822650968, + "median_e2e_s": 13.405580043443479, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.6355026679812, + "output_tok_s": 27.61297084006319, + "median_first_sse_s": 1.5174282259540632, + "median_tpot_est_s": 0.02455074081149715, + "median_e2e_s": 4.635372309014201, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.128292449051514, + "output_tok_s": 56.08935108703709, + "median_first_sse_s": 3.852934113529045, + "median_tpot_est_s": 0.02434130883445655, + "median_e2e_s": 6.967866144492291, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.278488956973888, + "output_tok_s": 75.07747233471369, + "median_first_sse_s": 12.975559875019826, + "median_tpot_est_s": 0.02475071167316669, + "median_e2e_s": 16.162191890529357, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.2262181820115075, + "output_tok_s": 14.374152658786699, + "median_first_sse_s": 1.4948794160736725, + "median_tpot_est_s": 0.023585418806291156, + "median_e2e_s": 2.2260273990686983, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.799865473993123, + "output_tok_s": 18.823901809462395, + "median_first_sse_s": 3.758145831001457, + "median_tpot_est_s": 0.02464070370901496, + "median_e2e_s": 4.521373027469963, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.871456557069905, + "output_tok_s": 20.585847026094658, + "median_first_sse_s": 13.620587156503461, + "median_tpot_est_s": 0.02503044254920115, + "median_e2e_s": 14.379491011437494, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.608603984932415, + "output_tok_s": 27.774137334969367, + "median_first_sse_s": 1.5076510719954967, + "median_tpot_est_s": 0.024415457385636163, + "median_e2e_s": 4.608414159971289, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.176488784956746, + "output_tok_s": 55.7947611551964, + "median_first_sse_s": 3.776593458955176, + "median_tpot_est_s": 0.025861740110471376, + "median_e2e_s": 7.061034452985041, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.35276161902584, + "output_tok_s": 74.8736097848148, + "median_first_sse_s": 13.083223339985125, + "median_tpot_est_s": 0.024744039658075714, + "median_e2e_s": 16.22622210800182, + "all_text_equal": true + } + ], + "config_sha256": "def8bcbe12b01fb247db995e81b318cc99b776920ae08a84322baf96a8d9b584", + "packages": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "stream/off": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 0.0, + "transferred_tokens": 935680.0 + }, + "raw_sha256": "b7a5d24446370bc53c93a468cf8368f0096e2b37560f4918b81857a017ff37c9", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.208784915972501, + "output_tok_s": 14.487603464056974, + "median_first_sse_s": 1.4633835150161758, + "median_tpot_est_s": 0.02404057232272481, + "median_e2e_s": 2.208641257020645, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.5753573960391805, + "output_tok_s": 19.466622464826592, + "median_first_sse_s": 3.654461637022905, + "median_tpot_est_s": 0.02462164083901312, + "median_e2e_s": 4.421989984984975, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.097053020028397, + "output_tok_s": 21.247411439666436, + "median_first_sse_s": 12.423074103018735, + "median_tpot_est_s": 0.025114445162246063, + "median_e2e_s": 13.193876743491273, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.487484783050604, + "output_tok_s": 28.523773603302395, + "median_first_sse_s": 1.4645146490074694, + "median_tpot_est_s": 0.023801707173147658, + "median_e2e_s": 4.487331459997222, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 8.898463926045224, + "output_tok_s": 57.5380205230039, + "median_first_sse_s": 3.680733249988407, + "median_tpot_est_s": 0.024517924035349345, + "median_e2e_s": 6.826566770032514, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.336110907024704, + "output_tok_s": 77.76394955314876, + "median_first_sse_s": 12.45146652951371, + "median_tpot_est_s": 0.0250761886338124, + "median_e2e_s": 15.624904347525444, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.1974527299171314, + "output_tok_s": 14.562315523031414, + "median_first_sse_s": 1.4644598189042881, + "median_tpot_est_s": 0.023640203646444265, + "median_e2e_s": 2.1973061319440603, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.654567905003205, + "output_tok_s": 19.234907784735928, + "median_first_sse_s": 3.6605074605322443, + "median_tpot_est_s": 0.02419535599971911, + "median_e2e_s": 4.402268527017441, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.137016809894703, + "output_tok_s": 21.212231985110574, + "median_first_sse_s": 12.423588597972412, + "median_tpot_est_s": 0.02459894633859456, + "median_e2e_s": 13.184217029483989, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.357999384985305, + "output_tok_s": 29.371275370299667, + "median_first_sse_s": 1.4642595020122826, + "median_tpot_est_s": 0.022784263330079088, + "median_e2e_s": 4.357860944932327, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 8.883686052984558, + "output_tok_s": 57.633734121883876, + "median_first_sse_s": 3.698318095470313, + "median_tpot_est_s": 0.024855834641402192, + "median_e2e_s": 6.860209960490465, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.494456356973387, + "output_tok_s": 77.2991894004635, + "median_first_sse_s": 12.526686669967603, + "median_tpot_est_s": 0.024864938393359872, + "median_e2e_s": 15.6774896789575, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.2191090310225263, + "output_tok_s": 14.420201780376228, + "median_first_sse_s": 1.4827205820474774, + "median_tpot_est_s": 0.023748166288339322, + "median_e2e_s": 2.2189137369859964, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.728352272999473, + "output_tok_s": 19.023974192560836, + "median_first_sse_s": 3.7194058959721588, + "median_tpot_est_s": 0.0248676183381899, + "median_e2e_s": 4.490302064456046, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.420636598020792, + "output_tok_s": 20.965874413015744, + "median_first_sse_s": 12.510112286487129, + "median_tpot_est_s": 0.02547007288611043, + "median_e2e_s": 13.306779410515446, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.579361124080606, + "output_tok_s": 27.95149727915342, + "median_first_sse_s": 1.4736862790305167, + "median_tpot_est_s": 0.024453261684751006, + "median_e2e_s": 4.5792505129938945, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.010187053936534, + "output_tok_s": 56.82456944956633, + "median_first_sse_s": 3.7041273544891737, + "median_tpot_est_s": 0.02467996403565777, + "median_e2e_s": 6.847893105470575, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.87396427197382, + "output_tok_s": 76.20758810548125, + "median_first_sse_s": 12.641634211468045, + "median_tpot_est_s": 0.02488390423218155, + "median_e2e_s": 15.758425751526374, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.233825940056704, + "output_tok_s": 14.325198497420844, + "median_first_sse_s": 1.4876205119071528, + "median_tpot_est_s": 0.024065800067487982, + "median_e2e_s": 2.2336603139992803, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.711063691996969, + "output_tok_s": 19.07298244727459, + "median_first_sse_s": 3.7502244460047223, + "median_tpot_est_s": 0.02529637779169265, + "median_e2e_s": 4.5344121575471945, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.589590384042822, + "output_tok_s": 20.82181898939876, + "median_first_sse_s": 13.402657889586408, + "median_tpot_est_s": 0.026513373967440378, + "median_e2e_s": 14.244673553097527, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.601558788912371, + "output_tok_s": 27.816660803817353, + "median_first_sse_s": 1.4871936520794407, + "median_tpot_est_s": 0.024521522700610594, + "median_e2e_s": 4.601427035056986, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.045747432042845, + "output_tok_s": 56.601182361817564, + "median_first_sse_s": 3.764507564541418, + "median_tpot_est_s": 0.025608588775480532, + "median_e2e_s": 7.016798339027446, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.193519476917572, + "output_tok_s": 75.31206108640646, + "median_first_sse_s": 13.009069821040612, + "median_tpot_est_s": 0.02543677709488873, + "median_e2e_s": 16.253884334524628, + "all_text_equal": true + } + ], + "config_sha256": "5e4fbf615f999301c89914edc836f5d60e7056671d7d60f7fff521147c952be3", + "packages": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "stream/on": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 850688.0, + "transferred_tokens": 84992.0 + }, + "raw_sha256": "5d1b70e813e198a43f3182784bb44b33a3bd87e6c1054237eb0f8fc210c98541", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.2193764109397307, + "output_tok_s": 14.418464503031519, + "median_first_sse_s": 1.4607410749886185, + "median_tpot_est_s": 0.02446787642134774, + "median_e2e_s": 2.2192452440503985, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.623916062060744, + "output_tok_s": 19.32391636620141, + "median_first_sse_s": 3.652397355530411, + "median_tpot_est_s": 0.025490233806082077, + "median_e2e_s": 4.4497615519794635, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.252061516977847, + "output_tok_s": 21.111607342805492, + "median_first_sse_s": 12.537051899009384, + "median_tpot_est_s": 0.025534329404528704, + "median_e2e_s": 13.309377963014413, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.588234180002473, + "output_tok_s": 27.897442671492197, + "median_first_sse_s": 1.4684002309804782, + "median_tpot_est_s": 0.024564731629624022, + "median_e2e_s": 4.588121147942729, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 8.714870621915907, + "output_tok_s": 58.75015501807188, + "median_first_sse_s": 3.6713809845387004, + "median_tpot_est_s": 0.023759379680910124, + "median_e2e_s": 6.688822204014286, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.41557551198639, + "output_tok_s": 77.53001629931155, + "median_first_sse_s": 12.573357547051273, + "median_tpot_est_s": 0.02497935570055971, + "median_e2e_s": 15.68049829103984, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.2010550249833614, + "output_tok_s": 14.538482517147385, + "median_first_sse_s": 1.4664556350326166, + "median_tpot_est_s": 0.023691118255467904, + "median_e2e_s": 2.2008803009521216, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.63359681610018, + "output_tok_s": 19.295715966538015, + "median_first_sse_s": 3.6619083679979667, + "median_tpot_est_s": 0.02578308109584595, + "median_e2e_s": 4.465914245462045, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.173571440041997, + "output_tok_s": 21.180155413523394, + "median_first_sse_s": 12.44185278203804, + "median_tpot_est_s": 0.025642615320506476, + "median_e2e_s": 13.242599404067732, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.584160504047759, + "output_tok_s": 27.92223350098181, + "median_first_sse_s": 1.4688922829227522, + "median_tpot_est_s": 0.024528703787987393, + "median_e2e_s": 4.584037663997151, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 8.996187166892923, + "output_tok_s": 56.91299997450287, + "median_first_sse_s": 3.7267173575237393, + "median_tpot_est_s": 0.02494246259837697, + "median_e2e_s": 6.9479099354939535, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.62871236994397, + "output_tok_s": 76.90946417340079, + "median_first_sse_s": 12.502239950466901, + "median_tpot_est_s": 0.02562358440083312, + "median_e2e_s": 15.707677363534458, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.261181575944647, + "output_tok_s": 14.151893125447678, + "median_first_sse_s": 1.4730743109248579, + "median_tpot_est_s": 0.025417398646353714, + "median_e2e_s": 2.261013668961823, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.5867575000738725, + "output_tok_s": 19.432930390797665, + "median_first_sse_s": 3.6612320354324766, + "median_tpot_est_s": 0.024821561806264422, + "median_e2e_s": 4.457853511965368, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.271716309944168, + "output_tok_s": 21.09451154841624, + "median_first_sse_s": 12.441811380034778, + "median_tpot_est_s": 0.025020278642900406, + "median_e2e_s": 13.19704235746758, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.663960383972153, + "output_tok_s": 27.444486972890257, + "median_first_sse_s": 1.461892998078838, + "median_tpot_est_s": 0.025212164652922493, + "median_e2e_s": 4.663837908999994, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.092023623059504, + "output_tok_s": 56.313096096830186, + "median_first_sse_s": 3.6667037500301376, + "median_tpot_est_s": 0.025807405223860956, + "median_e2e_s": 6.977219620952383, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.680595836951397, + "output_tok_s": 76.75990493299307, + "median_first_sse_s": 12.540620018960908, + "median_tpot_est_s": 0.025764916421236895, + "median_e2e_s": 15.884110663027968, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.3031733550596982, + "output_tok_s": 13.893873828342617, + "median_first_sse_s": 1.5569050030317158, + "median_tpot_est_s": 0.024067208193422805, + "median_e2e_s": 2.3029884570278227, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.612298600957729, + "output_tok_s": 19.357867471602145, + "median_first_sse_s": 3.6647086894954555, + "median_tpot_est_s": 0.02475741664473448, + "median_e2e_s": 4.425869460974354, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.245720005012117, + "output_tok_s": 21.11712912192991, + "median_first_sse_s": 13.209060697467066, + "median_tpot_est_s": 0.025803333790522187, + "median_e2e_s": 14.002716014510952, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.486034267931245, + "output_tok_s": 28.532996485340664, + "median_first_sse_s": 1.4778158440021798, + "median_tpot_est_s": 0.0236857090160016, + "median_e2e_s": 4.485900889034383, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 8.97912462207023, + "output_tok_s": 57.021148669830254, + "median_first_sse_s": 3.724316539533902, + "median_tpot_est_s": 0.025794144878420598, + "median_e2e_s": 7.000172939093318, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.04220046196133, + "output_tok_s": 75.73348192876541, + "median_first_sse_s": 12.852511344943196, + "median_tpot_est_s": 0.025671009602223148, + "median_e2e_s": 16.030342576967087, + "all_text_equal": true + } + ], + "config_sha256": "01c746af74c5dc1dc122c9ebefff5ad0118e245bd8666f0c18c973731503d644", + "packages": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + } + }, + "comparisons": { + "stream/off -> stream/on": { + "increased_conditions": 10, + "total_conditions": 24, + "min_throughput_change_pct": -4.933534043207166, + "max_throughput_change_pct": 2.5752037118164983 + }, + "whole/on -> stream/on": { + "increased_conditions": 20, + "total_conditions": 24, + "min_throughput_change_pct": -3.3412670774057407, + "max_throughput_change_pct": 4.94821890904793 + } + }, + "scope": { + "conditions": 96, + "responses": 1376, + "repeats_per_condition": 1 + }, + "limits": [ + "Single ordered runs, no statistical repeatability claim.", + "Same-reference parity is not native-engine equivalence or answer quality.", + "One A10080GB plus one RNGD research producer;four GPU slots,BF16,context9216,page/chunk256,width8192.", + "Cache counters count logical KV tokens;full producer prefill andCPUcopy remain. No physical DMA overlap orPCIe-byte reduction proof." + ], + "all_four_reference_sets_equal": true +} diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-backlog-controls.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-backlog-controls.json new file mode 100644 index 00000000..1fc3b4b1 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-backlog-controls.json @@ -0,0 +1,395 @@ +{ + "model": "Qwen/Qwen3-4B", + "hardware": "One assigned ATOM decoder; one execution lane, queued client concurrency", + "versions": { + "torch": "2.11.0+cpu", + "transformers": "5.8.1", + "optimum-rbln": "0.10.2", + "rebel-compiler": "0.11.0" + }, + "candidate_http_sha256": "9f3aaf0902f7b1958f32be12b1f276384f20cc3e0070571af10bb7bce148c1f0", + "configuration": { + "inputs": [ + 128, + 512, + 2048, + 8192 + ], + "outputs": [ + 32, + 128 + ], + "concurrency": [ + 1, + 4, + 16 + ], + "context": 9216, + "kv_blocks": 3, + "batch_size": 1, + "decoder_cache": true, + "producer_cache": false, + "repeats": 1 + }, + "before": { + "status": "failed_not_passed", + "raw_sha256": "456652095fdae63a2295f8533217d24f1c6d8df63fb0d0bb84fda0346b34b326", + "error": "ConnectionResetError(104, 'Connection reset by peer')", + "failed_condition": { + "input_tokens": 512, + "output_tokens": 128, + "concurrency": 16, + "stream": true, + "elapsed_s": 72.44087444990873 + }, + "completed_warmups": 8, + "completed_token_batches": 24, + "completed_token_responses": 168, + "completed_token_responses_equal": true, + "completed_timed_batches": 11, + "server_alive_at_failure": true, + "memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "listen_counter_delta": { + "ListenOverflows": 82, + "ListenDrops": 82 + }, + "limits": "Listen counters do not isolate the specific server socket; candidate retry needed. Incomplete result is not a full-grid pass." + }, + "after": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 867456.0 + }, + "raw_sha256": "f6b0ed35f371a37e6460adebb51eca0af569b1b2fb061f199896d63b0d8b91a5", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 1.21979605499655, + "output_tok_s": 26.233893665191847, + "median_first_sse_s": 0.06424786592833698, + "median_tpot_est_s": 0.03727080548302301, + "median_e2e_s": 1.2196428359020501, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 4.865430230041966, + "output_tok_s": 26.308053748187437, + "median_first_sse_s": 1.8862478689989075, + "median_tpot_est_s": 0.03721824728868782, + "median_e2e_s": 3.0406833300366998, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 19.443709495011717, + "output_tok_s": 26.332423868570633, + "median_first_sse_s": 9.170724456431344, + "median_tpot_est_s": 0.03721861124053717, + "median_e2e_s": 10.32500390149653, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.779329141136259, + "output_tok_s": 26.782001452523673, + "median_first_sse_s": 0.06421606685034931, + "median_tpot_est_s": 0.03712567288289213, + "median_e2e_s": 4.77917652297765, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.179921305039898, + "output_tok_s": 26.69458293686857, + "median_first_sse_s": 7.2513526930706576, + "median_tpot_est_s": 0.03726597728691702, + "median_e2e_s": 11.985176004120149, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 76.49607840902172, + "output_tok_s": 26.772614264608695, + "median_first_sse_s": 35.89145322609693, + "median_tpot_est_s": 0.0371851845594915, + "median_e2e_s": 40.606709040584974, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 1.409332315903157, + "output_tok_s": 22.705787441972554, + "median_first_sse_s": 0.248691565124318, + "median_tpot_est_s": 0.0374359464828646, + "median_e2e_s": 1.4092059060931206, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 4.901245624991134, + "output_tok_s": 26.115810100872377, + "median_first_sse_s": 1.9010033400263637, + "median_tpot_est_s": 0.03745433188704474, + "median_e2e_s": 3.062635939917527, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 19.62891486589797, + "output_tok_s": 26.08396865022408, + "median_first_sse_s": 9.260896492982283, + "median_tpot_est_s": 0.03751262429080182, + "median_e2e_s": 10.424593756091781, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.838200076017529, + "output_tok_s": 26.456119628967624, + "median_first_sse_s": 0.06642191391438246, + "median_tpot_est_s": 0.0375715390867077, + "median_e2e_s": 4.83800737792626, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.333293949021026, + "output_tok_s": 26.482812569346258, + "median_first_sse_s": 7.3223171149147674, + "median_tpot_est_s": 0.03758554547607136, + "median_e2e_s": 12.085917299962603, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 77.24892303696834, + "output_tok_s": 26.51169646753401, + "median_first_sse_s": 36.27493576845154, + "median_tpot_est_s": 0.03750518164927448, + "median_e2e_s": 41.03906974289566, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.0401992758270353, + "output_tok_s": 15.684742357840591, + "median_first_sse_s": 0.8407348680775613, + "median_tpot_est_s": 0.03868695670708774, + "median_e2e_s": 2.040030525997281, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 5.082977191079408, + "output_tok_s": 25.182092145650223, + "median_first_sse_s": 1.9748041100101545, + "median_tpot_est_s": 0.038678585807041775, + "median_e2e_s": 3.1738402700284496, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 20.373157310998067, + "output_tok_s": 25.13110718109492, + "median_first_sse_s": 9.618916884530336, + "median_tpot_est_s": 0.03872891262759484, + "median_e2e_s": 10.823435940546915, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.998283860972151, + "output_tok_s": 25.608789648674414, + "median_first_sse_s": 0.07471933006308973, + "median_tpot_est_s": 0.038767074181617595, + "median_e2e_s": 4.9981377511285245, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.97290690196678, + "output_tok_s": 25.63472620750974, + "median_first_sse_s": 7.558827344560996, + "median_tpot_est_s": 0.038741515790786975, + "median_e2e_s": 12.481202929164283, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 79.82538939104415, + "output_tok_s": 25.655997617091124, + "median_first_sse_s": 37.48764609743375, + "median_tpot_est_s": 0.0387145414293971, + "median_e2e_s": 42.39941652864218, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 5.352265110006556, + "output_tok_s": 5.978777086391523, + "median_first_sse_s": 4.025504268007353, + "median_tpot_est_s": 0.04279371299931119, + "median_e2e_s": 5.352109370986, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 5.7538805468939245, + "output_tok_s": 22.24585633240115, + "median_first_sse_s": 2.2709930365672335, + "median_tpot_est_s": 0.04280944161056992, + "median_e2e_s": 3.5957872320432216, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 23.044004359981045, + "output_tok_s": 22.218360663442486, + "median_first_sse_s": 10.923830434447154, + "median_tpot_est_s": 0.042838023904140196, + "median_e2e_s": 12.250851112417877, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.550273383036256, + "output_tok_s": 23.06192707393777, + "median_first_sse_s": 0.11678028688766062, + "median_tpot_est_s": 0.04278182463104448, + "median_e2e_s": 5.55007201503031, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 22.190961834043264, + "output_tok_s": 23.07245642748744, + "median_first_sse_s": 8.439156876062043, + "median_tpot_est_s": 0.0427968583077075, + "median_e2e_s": 13.874486317043193, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 88.77933540102094, + "output_tok_s": 23.068431304977402, + "median_first_sse_s": 41.72694445645902, + "median_tpot_est_s": 0.04281781166131923, + "median_e2e_s": 47.15564054495189, + "all_text_equal": true + } + ] + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + }, + "limitations": [ + "One successful retry; not proof that every connection reset is resolved.", + "Host-wide listen counters do not identify the individual socket.", + "Output reference is retained same-route cache-off evidence, not a fresh paired off/on speedup.", + "RNGD-to-ATOM whole/stream and remaining GPU model/routes still running." + ] +} diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-divergence-history.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-divergence-history.json new file mode 100644 index 00000000..954b1fd8 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-divergence-history.json @@ -0,0 +1,27 @@ +{ + "model": "Qwen3-4B", + "kind": "Historical-reference comparison, not new hardware experiment", + "sources": { + "mid201-atom-final-qwen4b-native.json": "a21a9292094540a8a66c7056a6aa32e4b4832c839b20acf3523e3182e3971261", + "edge201-atom-final-qwen4b-native.json": "03254958b234ac5a00d6f6839129f4fc4c8fdd3a7bd54174f191ea751ab7cda5", + "mid201-atom-final-qwen4b-whole.json": "55278b0eb5c9a03b1808cdd3a84db0150326904f54504a9c4894d2aeaf750732", + "edge201-atom-final-qwen4b-whole.json": "f82833eeea8da4de03276192aada64eb73af5e6294d6d5e507a0830a248da4f6" + }, + "conditions": { + "512:128": { + "first_divergent_generated_index": 50, + "same_native_tokens_off_and_on": true, + "same_whole_tokens_off_and_on": true, + "native_token_at_divergence": 4829, + "whole_token_at_divergence": 2299 + }, + "2048:128": { + "first_divergent_generated_index": 61, + "same_native_tokens_off_and_on": true, + "same_whole_tokens_off_and_on": true, + "native_token_at_divergence": 1366, + "whole_token_at_divergence": 5230 + } + }, + "conclusion": "The same native versus whole-PD token divergence is already present in archived cache-off references. Current cache-on outputs equal those same-route references. This does not identify the numerical root cause or prove all inputs are cache invariant." +} diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-full-summary.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-full-summary.json new file mode 100644 index 00000000..36837bfd --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-full-summary.json @@ -0,0 +1,1072 @@ +{ + "model": "Qwen3-4B", + "scope": { + "conditions": 72, + "responses": 1032, + "cache": "on", + "repeats": 1 + }, + "arms": { + "native": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 867456.0 + }, + "raw_sha256": "f6b0ed35f371a37e6460adebb51eca0af569b1b2fb061f199896d63b0d8b91a5", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 1.21979605499655, + "output_tok_s": 26.233893665191847, + "median_first_sse_s": 0.06424786592833698, + "median_tpot_est_s": 0.03727080548302301, + "median_e2e_s": 1.2196428359020501, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 4.865430230041966, + "output_tok_s": 26.308053748187437, + "median_first_sse_s": 1.8862478689989075, + "median_tpot_est_s": 0.03721824728868782, + "median_e2e_s": 3.0406833300366998, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 19.443709495011717, + "output_tok_s": 26.332423868570633, + "median_first_sse_s": 9.170724456431344, + "median_tpot_est_s": 0.03721861124053717, + "median_e2e_s": 10.32500390149653, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.779329141136259, + "output_tok_s": 26.782001452523673, + "median_first_sse_s": 0.06421606685034931, + "median_tpot_est_s": 0.03712567288289213, + "median_e2e_s": 4.77917652297765, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.179921305039898, + "output_tok_s": 26.69458293686857, + "median_first_sse_s": 7.2513526930706576, + "median_tpot_est_s": 0.03726597728691702, + "median_e2e_s": 11.985176004120149, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 76.49607840902172, + "output_tok_s": 26.772614264608695, + "median_first_sse_s": 35.89145322609693, + "median_tpot_est_s": 0.0371851845594915, + "median_e2e_s": 40.606709040584974, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 1.409332315903157, + "output_tok_s": 22.705787441972554, + "median_first_sse_s": 0.248691565124318, + "median_tpot_est_s": 0.0374359464828646, + "median_e2e_s": 1.4092059060931206, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 4.901245624991134, + "output_tok_s": 26.115810100872377, + "median_first_sse_s": 1.9010033400263637, + "median_tpot_est_s": 0.03745433188704474, + "median_e2e_s": 3.062635939917527, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 19.62891486589797, + "output_tok_s": 26.08396865022408, + "median_first_sse_s": 9.260896492982283, + "median_tpot_est_s": 0.03751262429080182, + "median_e2e_s": 10.424593756091781, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.838200076017529, + "output_tok_s": 26.456119628967624, + "median_first_sse_s": 0.06642191391438246, + "median_tpot_est_s": 0.0375715390867077, + "median_e2e_s": 4.83800737792626, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.333293949021026, + "output_tok_s": 26.482812569346258, + "median_first_sse_s": 7.3223171149147674, + "median_tpot_est_s": 0.03758554547607136, + "median_e2e_s": 12.085917299962603, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 77.24892303696834, + "output_tok_s": 26.51169646753401, + "median_first_sse_s": 36.27493576845154, + "median_tpot_est_s": 0.03750518164927448, + "median_e2e_s": 41.03906974289566, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.0401992758270353, + "output_tok_s": 15.684742357840591, + "median_first_sse_s": 0.8407348680775613, + "median_tpot_est_s": 0.03868695670708774, + "median_e2e_s": 2.040030525997281, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 5.082977191079408, + "output_tok_s": 25.182092145650223, + "median_first_sse_s": 1.9748041100101545, + "median_tpot_est_s": 0.038678585807041775, + "median_e2e_s": 3.1738402700284496, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 20.373157310998067, + "output_tok_s": 25.13110718109492, + "median_first_sse_s": 9.618916884530336, + "median_tpot_est_s": 0.03872891262759484, + "median_e2e_s": 10.823435940546915, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.998283860972151, + "output_tok_s": 25.608789648674414, + "median_first_sse_s": 0.07471933006308973, + "median_tpot_est_s": 0.038767074181617595, + "median_e2e_s": 4.9981377511285245, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.97290690196678, + "output_tok_s": 25.63472620750974, + "median_first_sse_s": 7.558827344560996, + "median_tpot_est_s": 0.038741515790786975, + "median_e2e_s": 12.481202929164283, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 79.82538939104415, + "output_tok_s": 25.655997617091124, + "median_first_sse_s": 37.48764609743375, + "median_tpot_est_s": 0.0387145414293971, + "median_e2e_s": 42.39941652864218, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 5.352265110006556, + "output_tok_s": 5.978777086391523, + "median_first_sse_s": 4.025504268007353, + "median_tpot_est_s": 0.04279371299931119, + "median_e2e_s": 5.352109370986, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 5.7538805468939245, + "output_tok_s": 22.24585633240115, + "median_first_sse_s": 2.2709930365672335, + "median_tpot_est_s": 0.04280944161056992, + "median_e2e_s": 3.5957872320432216, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 23.044004359981045, + "output_tok_s": 22.218360663442486, + "median_first_sse_s": 10.923830434447154, + "median_tpot_est_s": 0.042838023904140196, + "median_e2e_s": 12.250851112417877, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.550273383036256, + "output_tok_s": 23.06192707393777, + "median_first_sse_s": 0.11678028688766062, + "median_tpot_est_s": 0.04278182463104448, + "median_e2e_s": 5.55007201503031, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 22.190961834043264, + "output_tok_s": 23.07245642748744, + "median_first_sse_s": 8.439156876062043, + "median_tpot_est_s": 0.0427968583077075, + "median_e2e_s": 13.874486317043193, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 88.77933540102094, + "output_tok_s": 23.068431304977402, + "median_first_sse_s": 41.72694445645902, + "median_tpot_est_s": 0.04281781166131923, + "median_e2e_s": 47.15564054495189, + "all_text_equal": true + } + ], + "config_sha256": "fc6780d557b89f162940b8725bf11c65b306cb76c0c117ccb45fcc619d945d46", + "packages": { + "torch": "2.11.0+cpu", + "transformers": "5.8.1" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "whole": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 867456.0 + }, + "raw_sha256": "bd0bc2e81c131d9c179268afd97066316bb70bc6904a3002c670f8cf1eef79fa", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.229157143039629, + "output_tok_s": 7.566519502039735, + "median_first_sse_s": 3.07451313897036, + "median_tpot_est_s": 0.03723970416092103, + "median_e2e_s": 4.228943967958912, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 16.919324713060632, + "output_tok_s": 7.565313756357677, + "median_first_sse_s": 9.443536977516487, + "median_tpot_est_s": 0.0372378321612374, + "median_e2e_s": 10.594949356513098, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 67.71203166921623, + "output_tok_s": 7.561433136450541, + "median_first_sse_s": 34.59739960345905, + "median_tpot_est_s": 0.03715300685425679, + "median_e2e_s": 35.75067046692129, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 7.789689174853265, + "output_tok_s": 16.431977852622232, + "median_first_sse_s": 3.0535819258075207, + "median_tpot_est_s": 0.0372907919919221, + "median_e2e_s": 7.789512508781627, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 31.15392838884145, + "output_tok_s": 16.434524519976282, + "median_first_sse_s": 14.739977189921774, + "median_tpot_est_s": 0.03724773100019968, + "median_e2e_s": 19.466690970933996, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 124.65422019199468, + "output_tok_s": 16.429447770365364, + "median_first_sse_s": 61.595142140984535, + "median_tpot_est_s": 0.03722072855521756, + "median_e2e_s": 66.31608892697841, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.327635735040531, + "output_tok_s": 7.39433768440779, + "median_first_sse_s": 3.1801685569807887, + "median_tpot_est_s": 0.03700972913253692, + "median_e2e_s": 4.327470160089433, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 17.268865237943828, + "output_tok_s": 7.412183616949733, + "median_first_sse_s": 9.761258519953117, + "median_tpot_est_s": 0.037525404207107996, + "median_e2e_s": 10.924995203502476, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 67.5510564618744, + "output_tok_s": 7.579452148005578, + "median_first_sse_s": 34.785325877484865, + "median_tpot_est_s": 0.03732400219286641, + "median_e2e_s": 35.948298823903315, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 8.1633458530996, + "output_tok_s": 15.679845286892844, + "median_first_sse_s": 3.39684719289653, + "median_tpot_est_s": 0.03752971129257261, + "median_e2e_s": 8.163120527053252, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 31.40743782185018, + "output_tok_s": 16.30187100597557, + "median_first_sse_s": 14.859845734550618, + "median_tpot_est_s": 0.03752742784178456, + "median_e2e_s": 19.622518819523975, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 125.70304442220367, + "output_tok_s": 16.29236594398862, + "median_first_sse_s": 61.88206681644078, + "median_tpot_est_s": 0.03753838226323815, + "median_e2e_s": 66.65103639895096, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.603710082126781, + "output_tok_s": 6.95091555053287, + "median_first_sse_s": 3.4106448700185865, + "median_tpot_est_s": 0.03847976864105271, + "median_e2e_s": 4.6035176978912205, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 17.03182191797532, + "output_tok_s": 7.5153439612299655, + "median_first_sse_s": 9.448679500841536, + "median_tpot_est_s": 0.03850972472930387, + "median_e2e_s": 10.642480967449956, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 68.56243541510776, + "output_tok_s": 7.467646049912349, + "median_first_sse_s": 35.08746871794574, + "median_tpot_est_s": 0.03867314664662004, + "median_e2e_s": 36.28633626399096, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 8.001783073879778, + "output_tok_s": 15.996434646901442, + "median_first_sse_s": 3.0788507510442287, + "median_tpot_est_s": 0.03876198732338142, + "median_e2e_s": 8.001623141113669, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 32.082207593135536, + "output_tok_s": 15.959001527985562, + "median_first_sse_s": 15.150795516441576, + "median_tpot_est_s": 0.03876264726809101, + "median_e2e_s": 20.078690478461795, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 128.25819011521526, + "output_tok_s": 15.967791204290867, + "median_first_sse_s": 63.30086616508197, + "median_tpot_est_s": 0.038750230264116635, + "median_e2e_s": 68.22345729661174, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 5.651906156912446, + "output_tok_s": 5.661806673994944, + "median_first_sse_s": 4.324762323871255, + "median_tpot_est_s": 0.04280508706916965, + "median_e2e_s": 5.651720023015514, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 17.66353549505584, + "output_tok_s": 7.246567372416932, + "median_first_sse_s": 9.705989120993763, + "median_tpot_est_s": 0.04286090721331176, + "median_e2e_s": 11.035693836514838, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 70.67930217203684, + "output_tok_s": 7.24398776255271, + "median_first_sse_s": 36.408837280469015, + "median_tpot_est_s": 0.042833625098630305, + "median_e2e_s": 37.739714586525224, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 8.460688102990389, + "output_tok_s": 15.128793124374722, + "median_first_sse_s": 3.0657620751298964, + "median_tpot_est_s": 0.04247847278639087, + "median_e2e_s": 8.460528119001538, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 34.039199938997626, + "output_tok_s": 15.041481612892374, + "median_first_sse_s": 15.812518983031623, + "median_tpot_est_s": 0.04263964579825649, + "median_e2e_s": 21.22309566254262, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 137.80537095014006, + "output_tok_s": 14.86153976350454, + "median_first_sse_s": 67.86479224008508, + "median_tpot_est_s": 0.042862537114396984, + "median_e2e_s": 73.3110099316109, + "all_text_equal": true + } + ], + "config_sha256": "12bbf46aa3cfdcf821075371abd139f61397757f90a513617cdc9fac282dcbf5", + "packages": { + "torch": "2.11.0+cpu", + "transformers": "5.8.1" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "stream": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 867456.0 + }, + "raw_sha256": "807e9f39105b665672eab4afbe0c35e3c4759c728ad4243cefdc70bffa5c9c6b", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.5126539489720017, + "output_tok_s": 9.109921006982479, + "median_first_sse_s": 2.357954327017069, + "median_tpot_est_s": 0.037243864131009864, + "median_e2e_s": 3.5125141150783747, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 14.053763817995787, + "output_tok_s": 9.107880398281386, + "median_first_sse_s": 7.634039894444868, + "median_tpot_est_s": 0.03725226656458671, + "median_e2e_s": 8.789965662406757, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 56.79622251703404, + "output_tok_s": 9.014684028439452, + "median_first_sse_s": 28.73849768063519, + "median_tpot_est_s": 0.037373773224141094, + "median_e2e_s": 29.89875908952672, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 7.1622115729842335, + "output_tok_s": 17.871574819545167, + "median_first_sse_s": 2.415262032067403, + "median_tpot_est_s": 0.03737608453544869, + "median_e2e_s": 7.1620247680693865, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 28.278648821869865, + "output_tok_s": 18.105532666187163, + "median_first_sse_s": 12.956028542481363, + "median_tpot_est_s": 0.03734197042603779, + "median_e2e_s": 17.682385004940443, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 113.36889456398785, + "output_tok_s": 18.064919904851543, + "median_first_sse_s": 55.33246057049837, + "median_tpot_est_s": 0.03726828140463299, + "median_e2e_s": 60.070664858445525, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.539515095995739, + "output_tok_s": 9.040786416252798, + "median_first_sse_s": 2.375210168072954, + "median_tpot_est_s": 0.03755417258118189, + "median_e2e_s": 3.5393895180895925, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 14.154375656042248, + "output_tok_s": 9.043139952651963, + "median_first_sse_s": 7.684967851499096, + "median_tpot_est_s": 0.03737933471077873, + "median_e2e_s": 8.843727227533236, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 56.62839795812033, + "output_tok_s": 9.041400047704878, + "median_first_sse_s": 29.140717443544418, + "median_tpot_est_s": 0.037329213949100624, + "median_e2e_s": 30.300555902533233, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 7.042883754009381, + "output_tok_s": 18.174373519530548, + "median_first_sse_s": 2.2993753449991345, + "median_tpot_est_s": 0.03734899102343525, + "median_e2e_s": 7.042697204975411, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 28.71817106497474, + "output_tok_s": 17.82843339297625, + "median_first_sse_s": 13.322041620849632, + "median_tpot_est_s": 0.03748940284671922, + "median_e2e_s": 18.08985018741805, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 113.7767394981347, + "output_tok_s": 18.00016426058312, + "median_first_sse_s": 55.707091104472056, + "median_tpot_est_s": 0.03747298275615551, + "median_e2e_s": 60.47448480396997, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.685936684953049, + "output_tok_s": 8.68164668444586, + "median_first_sse_s": 2.4918664928991348, + "median_tpot_est_s": 0.038512731196298715, + "median_e2e_s": 3.685761159984395, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 14.054411371937022, + "output_tok_s": 9.107460754676818, + "median_first_sse_s": 7.50499695003964, + "median_tpot_est_s": 0.03866276983922768, + "median_e2e_s": 8.704841685015708, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 55.85377869592048, + "output_tok_s": 9.166792506330392, + "median_first_sse_s": 28.488389546517283, + "median_tpot_est_s": 0.03864750609312567, + "median_e2e_s": 29.683132148580626, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 7.289653543150052, + "output_tok_s": 17.55913353663826, + "median_first_sse_s": 2.3522673710249364, + "median_tpot_est_s": 0.03887548847270646, + "median_e2e_s": 7.289454407058656, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 29.20872266800143, + "output_tok_s": 17.5290102829763, + "median_first_sse_s": 13.483660011435859, + "median_tpot_est_s": 0.038700744814943434, + "median_e2e_s": 18.39187702606432, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 115.28838631813414, + "output_tok_s": 17.76414837092626, + "median_first_sse_s": 56.357253332971595, + "median_tpot_est_s": 0.03870382958722455, + "median_e2e_s": 61.261467059957795, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.428202794166282, + "output_tok_s": 7.226407978007879, + "median_first_sse_s": 3.101593024097383, + "median_tpot_est_s": 0.04278855993893118, + "median_e2e_s": 4.4280383822042495, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 14.548575706081465, + "output_tok_s": 8.798112102925277, + "median_first_sse_s": 7.8112124215113, + "median_tpot_est_s": 0.042900833387618825, + "median_e2e_s": 9.141138256527483, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 58.156303505878896, + "output_tok_s": 8.803860787820584, + "median_first_sse_s": 29.476370697491802, + "median_tpot_est_s": 0.04284157726027432, + "median_e2e_s": 30.802679410437122, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 7.745887920958921, + "output_tok_s": 16.52489699129986, + "median_first_sse_s": 2.3028251989744604, + "median_tpot_est_s": 0.04285746511057254, + "median_e2e_s": 7.745723268017173, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 30.992978358874097, + "output_tok_s": 16.51987085821331, + "median_first_sse_s": 13.941098610986955, + "median_tpot_est_s": 0.04283978970534366, + "median_e2e_s": 19.382869886001572, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 124.24822813295759, + "output_tok_s": 16.48313244200507, + "median_first_sse_s": 60.76766407804098, + "median_tpot_est_s": 0.04279533552288217, + "median_e2e_s": 66.20954955206253, + "all_text_equal": true + } + ], + "config_sha256": "45e0beaeb76c678614cb4e084fbc9d9317827f1e8acc24e3b51d0f5a39742a50", + "packages": { + "torch": "2.11.0+cpu", + "transformers": "5.8.1" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + } + }, + "reference_equality": { + "whole/stream": { + "512:32": true, + "512:128": true, + "2048:32": true, + "2048:128": true, + "128:32": true, + "128:128": true, + "8192:32": true, + "8192:128": true + }, + "native/whole": { + "512:32": true, + "512:128": false, + "2048:32": true, + "2048:128": false, + "128:32": true, + "128:128": true, + "8192:32": true, + "8192:128": true + }, + "native/stream": { + "512:32": true, + "512:128": false, + "2048:32": true, + "2048:128": false, + "128:32": true, + "128:128": true, + "8192:32": true, + "8192:128": true + } + }, + "limits": [ + "Historical same-route off references; not a fresh cache off/on performance pair.", + "One ATOM execution lane; concurrency denotes queued clients. PD adds one RNGD.", + "Single ordered runs, not stable speedup estimates.", + "Native/PD pre-EOS differences retained; no broad engine equivalence." + ], + "source_sha256": { + "benchmark267_deadline.py": "c8ebb077a608e3da70a9748aed19c03f22ae87355f0167a2fb1e68b88ec21d79", + "atom_server265_http.py": "f222ebe20de1b7e6d4c233907ff41a248694c01d75382dd8be2bccaad53a7300", + "llmd_mid201_1030.py": "e3449979b88d03edb6d5bf0d402a708e5d98054fdf203e22b480f3d7e6fd759a", + "container_bootstrap.py": "9225577d93008057d4dd7ff93364cd6c6fb37aa8ce40aa819d6f01a5399d054e", + "run-http267-deadline-atom.sh": "a33b0d7643e89eee54b76717b6c85dceedf4ad1d5e7bc2bc5552507826346b3c", + "furiosa_prefill_python.sh": "317036f016a063d4495f8d81af9523b327a95a52bc7d1d740d664e5f197a0edc", + "streaming_factories.py": "77745350973cc54bf508e0c8b473efc442afef20b1c64f7c4edc179ea1471389", + "prefill.py": "53dfd7123d0e7593cfb9073bcd125c92af267204fc480b06ef615cf58f4b5cbd", + "long_width_variant.py": "74849665643d87b6d44f86c64849c4f532c7dedff9880404950a2692ec443c84", + "batching.py": "d98a51d75a36cd8c379a6220adb92528b3607cf80145b85a032270494a429580", + "pd_batching.py": "c827775c1d399bf724a9747c3424ee5ea69f93054493bfbb71d5f0cf08fff65b", + "llmd_backlog267_1030.py": "9f3aaf0902f7b1958f32be12b1f276384f20cc3e0070571af10bb7bce148c1f0", + "atom_cache201_http.py": "e47eb03234fa4d5bac425351f22397e6cd9a5aad730f527a444ed4aac1b241b3" + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-native-whole-summary.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-native-whole-summary.json new file mode 100644 index 00000000..79d15ed9 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-native-whole-summary.json @@ -0,0 +1,2046 @@ +{ + "model": "Qwen3-4B", + "decoder_cache": true, + "arms": { + "native": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 867456.0 + }, + "raw_sha256": "f6b0ed35f371a37e6460adebb51eca0af569b1b2fb061f199896d63b0d8b91a5", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 1.21979605499655, + "output_tok_s": 26.233893665191847, + "median_first_sse_s": 0.06424786592833698, + "median_tpot_est_s": 0.03727080548302301, + "median_e2e_s": 1.2196428359020501, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 4.865430230041966, + "output_tok_s": 26.308053748187437, + "median_first_sse_s": 1.8862478689989075, + "median_tpot_est_s": 0.03721824728868782, + "median_e2e_s": 3.0406833300366998, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 19.443709495011717, + "output_tok_s": 26.332423868570633, + "median_first_sse_s": 9.170724456431344, + "median_tpot_est_s": 0.03721861124053717, + "median_e2e_s": 10.32500390149653, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.779329141136259, + "output_tok_s": 26.782001452523673, + "median_first_sse_s": 0.06421606685034931, + "median_tpot_est_s": 0.03712567288289213, + "median_e2e_s": 4.77917652297765, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.179921305039898, + "output_tok_s": 26.69458293686857, + "median_first_sse_s": 7.2513526930706576, + "median_tpot_est_s": 0.03726597728691702, + "median_e2e_s": 11.985176004120149, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 76.49607840902172, + "output_tok_s": 26.772614264608695, + "median_first_sse_s": 35.89145322609693, + "median_tpot_est_s": 0.0371851845594915, + "median_e2e_s": 40.606709040584974, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 1.409332315903157, + "output_tok_s": 22.705787441972554, + "median_first_sse_s": 0.248691565124318, + "median_tpot_est_s": 0.0374359464828646, + "median_e2e_s": 1.4092059060931206, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 4.901245624991134, + "output_tok_s": 26.115810100872377, + "median_first_sse_s": 1.9010033400263637, + "median_tpot_est_s": 0.03745433188704474, + "median_e2e_s": 3.062635939917527, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 19.62891486589797, + "output_tok_s": 26.08396865022408, + "median_first_sse_s": 9.260896492982283, + "median_tpot_est_s": 0.03751262429080182, + "median_e2e_s": 10.424593756091781, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.838200076017529, + "output_tok_s": 26.456119628967624, + "median_first_sse_s": 0.06642191391438246, + "median_tpot_est_s": 0.0375715390867077, + "median_e2e_s": 4.83800737792626, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.333293949021026, + "output_tok_s": 26.482812569346258, + "median_first_sse_s": 7.3223171149147674, + "median_tpot_est_s": 0.03758554547607136, + "median_e2e_s": 12.085917299962603, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 77.24892303696834, + "output_tok_s": 26.51169646753401, + "median_first_sse_s": 36.27493576845154, + "median_tpot_est_s": 0.03750518164927448, + "median_e2e_s": 41.03906974289566, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.0401992758270353, + "output_tok_s": 15.684742357840591, + "median_first_sse_s": 0.8407348680775613, + "median_tpot_est_s": 0.03868695670708774, + "median_e2e_s": 2.040030525997281, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 5.082977191079408, + "output_tok_s": 25.182092145650223, + "median_first_sse_s": 1.9748041100101545, + "median_tpot_est_s": 0.038678585807041775, + "median_e2e_s": 3.1738402700284496, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 20.373157310998067, + "output_tok_s": 25.13110718109492, + "median_first_sse_s": 9.618916884530336, + "median_tpot_est_s": 0.03872891262759484, + "median_e2e_s": 10.823435940546915, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.998283860972151, + "output_tok_s": 25.608789648674414, + "median_first_sse_s": 0.07471933006308973, + "median_tpot_est_s": 0.038767074181617595, + "median_e2e_s": 4.9981377511285245, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.97290690196678, + "output_tok_s": 25.63472620750974, + "median_first_sse_s": 7.558827344560996, + "median_tpot_est_s": 0.038741515790786975, + "median_e2e_s": 12.481202929164283, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 79.82538939104415, + "output_tok_s": 25.655997617091124, + "median_first_sse_s": 37.48764609743375, + "median_tpot_est_s": 0.0387145414293971, + "median_e2e_s": 42.39941652864218, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 5.352265110006556, + "output_tok_s": 5.978777086391523, + "median_first_sse_s": 4.025504268007353, + "median_tpot_est_s": 0.04279371299931119, + "median_e2e_s": 5.352109370986, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 5.7538805468939245, + "output_tok_s": 22.24585633240115, + "median_first_sse_s": 2.2709930365672335, + "median_tpot_est_s": 0.04280944161056992, + "median_e2e_s": 3.5957872320432216, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 23.044004359981045, + "output_tok_s": 22.218360663442486, + "median_first_sse_s": 10.923830434447154, + "median_tpot_est_s": 0.042838023904140196, + "median_e2e_s": 12.250851112417877, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.550273383036256, + "output_tok_s": 23.06192707393777, + "median_first_sse_s": 0.11678028688766062, + "median_tpot_est_s": 0.04278182463104448, + "median_e2e_s": 5.55007201503031, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 22.190961834043264, + "output_tok_s": 23.07245642748744, + "median_first_sse_s": 8.439156876062043, + "median_tpot_est_s": 0.0427968583077075, + "median_e2e_s": 13.874486317043193, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 88.77933540102094, + "output_tok_s": 23.068431304977402, + "median_first_sse_s": 41.72694445645902, + "median_tpot_est_s": 0.04281781166131923, + "median_e2e_s": 47.15564054495189, + "all_text_equal": true + } + ], + "config_sha256": "fc6780d557b89f162940b8725bf11c65b306cb76c0c117ccb45fcc619d945d46", + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "whole": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 867456.0 + }, + "raw_sha256": "bd0bc2e81c131d9c179268afd97066316bb70bc6904a3002c670f8cf1eef79fa", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.229157143039629, + "output_tok_s": 7.566519502039735, + "median_first_sse_s": 3.07451313897036, + "median_tpot_est_s": 0.03723970416092103, + "median_e2e_s": 4.228943967958912, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 16.919324713060632, + "output_tok_s": 7.565313756357677, + "median_first_sse_s": 9.443536977516487, + "median_tpot_est_s": 0.0372378321612374, + "median_e2e_s": 10.594949356513098, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 67.71203166921623, + "output_tok_s": 7.561433136450541, + "median_first_sse_s": 34.59739960345905, + "median_tpot_est_s": 0.03715300685425679, + "median_e2e_s": 35.75067046692129, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 7.789689174853265, + "output_tok_s": 16.431977852622232, + "median_first_sse_s": 3.0535819258075207, + "median_tpot_est_s": 0.0372907919919221, + "median_e2e_s": 7.789512508781627, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 31.15392838884145, + "output_tok_s": 16.434524519976282, + "median_first_sse_s": 14.739977189921774, + "median_tpot_est_s": 0.03724773100019968, + "median_e2e_s": 19.466690970933996, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 124.65422019199468, + "output_tok_s": 16.429447770365364, + "median_first_sse_s": 61.595142140984535, + "median_tpot_est_s": 0.03722072855521756, + "median_e2e_s": 66.31608892697841, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.327635735040531, + "output_tok_s": 7.39433768440779, + "median_first_sse_s": 3.1801685569807887, + "median_tpot_est_s": 0.03700972913253692, + "median_e2e_s": 4.327470160089433, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 17.268865237943828, + "output_tok_s": 7.412183616949733, + "median_first_sse_s": 9.761258519953117, + "median_tpot_est_s": 0.037525404207107996, + "median_e2e_s": 10.924995203502476, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 67.5510564618744, + "output_tok_s": 7.579452148005578, + "median_first_sse_s": 34.785325877484865, + "median_tpot_est_s": 0.03732400219286641, + "median_e2e_s": 35.948298823903315, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 8.1633458530996, + "output_tok_s": 15.679845286892844, + "median_first_sse_s": 3.39684719289653, + "median_tpot_est_s": 0.03752971129257261, + "median_e2e_s": 8.163120527053252, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 31.40743782185018, + "output_tok_s": 16.30187100597557, + "median_first_sse_s": 14.859845734550618, + "median_tpot_est_s": 0.03752742784178456, + "median_e2e_s": 19.622518819523975, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 125.70304442220367, + "output_tok_s": 16.29236594398862, + "median_first_sse_s": 61.88206681644078, + "median_tpot_est_s": 0.03753838226323815, + "median_e2e_s": 66.65103639895096, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.603710082126781, + "output_tok_s": 6.95091555053287, + "median_first_sse_s": 3.4106448700185865, + "median_tpot_est_s": 0.03847976864105271, + "median_e2e_s": 4.6035176978912205, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 17.03182191797532, + "output_tok_s": 7.5153439612299655, + "median_first_sse_s": 9.448679500841536, + "median_tpot_est_s": 0.03850972472930387, + "median_e2e_s": 10.642480967449956, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 68.56243541510776, + "output_tok_s": 7.467646049912349, + "median_first_sse_s": 35.08746871794574, + "median_tpot_est_s": 0.03867314664662004, + "median_e2e_s": 36.28633626399096, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 8.001783073879778, + "output_tok_s": 15.996434646901442, + "median_first_sse_s": 3.0788507510442287, + "median_tpot_est_s": 0.03876198732338142, + "median_e2e_s": 8.001623141113669, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 32.082207593135536, + "output_tok_s": 15.959001527985562, + "median_first_sse_s": 15.150795516441576, + "median_tpot_est_s": 0.03876264726809101, + "median_e2e_s": 20.078690478461795, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 128.25819011521526, + "output_tok_s": 15.967791204290867, + "median_first_sse_s": 63.30086616508197, + "median_tpot_est_s": 0.038750230264116635, + "median_e2e_s": 68.22345729661174, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 5.651906156912446, + "output_tok_s": 5.661806673994944, + "median_first_sse_s": 4.324762323871255, + "median_tpot_est_s": 0.04280508706916965, + "median_e2e_s": 5.651720023015514, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 17.66353549505584, + "output_tok_s": 7.246567372416932, + "median_first_sse_s": 9.705989120993763, + "median_tpot_est_s": 0.04286090721331176, + "median_e2e_s": 11.035693836514838, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 70.67930217203684, + "output_tok_s": 7.24398776255271, + "median_first_sse_s": 36.408837280469015, + "median_tpot_est_s": 0.042833625098630305, + "median_e2e_s": 37.739714586525224, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 8.460688102990389, + "output_tok_s": 15.128793124374722, + "median_first_sse_s": 3.0657620751298964, + "median_tpot_est_s": 0.04247847278639087, + "median_e2e_s": 8.460528119001538, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 34.039199938997626, + "output_tok_s": 15.041481612892374, + "median_first_sse_s": 15.812518983031623, + "median_tpot_est_s": 0.04263964579825649, + "median_e2e_s": 21.22309566254262, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 137.80537095014006, + "output_tok_s": 14.86153976350454, + "median_first_sse_s": 67.86479224008508, + "median_tpot_est_s": 0.042862537114396984, + "median_e2e_s": 73.3110099316109, + "all_text_equal": true + } + ], + "config_sha256": "12bbf46aa3cfdcf821075371abd139f61397757f90a513617cdc9fac282dcbf5", + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + } + }, + "reference_comparison": { + "512:32": { + "tokens_equal": true, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 1083, + 3897, + 264 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 1083, + 3897, + 264 + ], + "first_different_generated_index": null + }, + "512:128": { + "tokens_equal": false, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 1083, + 3897, + 264, + 15493, + 315, + 11504, + 4004, + 2266, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2938, + 4977, + 45205, + 13, + 10696, + 807, + 4829, + 752, + 311, + 990, + 429, + 2266, + 369, + 279, + 3405, + 30, + 1988, + 279, + 3405, + 572, + 30339, + 13, + 14568, + 358, + 3116, + 2494, + 1939, + 10061, + 752, + 1779, + 1549, + 13, + 576, + 1196, + 594, + 3405, + 572, + 330, + 3838, + 374, + 279, + 6722, + 315, + 9625, + 7521, + 323, + 279, + 4226, + 374, + 12095, + 13, + 576, + 4004, + 2266, + 911, + 14697, + 374, + 4658, + 1101, + 4960, + 1995, + 11, + 7196, + 264, + 1273, + 311, + 1490, + 421, + 358, + 646, + 5244, + 389, + 279, + 5042, + 3405, + 13, + 2521, + 7196, + 432, + 594, + 949, + 315, + 264, + 8131 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 1083, + 3897, + 264, + 15493, + 315, + 11504, + 4004, + 2266, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2938, + 4977, + 45205, + 13, + 10696, + 807, + 2299, + 7497, + 421, + 358, + 646, + 10034, + 39715, + 1995, + 30, + 2521, + 7196, + 1052, + 594, + 264, + 14068, + 1588, + 13, + 6771, + 752, + 1744, + 13, + 576, + 4004, + 2266, + 374, + 1101, + 11504, + 5128, + 911, + 14697, + 11, + 714, + 279, + 3405, + 374, + 911, + 279, + 6722, + 315, + 9625, + 13, + 8704, + 279, + 3405, + 374, + 30339, + 11, + 279, + 4226, + 1265, + 387, + 12095, + 15484, + 315, + 279, + 2266, + 13, + 576, + 14697, + 949, + 374, + 4658, + 1101, + 1052, + 311, + 1490, + 421, + 358, + 633, + 48704, + 13, + 358, + 1265, + 1281, + 2704, + 537, + 311, + 1077 + ], + "first_different_generated_index": 50, + "eos_ids": [ + 151645, + 151643 + ], + "native_first_eos_index": null, + "whole_first_eos_index": null, + "difference_before_both_eos": true + }, + "2048:32": { + "tokens_equal": true, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "first_different_generated_index": null + }, + "2048:128": { + "tokens_equal": false, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493, + 315, + 11504, + 4004, + 2266, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2938, + 4977, + 45205, + 13, + 10696, + 807, + 2299, + 7497, + 421, + 358, + 646, + 10034, + 39715, + 1995, + 30, + 2521, + 8365, + 807, + 1366, + 311, + 1490, + 421, + 358, + 646, + 5244, + 389, + 279, + 5042, + 3405, + 8818, + 279, + 76449, + 13, + 6771, + 752, + 1281, + 2704, + 358, + 3207, + 944, + 3116, + 4113, + 13, + 576, + 4004, + 2266, + 374, + 1101, + 911, + 14697, + 11, + 714, + 279, + 3405, + 374, + 30339, + 13, + 358, + 1265, + 7683, + 429, + 847, + 4226, + 374, + 4396, + 13, + 12095, + 374, + 12824, + 279, + 6722, + 315, + 9625, + 13, + 576, + 14697, + 949, + 374, + 4658, + 1101, + 1052, + 311, + 1490, + 421, + 358 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493, + 315, + 11504, + 4004, + 2266, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2938, + 4977, + 45205, + 13, + 10696, + 807, + 2299, + 7497, + 421, + 358, + 646, + 10034, + 39715, + 1995, + 30, + 2521, + 8365, + 807, + 5230, + 429, + 438, + 949, + 315, + 264, + 15048, + 1380, + 358, + 1184, + 311, + 5244, + 389, + 3151, + 3565, + 13, + 1988, + 279, + 3405, + 374, + 30339, + 13, + 576, + 4226, + 374, + 12095, + 11, + 15484, + 315, + 279, + 2266, + 13, + 358, + 1265, + 1281, + 2704, + 537, + 311, + 387, + 21815, + 553, + 279, + 11504, + 1995, + 13, + 4599, + 4226, + 279, + 3405, + 3118, + 389, + 4586, + 6540, + 13, + 84194, + 11, + 12095, + 374, + 279, + 6722, + 315, + 9625, + 13, + 2308, + 1184, + 311, + 916 + ], + "first_different_generated_index": 61, + "eos_ids": [ + 151645, + 151643 + ], + "native_first_eos_index": null, + "whole_first_eos_index": null, + "difference_before_both_eos": true + }, + "128:32": { + "tokens_equal": true, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "first_different_generated_index": null + }, + "128:128": { + "tokens_equal": true, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518, + 279, + 2266, + 2661, + 11, + 432, + 594, + 678, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2619, + 594, + 902, + 6286, + 315, + 9625, + 11, + 92999, + 11, + 476, + 894, + 5435, + 1995, + 13, + 4710, + 80022, + 11, + 7196, + 279, + 1196, + 374, + 7497, + 421, + 358, + 646, + 10034, + 279, + 39715, + 2266, + 323, + 4226, + 279, + 5042, + 3405, + 13, + 2521, + 8365, + 807, + 5230, + 279, + 2266, + 553, + 16523, + 13, + 20988, + 1616, + 11, + 279, + 4226, + 1265, + 387, + 12095, + 13, + 576, + 2266, + 911, + 14697, + 3171, + 944, + 7802, + 279, + 3405, + 13, + 358, + 1184, + 311, + 1281, + 2704, + 358, + 1513, + 944, + 633, + 21815, + 553, + 279, + 14590, + 1995, + 323, + 1101, + 5244, + 389, + 279 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518, + 279, + 2266, + 2661, + 11, + 432, + 594, + 678, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2619, + 594, + 902, + 6286, + 315, + 9625, + 11, + 92999, + 11, + 476, + 894, + 5435, + 1995, + 13, + 4710, + 80022, + 11, + 7196, + 279, + 1196, + 374, + 7497, + 421, + 358, + 646, + 10034, + 279, + 39715, + 2266, + 323, + 4226, + 279, + 5042, + 3405, + 13, + 2521, + 8365, + 807, + 5230, + 279, + 2266, + 553, + 16523, + 13, + 20988, + 1616, + 11, + 279, + 4226, + 1265, + 387, + 12095, + 13, + 576, + 2266, + 911, + 14697, + 3171, + 944, + 7802, + 279, + 3405, + 13, + 358, + 1184, + 311, + 1281, + 2704, + 358, + 1513, + 944, + 633, + 21815, + 553, + 279, + 14590, + 1995, + 323, + 1101, + 5244, + 389, + 279 + ], + "first_different_generated_index": null + }, + "8192:32": { + "tokens_equal": true, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "first_different_generated_index": null + }, + "8192:128": { + "tokens_equal": true, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518, + 279, + 2266, + 2661, + 11, + 432, + 594, + 678, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2619, + 594, + 902, + 6286, + 315, + 9625, + 11, + 92999, + 11, + 476, + 894, + 5435, + 1995, + 13, + 4710, + 80022, + 11, + 279, + 1196, + 2578, + 614, + 3381, + 429, + 279, + 3681, + 2266, + 572, + 9760, + 311, + 862, + 3405, + 11, + 714, + 432, + 594, + 3520, + 45205, + 13, + 10696, + 807, + 1033, + 7497, + 421, + 358, + 646, + 15282, + 429, + 279, + 2266, + 3171, + 944, + 3410, + 894, + 1995, + 911, + 279, + 6722, + 315, + 9625, + 13, + 2521, + 8365, + 807, + 1101, + 4829, + 311, + 1490, + 421, + 358, + 1410, + 4226, + 264, + 30339, + 3405, + 8818, + 279, + 39715, + 2266, + 382, + 40, + 1265, + 1281 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518, + 279, + 2266, + 2661, + 11, + 432, + 594, + 678, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2619, + 594, + 902, + 6286, + 315, + 9625, + 11, + 92999, + 11, + 476, + 894, + 5435, + 1995, + 13, + 4710, + 80022, + 11, + 279, + 1196, + 2578, + 614, + 3381, + 429, + 279, + 3681, + 2266, + 572, + 9760, + 311, + 862, + 3405, + 11, + 714, + 432, + 594, + 3520, + 45205, + 13, + 10696, + 807, + 1033, + 7497, + 421, + 358, + 646, + 15282, + 429, + 279, + 2266, + 3171, + 944, + 3410, + 894, + 1995, + 911, + 279, + 6722, + 315, + 9625, + 13, + 2521, + 8365, + 807, + 1101, + 4829, + 311, + 1490, + 421, + 358, + 1410, + 4226, + 264, + 30339, + 3405, + 8818, + 279, + 39715, + 2266, + 382, + 40, + 1265, + 1281 + ], + "first_different_generated_index": null + } + }, + "limits": [ + "Each arm uses same-route historical off references; not a fresh cache off/on timing comparison.", + "One ATOM execution lane; concurrency is queued clients, not hardware batch size.", + "PD adds one RNGD research producer to the native single-ATOM configuration; not an equal-resource speedup test.", + "Single ordered runs; no repeatability claim. Same-route correctness does not imply native-PD equivalence." + ], + "packages": { + "native": { + "torch": "2.11.0+cpu", + "transformers": "5.8.1" + }, + "whole": { + "torch": "2.11.0+cpu", + "transformers": "5.8.1" + } + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-backlog-controls.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-backlog-controls.json new file mode 100644 index 00000000..894efb83 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-backlog-controls.json @@ -0,0 +1,401 @@ +{ + "model": "Qwen/Qwen3-4B", + "base_commit": "f52c41305f6bb2ee65d3c0fe8166acbca51dcc72", + "candidate_http_sha256": "9f3aaf0902f7b1958f32be12b1f276384f20cc3e0070571af10bb7bce148c1f0", + "hardware": "A10080GB GPU decoder plus one RNGD research prefiller", + "decoder": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "producer": { + "torch": "2.10.0+cpu", + "transformers": "5.8.1", + "furiosa-torch": "2026.4.0" + }, + "configuration": { + "inputs": [ + 128, + 512, + 2048, + 8192 + ], + "outputs": [ + 32, + 128 + ], + "concurrency": [ + 1, + 4, + 16 + ], + "gpu_slots": 4, + "context": 9216, + "page_size": 256, + "local_chunk": 256, + "producer_width": 8192, + "producer_cache": false, + "decoder_cache": false, + "repeats": 1 + }, + "before": { + "status": "failed_not_passed", + "raw_sha256": "e8c4e2fd391bbab5ae82c9d65bfbcab4fc922e9f5cd208c4979695ca31caa134", + "error": "ConnectionResetError(104, 'Connection reset by peer')", + "failed_condition": { + "input_tokens": 2048, + "output_tokens": 128, + "concurrency": 16, + "stream": false, + "elapsed_s": 47.106650558067486 + }, + "completed_warmups": 8, + "completed_token_batches": 17, + "completed_token_responses": 110, + "completed_token_responses_equal": true, + "timed_batches": 0, + "server_alive_at_failure": true, + "host_listen_counter_delta": { + "ListenOverflows": 36, + "ListenDrops": 36 + }, + "limits": "Host-wide listen counters do not identify a particular socket. Candidate backlog fix requires live retry; reset root cause not proven." + }, + "after": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 0.0, + "transferred_tokens": 935680.0 + }, + "raw_sha256": "c8100535bfc02fe0b532c39323cb23d8362026fa48cedea19684a277565fcf18", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.5938640710664913, + "output_tok_s": 8.90406519757546, + "median_first_sse_s": 2.769333482021466, + "median_tpot_est_s": 0.02659360800058611, + "median_e2e_s": 3.5937353300396353, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 11.845253351028077, + "output_tok_s": 10.80601623340463, + "median_first_sse_s": 6.894638849480543, + "median_tpot_est_s": 0.026379545144135912, + "median_e2e_s": 7.7222023899666965, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.374117970932275, + "output_tok_s": 11.040641254264422, + "median_first_sse_s": 24.28165605297545, + "median_tpot_est_s": 0.026395979968290177, + "median_e2e_s": 25.099912968988065, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.118984114029445, + "output_tok_s": 20.918505035259855, + "median_first_sse_s": 2.848816549987532, + "median_tpot_est_s": 0.025747966134048058, + "median_e2e_s": 6.118808249011636, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.784803000977263, + "output_tok_s": 34.630153676458, + "median_first_sse_s": 7.131008746451698, + "median_tpot_est_s": 0.027017266000071028, + "median_e2e_s": 10.569683398469351, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.64825740107335, + "output_tok_s": 42.0981163439333, + "median_first_sse_s": 24.06720970547758, + "median_tpot_est_s": 0.026488267799645047, + "median_e2e_s": 27.375394922448322, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.636839246028103, + "output_tok_s": 8.798849175131433, + "median_first_sse_s": 2.866227310965769, + "median_tpot_est_s": 0.024852504938720696, + "median_e2e_s": 3.6366549640661106, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.187823576037772, + "output_tok_s": 10.502285268688837, + "median_first_sse_s": 7.140909380977973, + "median_tpot_est_s": 0.025063636806613256, + "median_e2e_s": 7.917882121982984, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.370749281020835, + "output_tok_s": 11.041443322322104, + "median_first_sse_s": 24.260911067016423, + "median_tpot_est_s": 0.025874566645436588, + "median_e2e_s": 25.080011933052447, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.158095913007855, + "output_tok_s": 20.78564572689154, + "median_first_sse_s": 2.877790122060105, + "median_tpot_est_s": 0.025827755779394602, + "median_e2e_s": 6.15791510604322, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.883136463933624, + "output_tok_s": 34.4013509007817, + "median_first_sse_s": 7.182273587503005, + "median_tpot_est_s": 0.026408043138198847, + "median_e2e_s": 10.536095066054258, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.92640683695208, + "output_tok_s": 41.85878613209811, + "median_first_sse_s": 24.215235407522414, + "median_tpot_est_s": 0.026753554602362596, + "median_e2e_s": 27.697570833028294, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.843274821061641, + "output_tok_s": 8.326232572449902, + "median_first_sse_s": 3.003903826000169, + "median_tpot_est_s": 0.027072391161064226, + "median_e2e_s": 3.84314795199316, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.749716062913649, + "output_tok_s": 10.039439260324093, + "median_first_sse_s": 7.466653063020203, + "median_tpot_est_s": 0.025596863032513927, + "median_e2e_s": 8.284569455485325, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.98791066301055, + "output_tok_s": 10.896419797678993, + "median_first_sse_s": 24.6336495334981, + "median_tpot_est_s": 0.0250324510477273, + "median_e2e_s": 25.404889553028625, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.079132216051221, + "output_tok_s": 21.055636799941826, + "median_first_sse_s": 2.9262310749618337, + "median_tpot_est_s": 0.024825054519685117, + "median_e2e_s": 6.0790129989618436, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.522591115906835, + "output_tok_s": 35.255416606696166, + "median_first_sse_s": 6.977572105941363, + "median_tpot_est_s": 0.02599662249196526, + "median_e2e_s": 10.286594730452634, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.192781265010126, + "output_tok_s": 41.63212461940433, + "median_first_sse_s": 24.612169308995362, + "median_tpot_est_s": 0.026611734634012306, + "median_e2e_s": 27.96816182450857, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.199458156013861, + "output_tok_s": 7.620030682809446, + "median_first_sse_s": 3.3747121680062264, + "median_tpot_est_s": 0.02660094451669964, + "median_e2e_s": 4.199341448023915, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 13.205650482908823, + "output_tok_s": 9.692820521462515, + "median_first_sse_s": 8.0374622924719, + "median_tpot_est_s": 0.026171039097056155, + "median_e2e_s": 8.848766844486818, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 51.270903732045554, + "output_tok_s": 9.986170766090625, + "median_first_sse_s": 26.803447068086825, + "median_tpot_est_s": 0.1057412954021035, + "median_e2e_s": 30.457469970046077, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.657406355952844, + "output_tok_s": 19.226706791834392, + "median_first_sse_s": 3.4630271260393783, + "median_tpot_est_s": 0.02515140170869454, + "median_e2e_s": 6.657255143043585, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 16.016934926970862, + "output_tok_s": 31.966165957123604, + "median_first_sse_s": 8.493684782530181, + "median_tpot_est_s": 0.02901831886632089, + "median_e2e_s": 12.179011278552935, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 53.753593918983825, + "output_tok_s": 38.09977809272992, + "median_first_sse_s": 27.78725230850978, + "median_tpot_est_s": 0.030791497555117703, + "median_e2e_s": 32.433394862979185, + "all_text_equal": true + } + ] + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + }, + "limitations": [ + "One hardware retry,not proof all resets are eliminated.", + "Listen counters are host-wide,not socket-specific.", + "ATOM retry and remaining GPU routes/models still running.", + "Failed original has no timed batches;no throughput speedup is claimed." + ] +} diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-full-pd-cache-summary.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-full-pd-cache-summary.json new file mode 100644 index 00000000..44a3cf2e --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-full-pd-cache-summary.json @@ -0,0 +1,1392 @@ +{ + "model": "Qwen3-4B", + "runtime_base": "f52c41305f6bb2ee65d3c0fe8166acbca51dcc72", + "source_sha256": { + "benchmark201_native_http.py": "e1ea87a51343d1af83c40aaa9c41ae2a20e9864aa8b53bd475d047d175c41433", + "llmd.py": "9f3aaf0902f7b1958f32be12b1f276384f20cc3e0070571af10bb7bce148c1f0", + "serve_batched.py": "cf509e2c392a39aab7b47d7296447ff62cc8237e193e42d7d9cd76fdc47d9c30", + "streaming_factories.py": "77745350973cc54bf508e0c8b473efc442afef20b1c64f7c4edc179ea1471389", + "stream265-python.sh": "1a417cc6dde8ec6837362153406e618b006169513851131f8d324ff46aa96d9a", + "fsdk.py": "2cf093e93fbc2a98c1ce242563e65d7feaaabff468a91654392412c74df98bb6" + }, + "arms": { + "whole/off": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 0.0, + "transferred_tokens": 935680.0 + }, + "raw_sha256": "c8100535bfc02fe0b532c39323cb23d8362026fa48cedea19684a277565fcf18", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.5938640710664913, + "output_tok_s": 8.90406519757546, + "median_first_sse_s": 2.769333482021466, + "median_tpot_est_s": 0.02659360800058611, + "median_e2e_s": 3.5937353300396353, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 11.845253351028077, + "output_tok_s": 10.80601623340463, + "median_first_sse_s": 6.894638849480543, + "median_tpot_est_s": 0.026379545144135912, + "median_e2e_s": 7.7222023899666965, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.374117970932275, + "output_tok_s": 11.040641254264422, + "median_first_sse_s": 24.28165605297545, + "median_tpot_est_s": 0.026395979968290177, + "median_e2e_s": 25.099912968988065, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.118984114029445, + "output_tok_s": 20.918505035259855, + "median_first_sse_s": 2.848816549987532, + "median_tpot_est_s": 0.025747966134048058, + "median_e2e_s": 6.118808249011636, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.784803000977263, + "output_tok_s": 34.630153676458, + "median_first_sse_s": 7.131008746451698, + "median_tpot_est_s": 0.027017266000071028, + "median_e2e_s": 10.569683398469351, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.64825740107335, + "output_tok_s": 42.0981163439333, + "median_first_sse_s": 24.06720970547758, + "median_tpot_est_s": 0.026488267799645047, + "median_e2e_s": 27.375394922448322, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.636839246028103, + "output_tok_s": 8.798849175131433, + "median_first_sse_s": 2.866227310965769, + "median_tpot_est_s": 0.024852504938720696, + "median_e2e_s": 3.6366549640661106, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.187823576037772, + "output_tok_s": 10.502285268688837, + "median_first_sse_s": 7.140909380977973, + "median_tpot_est_s": 0.025063636806613256, + "median_e2e_s": 7.917882121982984, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.370749281020835, + "output_tok_s": 11.041443322322104, + "median_first_sse_s": 24.260911067016423, + "median_tpot_est_s": 0.025874566645436588, + "median_e2e_s": 25.080011933052447, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.158095913007855, + "output_tok_s": 20.78564572689154, + "median_first_sse_s": 2.877790122060105, + "median_tpot_est_s": 0.025827755779394602, + "median_e2e_s": 6.15791510604322, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.883136463933624, + "output_tok_s": 34.4013509007817, + "median_first_sse_s": 7.182273587503005, + "median_tpot_est_s": 0.026408043138198847, + "median_e2e_s": 10.536095066054258, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.92640683695208, + "output_tok_s": 41.85878613209811, + "median_first_sse_s": 24.215235407522414, + "median_tpot_est_s": 0.026753554602362596, + "median_e2e_s": 27.697570833028294, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.843274821061641, + "output_tok_s": 8.326232572449902, + "median_first_sse_s": 3.003903826000169, + "median_tpot_est_s": 0.027072391161064226, + "median_e2e_s": 3.84314795199316, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.749716062913649, + "output_tok_s": 10.039439260324093, + "median_first_sse_s": 7.466653063020203, + "median_tpot_est_s": 0.025596863032513927, + "median_e2e_s": 8.284569455485325, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.98791066301055, + "output_tok_s": 10.896419797678993, + "median_first_sse_s": 24.6336495334981, + "median_tpot_est_s": 0.0250324510477273, + "median_e2e_s": 25.404889553028625, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.079132216051221, + "output_tok_s": 21.055636799941826, + "median_first_sse_s": 2.9262310749618337, + "median_tpot_est_s": 0.024825054519685117, + "median_e2e_s": 6.0790129989618436, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.522591115906835, + "output_tok_s": 35.255416606696166, + "median_first_sse_s": 6.977572105941363, + "median_tpot_est_s": 0.02599662249196526, + "median_e2e_s": 10.286594730452634, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.192781265010126, + "output_tok_s": 41.63212461940433, + "median_first_sse_s": 24.612169308995362, + "median_tpot_est_s": 0.026611734634012306, + "median_e2e_s": 27.96816182450857, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.199458156013861, + "output_tok_s": 7.620030682809446, + "median_first_sse_s": 3.3747121680062264, + "median_tpot_est_s": 0.02660094451669964, + "median_e2e_s": 4.199341448023915, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 13.205650482908823, + "output_tok_s": 9.692820521462515, + "median_first_sse_s": 8.0374622924719, + "median_tpot_est_s": 0.026171039097056155, + "median_e2e_s": 8.848766844486818, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 51.270903732045554, + "output_tok_s": 9.986170766090625, + "median_first_sse_s": 26.803447068086825, + "median_tpot_est_s": 0.1057412954021035, + "median_e2e_s": 30.457469970046077, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.657406355952844, + "output_tok_s": 19.226706791834392, + "median_first_sse_s": 3.4630271260393783, + "median_tpot_est_s": 0.02515140170869454, + "median_e2e_s": 6.657255143043585, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 16.016934926970862, + "output_tok_s": 31.966165957123604, + "median_first_sse_s": 8.493684782530181, + "median_tpot_est_s": 0.02901831886632089, + "median_e2e_s": 12.179011278552935, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 53.753593918983825, + "output_tok_s": 38.09977809272992, + "median_first_sse_s": 27.78725230850978, + "median_tpot_est_s": 0.030791497555117703, + "median_e2e_s": 32.433394862979185, + "all_text_equal": true + } + ], + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "whole/on": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 850688.0, + "transferred_tokens": 84992.0 + }, + "raw_sha256": "06fae33330c3d3499a0fa80e56c9494117f077e64bb6e4220dded2f591cc9e72", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.66532118502073, + "output_tok_s": 8.730476371559513, + "median_first_sse_s": 2.918039128999226, + "median_tpot_est_s": 0.024100132226463285, + "median_e2e_s": 3.6651432280195877, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.30348469002638, + "output_tok_s": 10.403556652836828, + "median_first_sse_s": 7.266616121982224, + "median_tpot_est_s": 0.02600571070985508, + "median_e2e_s": 8.072793153987732, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.18980112497229, + "output_tok_s": 11.084698083343548, + "median_first_sse_s": 24.03905806154944, + "median_tpot_est_s": 0.026055202466405688, + "median_e2e_s": 24.958345417049713, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.242904701037332, + "output_tok_s": 20.50327630001004, + "median_first_sse_s": 2.869474889943376, + "median_tpot_est_s": 0.026561346441309343, + "median_e2e_s": 6.242765887989663, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 15.31299539806787, + "output_tok_s": 33.43565296601618, + "median_first_sse_s": 7.409371640475001, + "median_tpot_est_s": 0.026311591090671133, + "median_e2e_s": 10.784471123944968, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.35642505402211, + "output_tok_s": 41.494091149397505, + "median_first_sse_s": 24.396923166990746, + "median_tpot_est_s": 0.026859728673080674, + "median_e2e_s": 27.779408978007268, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.7718445679638535, + "output_tok_s": 8.483912691363761, + "median_first_sse_s": 2.9373642350547016, + "median_tpot_est_s": 0.026914485902976123, + "median_e2e_s": 3.7717132980469614, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.398822656017728, + "output_tok_s": 10.323560837276402, + "median_first_sse_s": 7.272774892975576, + "median_tpot_est_s": 0.027120182258752925, + "median_e2e_s": 8.096748902986292, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 45.94501227198634, + "output_tok_s": 11.143755865577978, + "median_first_sse_s": 24.065952128032222, + "median_tpot_est_s": 0.02582315859692772, + "median_e2e_s": 24.863611125038005, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.160433033946902, + "output_tok_s": 20.777760150083513, + "median_first_sse_s": 2.8113285440485924, + "median_tpot_est_s": 0.02636973762162411, + "median_e2e_s": 6.1602852219948545, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.768116903025657, + "output_tok_s": 34.66928135537054, + "median_first_sse_s": 7.088801535486709, + "median_tpot_est_s": 0.026913238858426447, + "median_e2e_s": 10.556075934495311, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.03478913893923, + "output_tok_s": 41.76626505310398, + "median_first_sse_s": 24.075821354053915, + "median_tpot_est_s": 0.026899172204883963, + "median_e2e_s": 27.450733256991953, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.945928778965026, + "output_tok_s": 8.109624322310564, + "median_first_sse_s": 3.141313388943672, + "median_tpot_est_s": 0.025950086614747923, + "median_e2e_s": 3.9457660740008578, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.172303085098974, + "output_tok_s": 10.515676376535051, + "median_first_sse_s": 7.091708452033345, + "median_tpot_est_s": 0.0255083825792216, + "median_e2e_s": 7.890375397459138, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.20221433893312, + "output_tok_s": 11.08171994190664, + "median_first_sse_s": 24.13236670056358, + "median_tpot_est_s": 0.025335372613382437, + "median_e2e_s": 24.96728817099938, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.266925232950598, + "output_tok_s": 20.42468918042843, + "median_first_sse_s": 2.8313267999328673, + "median_tpot_est_s": 0.027050363503975838, + "median_e2e_s": 6.266722964937799, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.83509479695931, + "output_tok_s": 34.512755530550606, + "median_first_sse_s": 7.151992013503332, + "median_tpot_est_s": 0.027155561228722744, + "median_e2e_s": 10.588076097541489, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.54142292402685, + "output_tok_s": 41.33914367257204, + "median_first_sse_s": 24.450606751954183, + "median_tpot_est_s": 0.02662571351590207, + "median_e2e_s": 27.830910447984934, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.643375781015493, + "output_tok_s": 8.78306326971325, + "median_first_sse_s": 2.857726918067783, + "median_tpot_est_s": 0.025337460514668737, + "median_e2e_s": 3.643188194022514, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.517737936927006, + "output_tok_s": 10.225489672730989, + "median_first_sse_s": 7.47981611196883, + "median_tpot_est_s": 0.026645917452931885, + "median_e2e_s": 8.298069035517983, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.68368743208703, + "output_tok_s": 10.96742841372226, + "median_first_sse_s": 25.86334280553274, + "median_tpot_est_s": 0.028045082806549486, + "median_e2e_s": 26.712535800528713, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.16653712594416, + "output_tok_s": 20.75719279487868, + "median_first_sse_s": 2.874457805068232, + "median_tpot_est_s": 0.025920808952353075, + "median_e2e_s": 6.166400542017072, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.728964269976132, + "output_tok_s": 34.76143947498555, + "median_first_sse_s": 7.113750016025733, + "median_tpot_est_s": 0.027268773952213034, + "median_e2e_s": 10.576884307956789, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.11081874195952, + "output_tok_s": 41.701605724813966, + "median_first_sse_s": 24.4042886030511, + "median_tpot_est_s": 0.027804161051625576, + "median_e2e_s": 28.926480908005033, + "all_text_equal": true + } + ], + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "stream/off": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 0.0, + "transferred_tokens": 935680.0 + }, + "raw_sha256": "3ddd7794442c6e6e5a7904871ca859c521738c8768433d35e8d20399420493a6", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.0344492259901017, + "output_tok_s": 10.545571079561832, + "median_first_sse_s": 2.222173067042604, + "median_tpot_est_s": 0.026198500128943595, + "median_e2e_s": 3.0343265710398555, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.291457520914264, + "output_tok_s": 13.776094838929534, + "median_first_sse_s": 5.330548713565804, + "median_tpot_est_s": 0.025170327402696376, + "median_e2e_s": 6.114778548071627, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.56748351501301, + "output_tok_s": 14.395170796491271, + "median_first_sse_s": 18.518849209474865, + "median_tpot_est_s": 0.026875704419874254, + "median_e2e_s": 19.44195634953212, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.373530109995045, + "output_tok_s": 23.820467621817798, + "median_first_sse_s": 2.137613946921192, + "median_tpot_est_s": 0.025478321228425686, + "median_e2e_s": 5.373360742931254, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 11.927924883086234, + "output_tok_s": 42.924482256424554, + "median_first_sse_s": 5.350217982544564, + "median_tpot_est_s": 0.027437909999880647, + "median_e2e_s": 8.846565436047968, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 37.57740541594103, + "output_tok_s": 54.50083573708366, + "median_first_sse_s": 18.221833322022576, + "median_tpot_est_s": 0.02667981534270482, + "median_e2e_s": 21.64534808701137, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.211695746984333, + "output_tok_s": 9.963583888681502, + "median_first_sse_s": 2.3731549710500985, + "median_tpot_est_s": 0.02704614116209409, + "median_e2e_s": 3.2115853470750153, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.43295849103015, + "output_tok_s": 13.5694437881515, + "median_first_sse_s": 5.399655768938828, + "median_tpot_est_s": 0.026163732065939374, + "median_e2e_s": 6.210731462982949, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.71247306501027, + "output_tok_s": 14.336727648851577, + "median_first_sse_s": 18.574257490516175, + "median_tpot_est_s": 0.02652731508151778, + "median_e2e_s": 19.40594940056326, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.338220724021085, + "output_tok_s": 23.97802687776131, + "median_first_sse_s": 2.165918902028352, + "median_tpot_est_s": 0.024977886991568027, + "median_e2e_s": 5.3381105499574915, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 11.92513715790119, + "output_tok_s": 42.934516661786674, + "median_first_sse_s": 5.504016306484118, + "median_tpot_est_s": 0.02519130685841518, + "median_e2e_s": 8.703312277502846, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 37.93958489596844, + "output_tok_s": 53.980558975953, + "median_first_sse_s": 18.479511058481876, + "median_tpot_est_s": 0.0253564650866599, + "median_e2e_s": 21.677008820464835, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.058836486074142, + "output_tok_s": 10.461494148407501, + "median_first_sse_s": 2.2814225980546325, + "median_tpot_est_s": 0.02507314690193462, + "median_e2e_s": 3.0586901520146057, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.86284664098639, + "output_tok_s": 12.97799759636115, + "median_first_sse_s": 5.68974005500786, + "median_tpot_est_s": 0.025835917564108968, + "median_e2e_s": 6.488205989997368, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 37.1058676840039, + "output_tok_s": 13.798356754792179, + "median_first_sse_s": 19.3962357249693, + "median_tpot_est_s": 0.029022938258300025, + "median_e2e_s": 20.305324282497168, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.826587219024077, + "output_tok_s": 21.968262927923583, + "median_first_sse_s": 2.3179203160107136, + "median_tpot_est_s": 0.027626068841878647, + "median_e2e_s": 5.826431058929302, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 12.11718301300425, + "output_tok_s": 42.254045304962204, + "median_first_sse_s": 5.6174396550050005, + "median_tpot_est_s": 0.026623807287422984, + "median_e2e_s": 9.04161945101805, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 39.340143382083625, + "output_tok_s": 52.058783317315125, + "median_first_sse_s": 19.44411149097141, + "median_tpot_est_s": 0.02653768402386794, + "median_e2e_s": 22.713961433502845, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.316174581996165, + "output_tok_s": 9.649672901339729, + "median_first_sse_s": 2.5564451449317858, + "median_tpot_est_s": 0.024503057582243797, + "median_e2e_s": 3.3160399299813434, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 11.041767364018597, + "output_tok_s": 11.592347110763166, + "median_first_sse_s": 6.5184954924625345, + "median_tpot_est_s": 0.028367914580514714, + "median_e2e_s": 7.396435436501633, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 41.80970816500485, + "output_tok_s": 12.245959669925398, + "median_first_sse_s": 23.20848668902181, + "median_tpot_est_s": 0.03088053917482255, + "median_e2e_s": 24.188921280030627, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.676418976043351, + "output_tok_s": 22.549427824163214, + "median_first_sse_s": 2.630975637002848, + "median_tpot_est_s": 0.02397870525211741, + "median_e2e_s": 5.676271204021759, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 13.63700374995824, + "output_tok_s": 37.54490424640148, + "median_first_sse_s": 6.511755163548514, + "median_tpot_est_s": 0.029209394330584157, + "median_e2e_s": 10.221348243532702, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 44.1992259939434, + "output_tok_s": 46.33565303339557, + "median_first_sse_s": 22.067861926509067, + "median_tpot_est_s": 0.03135937557869231, + "median_e2e_s": 26.46022372151492, + "all_text_equal": true + } + ], + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "stream/on": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 850688.0, + "transferred_tokens": 84992.0 + }, + "raw_sha256": "1c2c47cc3a9cdae054096f95ec26653cd9ac8594dfd9be13e3a98ade28c0822c", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.9660799669800326, + "output_tok_s": 10.788650459947434, + "median_first_sse_s": 2.138459959998727, + "median_tpot_est_s": 0.026691501002548444, + "median_e2e_s": 2.9658964910777286, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.432780474075116, + "output_tok_s": 13.569699872884023, + "median_first_sse_s": 5.423826438491233, + "median_tpot_est_s": 0.027401641080711758, + "median_e2e_s": 6.273733754991554, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 36.13144242495764, + "output_tok_s": 14.170483258823294, + "median_first_sse_s": 18.93030613096198, + "median_tpot_est_s": 0.027004231435937748, + "median_e2e_s": 19.77994070999557, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.6312565330881625, + "output_tok_s": 22.7302732965364, + "median_first_sse_s": 2.1788647409994155, + "median_tpot_est_s": 0.027182801669705513, + "median_e2e_s": 5.631080553052016, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 11.93878390698228, + "output_tok_s": 42.885439923287485, + "median_first_sse_s": 5.39331592497183, + "median_tpot_est_s": 0.026827251126139064, + "median_e2e_s": 8.800376817991491, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 38.001310175983235, + "output_tok_s": 53.892878706438196, + "median_first_sse_s": 18.469030371517874, + "median_tpot_est_s": 0.027872965275584978, + "median_e2e_s": 22.035761670034844, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.0288188609993085, + "output_tok_s": 10.565174567567944, + "median_first_sse_s": 2.203755221911706, + "median_tpot_est_s": 0.02660922609990643, + "median_e2e_s": 3.0286412310088053, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.58583946689032, + "output_tok_s": 13.353029793803092, + "median_first_sse_s": 5.536463976022787, + "median_tpot_est_s": 0.02809929470681856, + "median_e2e_s": 6.408132624987047, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.72137267398648, + "output_tok_s": 14.333155802068486, + "median_first_sse_s": 18.58372621802846, + "median_tpot_est_s": 0.027761704613259365, + "median_e2e_s": 19.43553564354079, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.674119274015538, + "output_tok_s": 22.55856703368437, + "median_first_sse_s": 2.39965068094898, + "median_tpot_est_s": 0.0257820155357898, + "median_e2e_s": 5.673966653994285, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 11.962467286037281, + "output_tok_s": 42.80053501986265, + "median_first_sse_s": 5.4564229375100695, + "median_tpot_est_s": 0.026840349098330173, + "median_e2e_s": 8.865147272998001, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 37.98896909598261, + "output_tok_s": 53.91038632360726, + "median_first_sse_s": 18.385367062990554, + "median_tpot_est_s": 0.027335515952995413, + "median_e2e_s": 21.945599335012957, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.027376718004234, + "output_tok_s": 10.570207470279966, + "median_first_sse_s": 2.1692160370294005, + "median_tpot_est_s": 0.027678460581228137, + "median_e2e_s": 3.0272483150474727, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.446687264018692, + "output_tok_s": 13.549723455706719, + "median_first_sse_s": 5.366796983988024, + "median_tpot_est_s": 0.027426751824321166, + "median_e2e_s": 6.208029180474114, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.83456855302211, + "output_tok_s": 14.287879571995585, + "median_first_sse_s": 18.475361039978452, + "median_tpot_est_s": 0.02735608807885118, + "median_e2e_s": 19.322635578981135, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.534334080060944, + "output_tok_s": 23.128347177514527, + "median_first_sse_s": 2.167296726955101, + "median_tpot_est_s": 0.02651036960644809, + "median_e2e_s": 5.534113666974008, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 12.09984776400961, + "output_tok_s": 42.31458196713171, + "median_first_sse_s": 5.475183577975258, + "median_tpot_est_s": 0.02788927299236351, + "median_e2e_s": 9.052137463004328, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 38.07431497902144, + "output_tok_s": 53.78954292752023, + "median_first_sse_s": 18.418084530567285, + "median_tpot_est_s": 0.02799524525574621, + "median_e2e_s": 21.927415508020204, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.001718234969303, + "output_tok_s": 10.66056088383234, + "median_first_sse_s": 2.1595299169421196, + "median_tpot_est_s": 0.027163157968090906, + "median_e2e_s": 3.0015878139529377, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.459372852928936, + "output_tok_s": 13.531552460200038, + "median_first_sse_s": 5.385697565565351, + "median_tpot_est_s": 0.02743237182280169, + "median_e2e_s": 6.236101092072204, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.61464168399107, + "output_tok_s": 14.376109818624013, + "median_first_sse_s": 19.533553374931216, + "median_tpot_est_s": 0.027933973081094486, + "median_e2e_s": 20.4053214424639, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.434419253957458, + "output_tok_s": 23.55357472774809, + "median_first_sse_s": 2.1673672610195354, + "median_tpot_est_s": 0.02572301870915951, + "median_e2e_s": 5.434190637082793, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 12.116605716058984, + "output_tok_s": 42.256058503365395, + "median_first_sse_s": 5.459057881555054, + "median_tpot_est_s": 0.02775848159011028, + "median_e2e_s": 8.98438504349906, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 38.63029872998595, + "output_tok_s": 53.01538086243903, + "median_first_sse_s": 18.56699193548411, + "median_tpot_est_s": 0.02884565194515057, + "median_e2e_s": 22.505755685968325, + "all_text_equal": true + } + ], + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + } + }, + "comparisons": { + "stream/off -> stream/on": { + "increased_conditions": 14, + "total_conditions": 24, + "min_throughput_change_pct": -5.919835903568171, + "max_throughput_change_pct": 17.394717981392738 + }, + "whole/on -> stream/on": { + "increased_conditions": 24, + "total_conditions": 24, + "min_throughput_change_pct": 8.570735588136525, + "max_throughput_change_pct": 32.331584044190585 + } + }, + "scope": { + "conditions": 96, + "responses": 1376, + "repeats_per_condition": 1 + }, + "limits": [ + "Single ordered runs, no statistical repeatability claim.", + "Same-reference parity is not native-engine equivalence or answer quality.", + "One A10080GB plus one RNGD research producer; four GPU slots, BF16, context9216,page/chunk256,width8192.", + "Cache counters count logical KV tokens; full producer prefill and CPU copy remain. No physical DMA overlap or PCIe-byte reduction proof." + ], + "all_four_reference_sets_equal": true, + "config_sha256": { + "whole/off": "a0dea0a54b95edcb9677ba473ae7cc831d3ee6cfa5de6f7a4175ccbda532fe12", + "whole/on": "2e26fe1f9af3b001621c3455d3ce668457b294d5ad9951478df500a62041929b", + "stream/off": "73c2047141df251249a8c9fdd60ef6ce1cfcf6838c507b923f50a9bbde6b8db9", + "stream/on": "74f4db840b0fa3c2ac6b1d32326365281055067beb185bb1f351f0b44b5336a2" + }, + "packages": { + "whole/off": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "whole/on": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "stream/off": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "stream/on": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + } + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-off-route-comparison.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-off-route-comparison.json new file mode 100644 index 00000000..fd901e5d --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-off-route-comparison.json @@ -0,0 +1,853 @@ +{ + "model": "Qwen3-4B", + "comparison": "RNGD-GPU whole versus streamed KV, decoder/producer cache off", + "runtime_base": "f52c41305f6bb2ee65d3c0fe8166acbca51dcc72", + "raw_sha256": { + "whole": "c8100535bfc02fe0b532c39323cb23d8362026fa48cedea19684a277565fcf18", + "stream": "3ddd7794442c6e6e5a7904871ca859c521738c8768433d35e8d20399420493a6" + }, + "reference_tokens_equal": true, + "conditions": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "whole": { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.5938640710664913, + "output_tok_s": 8.90406519757546, + "median_first_sse_s": 2.769333482021466, + "median_tpot_est_s": 0.02659360800058611, + "median_e2e_s": 3.5937353300396353, + "all_text_equal": true + }, + "stream": { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.0344492259901017, + "output_tok_s": 10.545571079561832, + "median_first_sse_s": 2.222173067042604, + "median_tpot_est_s": 0.026198500128943595, + "median_e2e_s": 3.0343265710398555, + "all_text_equal": true + }, + "throughput_change_pct": 18.435465661609808, + "first_sse_change_pct": -19.757837708280036 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "whole": { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 11.845253351028077, + "output_tok_s": 10.80601623340463, + "median_first_sse_s": 6.894638849480543, + "median_tpot_est_s": 0.026379545144135912, + "median_e2e_s": 7.7222023899666965, + "all_text_equal": true + }, + "stream": { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.291457520914264, + "output_tok_s": 13.776094838929534, + "median_first_sse_s": 5.330548713565804, + "median_tpot_est_s": 0.025170327402696376, + "median_e2e_s": 6.114778548071627, + "all_text_equal": true + }, + "throughput_change_pct": 27.48541683977397, + "first_sse_change_pct": -22.685599203395277 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "whole": { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.374117970932275, + "output_tok_s": 11.040641254264422, + "median_first_sse_s": 24.28165605297545, + "median_tpot_est_s": 0.026395979968290177, + "median_e2e_s": 25.099912968988065, + "all_text_equal": true + }, + "stream": { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.56748351501301, + "output_tok_s": 14.395170796491271, + "median_first_sse_s": 18.518849209474865, + "median_tpot_est_s": 0.026875704419874254, + "median_e2e_s": 19.44195634953212, + "all_text_equal": true + }, + "throughput_change_pct": 30.38346654847761, + "first_sse_change_pct": -23.733170550343974 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "whole": { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.118984114029445, + "output_tok_s": 20.918505035259855, + "median_first_sse_s": 2.848816549987532, + "median_tpot_est_s": 0.025747966134048058, + "median_e2e_s": 6.118808249011636, + "all_text_equal": true + }, + "stream": { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.373530109995045, + "output_tok_s": 23.820467621817798, + "median_first_sse_s": 2.137613946921192, + "median_tpot_est_s": 0.025478321228425686, + "median_e2e_s": 5.373360742931254, + "all_text_equal": true + }, + "throughput_change_pct": 13.872705442699873, + "first_sse_change_pct": -24.964843842593254 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "whole": { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.784803000977263, + "output_tok_s": 34.630153676458, + "median_first_sse_s": 7.131008746451698, + "median_tpot_est_s": 0.027017266000071028, + "median_e2e_s": 10.569683398469351, + "all_text_equal": true + }, + "stream": { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 11.927924883086234, + "output_tok_s": 42.924482256424554, + "median_first_sse_s": 5.350217982544564, + "median_tpot_est_s": 0.027437909999880647, + "median_e2e_s": 8.846565436047968, + "all_text_equal": true + }, + "throughput_change_pct": 23.951174625035353, + "first_sse_change_pct": -24.972494456597516 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "whole": { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.64825740107335, + "output_tok_s": 42.0981163439333, + "median_first_sse_s": 24.06720970547758, + "median_tpot_est_s": 0.026488267799645047, + "median_e2e_s": 27.375394922448322, + "all_text_equal": true + }, + "stream": { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 37.57740541594103, + "output_tok_s": 54.50083573708366, + "median_first_sse_s": 18.221833322022576, + "median_tpot_est_s": 0.02667981534270482, + "median_e2e_s": 21.64534808701137, + "all_text_equal": true + }, + "throughput_change_pct": 29.46145925347965, + "first_sse_change_pct": -24.287719494647632 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "whole": { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.636839246028103, + "output_tok_s": 8.798849175131433, + "median_first_sse_s": 2.866227310965769, + "median_tpot_est_s": 0.024852504938720696, + "median_e2e_s": 3.6366549640661106, + "all_text_equal": true + }, + "stream": { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.211695746984333, + "output_tok_s": 9.963583888681502, + "median_first_sse_s": 2.3731549710500985, + "median_tpot_est_s": 0.02704614116209409, + "median_e2e_s": 3.2115853470750153, + "all_text_equal": true + }, + "throughput_change_pct": 13.237352867031849, + "first_sse_change_pct": -17.202834472661934 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "whole": { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.187823576037772, + "output_tok_s": 10.502285268688837, + "median_first_sse_s": 7.140909380977973, + "median_tpot_est_s": 0.025063636806613256, + "median_e2e_s": 7.917882121982984, + "all_text_equal": true + }, + "stream": { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.43295849103015, + "output_tok_s": 13.5694437881515, + "median_first_sse_s": 5.399655768938828, + "median_tpot_est_s": 0.026163732065939374, + "median_e2e_s": 6.210731462982949, + "all_text_equal": true + }, + "throughput_change_pct": 29.204677277306356, + "first_sse_change_pct": -24.38419981462745 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "whole": { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.370749281020835, + "output_tok_s": 11.041443322322104, + "median_first_sse_s": 24.260911067016423, + "median_tpot_est_s": 0.025874566645436588, + "median_e2e_s": 25.080011933052447, + "all_text_equal": true + }, + "stream": { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.71247306501027, + "output_tok_s": 14.336727648851577, + "median_first_sse_s": 18.574257490516175, + "median_tpot_est_s": 0.02652731508151778, + "median_e2e_s": 19.40594940056326, + "all_text_equal": true + }, + "throughput_change_pct": 29.84468814749526, + "first_sse_change_pct": -23.43957141919315 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "whole": { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.158095913007855, + "output_tok_s": 20.78564572689154, + "median_first_sse_s": 2.877790122060105, + "median_tpot_est_s": 0.025827755779394602, + "median_e2e_s": 6.15791510604322, + "all_text_equal": true + }, + "stream": { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.338220724021085, + "output_tok_s": 23.97802687776131, + "median_first_sse_s": 2.165918902028352, + "median_tpot_est_s": 0.024977886991568027, + "median_e2e_s": 5.3381105499574915, + "all_text_equal": true + }, + "throughput_change_pct": 15.358585404636283, + "first_sse_change_pct": -24.736731653041833 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "whole": { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.883136463933624, + "output_tok_s": 34.4013509007817, + "median_first_sse_s": 7.182273587503005, + "median_tpot_est_s": 0.026408043138198847, + "median_e2e_s": 10.536095066054258, + "all_text_equal": true + }, + "stream": { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 11.92513715790119, + "output_tok_s": 42.934516661786674, + "median_first_sse_s": 5.504016306484118, + "median_tpot_est_s": 0.02519130685841518, + "median_e2e_s": 8.703312277502846, + "all_text_equal": true + }, + "throughput_change_pct": 24.80474033015685, + "first_sse_change_pct": -23.366657654743438 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "whole": { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.92640683695208, + "output_tok_s": 41.85878613209811, + "median_first_sse_s": 24.215235407522414, + "median_tpot_est_s": 0.026753554602362596, + "median_e2e_s": 27.697570833028294, + "all_text_equal": true + }, + "stream": { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 37.93958489596844, + "output_tok_s": 53.980558975953, + "median_first_sse_s": 18.479511058481876, + "median_tpot_est_s": 0.0253564650866599, + "median_e2e_s": 21.677008820464835, + "all_text_equal": true + }, + "throughput_change_pct": 28.95872996794737, + "first_sse_change_pct": -23.686428203207754 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "whole": { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.843274821061641, + "output_tok_s": 8.326232572449902, + "median_first_sse_s": 3.003903826000169, + "median_tpot_est_s": 0.027072391161064226, + "median_e2e_s": 3.84314795199316, + "all_text_equal": true + }, + "stream": { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.058836486074142, + "output_tok_s": 10.461494148407501, + "median_first_sse_s": 2.2814225980546325, + "median_tpot_est_s": 0.02507314690193462, + "median_e2e_s": 3.0586901520146057, + "all_text_equal": true + }, + "throughput_change_pct": 25.64499078518201, + "first_sse_change_pct": -24.051410091499235 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "whole": { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.749716062913649, + "output_tok_s": 10.039439260324093, + "median_first_sse_s": 7.466653063020203, + "median_tpot_est_s": 0.025596863032513927, + "median_e2e_s": 8.284569455485325, + "all_text_equal": true + }, + "stream": { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.86284664098639, + "output_tok_s": 12.97799759636115, + "median_first_sse_s": 5.68974005500786, + "median_tpot_est_s": 0.025835917564108968, + "median_e2e_s": 6.488205989997368, + "all_text_equal": true + }, + "throughput_change_pct": 29.270144077172233, + "first_sse_change_pct": -23.797985429546607 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "whole": { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.98791066301055, + "output_tok_s": 10.896419797678993, + "median_first_sse_s": 24.6336495334981, + "median_tpot_est_s": 0.0250324510477273, + "median_e2e_s": 25.404889553028625, + "all_text_equal": true + }, + "stream": { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 37.1058676840039, + "output_tok_s": 13.798356754792179, + "median_first_sse_s": 19.3962357249693, + "median_tpot_est_s": 0.029022938258300025, + "median_e2e_s": 20.305324282497168, + "all_text_equal": true + }, + "throughput_change_pct": 26.632022361430295, + "first_sse_change_pct": -21.26121751227602 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "whole": { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.079132216051221, + "output_tok_s": 21.055636799941826, + "median_first_sse_s": 2.9262310749618337, + "median_tpot_est_s": 0.024825054519685117, + "median_e2e_s": 6.0790129989618436, + "all_text_equal": true + }, + "stream": { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.826587219024077, + "output_tok_s": 21.968262927923583, + "median_first_sse_s": 2.3179203160107136, + "median_tpot_est_s": 0.027626068841878647, + "median_e2e_s": 5.826431058929302, + "all_text_equal": true + }, + "throughput_change_pct": 4.334355387362487, + "first_sse_change_pct": -20.78819967965292 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "whole": { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.522591115906835, + "output_tok_s": 35.255416606696166, + "median_first_sse_s": 6.977572105941363, + "median_tpot_est_s": 0.02599662249196526, + "median_e2e_s": 10.286594730452634, + "all_text_equal": true + }, + "stream": { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 12.11718301300425, + "output_tok_s": 42.254045304962204, + "median_first_sse_s": 5.6174396550050005, + "median_tpot_est_s": 0.026623807287422984, + "median_e2e_s": 9.04161945101805, + "all_text_equal": true + }, + "throughput_change_pct": 19.85121542128301, + "first_sse_change_pct": -19.49291860098754 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "whole": { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.192781265010126, + "output_tok_s": 41.63212461940433, + "median_first_sse_s": 24.612169308995362, + "median_tpot_est_s": 0.026611734634012306, + "median_e2e_s": 27.96816182450857, + "all_text_equal": true + }, + "stream": { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 39.340143382083625, + "output_tok_s": 52.058783317315125, + "median_first_sse_s": 19.44411149097141, + "median_tpot_est_s": 0.02653768402386794, + "median_e2e_s": 22.713961433502845, + "all_text_equal": true + }, + "throughput_change_pct": 25.044743195861386, + "first_sse_change_pct": -20.997977679826498 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "whole": { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.199458156013861, + "output_tok_s": 7.620030682809446, + "median_first_sse_s": 3.3747121680062264, + "median_tpot_est_s": 0.02660094451669964, + "median_e2e_s": 4.199341448023915, + "all_text_equal": true + }, + "stream": { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.316174581996165, + "output_tok_s": 9.649672901339729, + "median_first_sse_s": 2.5564451449317858, + "median_tpot_est_s": 0.024503057582243797, + "median_e2e_s": 3.3160399299813434, + "all_text_equal": true + }, + "throughput_change_pct": 26.63561740124083, + "first_sse_change_pct": -24.247016703586645 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "whole": { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 13.205650482908823, + "output_tok_s": 9.692820521462515, + "median_first_sse_s": 8.0374622924719, + "median_tpot_est_s": 0.026171039097056155, + "median_e2e_s": 8.848766844486818, + "all_text_equal": true + }, + "stream": { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 11.041767364018597, + "output_tok_s": 11.592347110763166, + "median_first_sse_s": 6.5184954924625345, + "median_tpot_est_s": 0.028367914580514714, + "median_e2e_s": 7.396435436501633, + "all_text_equal": true + }, + "throughput_change_pct": 19.59725329788773, + "first_sse_change_pct": -18.898586951158336 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "whole": { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 51.270903732045554, + "output_tok_s": 9.986170766090625, + "median_first_sse_s": 26.803447068086825, + "median_tpot_est_s": 0.1057412954021035, + "median_e2e_s": 30.457469970046077, + "all_text_equal": true + }, + "stream": { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 41.80970816500485, + "output_tok_s": 12.245959669925398, + "median_first_sse_s": 23.20848668902181, + "median_tpot_est_s": 0.03088053917482255, + "median_e2e_s": 24.188921280030627, + "all_text_equal": true + }, + "throughput_change_pct": 22.62918346547995, + "first_sse_change_pct": -13.412306148283836 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "whole": { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.657406355952844, + "output_tok_s": 19.226706791834392, + "median_first_sse_s": 3.4630271260393783, + "median_tpot_est_s": 0.02515140170869454, + "median_e2e_s": 6.657255143043585, + "all_text_equal": true + }, + "stream": { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.676418976043351, + "output_tok_s": 22.549427824163214, + "median_first_sse_s": 2.630975637002848, + "median_tpot_est_s": 0.02397870525211741, + "median_e2e_s": 5.676271204021759, + "all_text_equal": true + }, + "throughput_change_pct": 17.281800093503197, + "first_sse_change_pct": -24.02671012248574 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "whole": { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 16.016934926970862, + "output_tok_s": 31.966165957123604, + "median_first_sse_s": 8.493684782530181, + "median_tpot_est_s": 0.02901831886632089, + "median_e2e_s": 12.179011278552935, + "all_text_equal": true + }, + "stream": { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 13.63700374995824, + "output_tok_s": 37.54490424640148, + "median_first_sse_s": 6.511755163548514, + "median_tpot_est_s": 0.029209394330584157, + "median_e2e_s": 10.221348243532702, + "all_text_equal": true + }, + "throughput_change_pct": 17.452009405071188, + "first_sse_change_pct": -23.334155548815538 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "whole": { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 53.753593918983825, + "output_tok_s": 38.09977809272992, + "median_first_sse_s": 27.78725230850978, + "median_tpot_est_s": 0.030791497555117703, + "median_e2e_s": 32.433394862979185, + "all_text_equal": true + }, + "stream": { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 44.1992259939434, + "output_tok_s": 46.33565303339557, + "median_first_sse_s": 22.067861926509067, + "median_tpot_est_s": 0.03135937557869231, + "median_e2e_s": 26.46022372151492, + "all_text_equal": true + }, + "throughput_change_pct": 21.616595562894368, + "first_sse_change_pct": -20.582784934979557 + } + ], + "limits": [ + "One ordered whole/stream pair, not repeated ABBA or a stable speedup estimate.", + "Both routes use one A10080GB plus one RNGD research producer, four GPU slots, BF16, context9216, producer width8192, page/chunk256.", + "No proof of physical DMA overlap or reduced producer compute from timing alone.", + "Same-route and whole/stream parity here does not establish native-decoder equivalence or answer quality." + ], + "aggregate_scope": { + "conditions": 24, + "responses": 688, + "throughput_increased_conditions": 24, + "min_throughput_change_pct": 4.334355387362487, + "max_throughput_change_pct": 30.38346654847761 + }, + "config_comparison": { + "metadata_equal": true, + "only_added_server_argument": "--pd-prefill-stream-layers" + }, + "listen_counter_delta": { + "whole": { + "ListenOverflows": 0, + "ListenDrops": 0 + }, + "stream": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "source_sha256": { + "benchmark201_native_http.py": "e1ea87a51343d1af83c40aaa9c41ae2a20e9864aa8b53bd475d047d175c41433", + "llmd.py": "9f3aaf0902f7b1958f32be12b1f276384f20cc3e0070571af10bb7bce148c1f0", + "serve_batched.py": "cf509e2c392a39aab7b47d7296447ff62cc8237e193e42d7d9cd76fdc47d9c30", + "streaming_factories.py": "77745350973cc54bf508e0c8b473efc442afef20b1c64f7c4edc179ea1471389", + "stream265-python.sh": "1a417cc6dde8ec6837362153406e618b006169513851131f8d324ff46aa96d9a", + "fsdk.py": "2cf093e93fbc2a98c1ce242563e65d7feaaabff468a91654392412c74df98bb6" + }, + "config_sha256": { + "whole": "a0dea0a54b95edcb9677ba473ae7cc831d3ee6cfa5de6f7a4175ccbda532fe12", + "stream": "73c2047141df251249a8c9fdd60ef6ce1cfcf6838c507b923f50a9bbde6b8db9" + }, + "packages": { + "whole": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "stream": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + } + } +} diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-whole-cache-pair.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-whole-cache-pair.json new file mode 100644 index 00000000..a2c55251 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-whole-cache-pair.json @@ -0,0 +1,660 @@ +{ + "model": "Qwen3-4B", + "route": "RNGD-GPU whole", + "raw_sha256": { + "off": "c8100535bfc02fe0b532c39323cb23d8362026fa48cedea19684a277565fcf18", + "on": "06fae33330c3d3499a0fa80e56c9494117f077e64bb6e4220dded2f591cc9e72" + }, + "reference_tokens_equal": true, + "conditions": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.5938640710664913, + "output_tok_s": 8.90406519757546, + "median_first_sse_s": 2.769333482021466, + "median_tpot_est_s": 0.02659360800058611, + "median_e2e_s": 3.5937353300396353, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.66532118502073, + "output_tok_s": 8.730476371559513, + "median_first_sse_s": 2.918039128999226, + "median_tpot_est_s": 0.024100132226463285, + "median_e2e_s": 3.6651432280195877, + "all_text_equal": true + }, + "throughput_change_pct": -1.9495457654916226, + "first_sse_change_pct": 5.369726973770339 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 128, + "wall_s": 11.845253351028077, + "output_tok_s": 10.80601623340463, + "median_first_sse_s": 6.894638849480543, + "median_tpot_est_s": 0.026379545144135912, + "median_e2e_s": 7.7222023899666965, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 128, + "wall_s": 12.30348469002638, + "output_tok_s": 10.403556652836828, + "median_first_sse_s": 7.266616121982224, + "median_tpot_est_s": 0.02600571070985508, + "median_e2e_s": 8.072793153987732, + "all_text_equal": true + }, + "throughput_change_pct": -3.7244028870110246, + "first_sse_change_pct": 5.3951668915871664 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 512, + "wall_s": 46.374117970932275, + "output_tok_s": 11.040641254264422, + "median_first_sse_s": 24.28165605297545, + "median_tpot_est_s": 0.026395979968290177, + "median_e2e_s": 25.099912968988065, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 512, + "wall_s": 46.18980112497229, + "output_tok_s": 11.084698083343548, + "median_first_sse_s": 24.03905806154944, + "median_tpot_est_s": 0.026055202466405688, + "median_e2e_s": 24.958345417049713, + "all_text_equal": true + }, + "throughput_change_pct": 0.3990423025665324, + "first_sse_change_pct": -0.9990998591559497 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.118984114029445, + "output_tok_s": 20.918505035259855, + "median_first_sse_s": 2.848816549987532, + "median_tpot_est_s": 0.025747966134048058, + "median_e2e_s": 6.118808249011636, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.242904701037332, + "output_tok_s": 20.50327630001004, + "median_first_sse_s": 2.869474889943376, + "median_tpot_est_s": 0.026561346441309343, + "median_e2e_s": 6.242765887989663, + "all_text_equal": true + }, + "throughput_change_pct": -1.9849828395954394, + "first_sse_change_pct": 0.7251551510375265 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 512, + "wall_s": 14.784803000977263, + "output_tok_s": 34.630153676458, + "median_first_sse_s": 7.131008746451698, + "median_tpot_est_s": 0.027017266000071028, + "median_e2e_s": 10.569683398469351, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 512, + "wall_s": 15.31299539806787, + "output_tok_s": 33.43565296601618, + "median_first_sse_s": 7.409371640475001, + "median_tpot_est_s": 0.026311591090671133, + "median_e2e_s": 10.784471123944968, + "all_text_equal": true + }, + "throughput_change_pct": -3.4493081422675265, + "first_sse_change_pct": 3.903555638770073 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.64825740107335, + "output_tok_s": 42.0981163439333, + "median_first_sse_s": 24.06720970547758, + "median_tpot_est_s": 0.026488267799645047, + "median_e2e_s": 27.375394922448322, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.35642505402211, + "output_tok_s": 41.494091149397505, + "median_first_sse_s": 24.396923166990746, + "median_tpot_est_s": 0.026859728673080674, + "median_e2e_s": 27.779408978007268, + "all_text_equal": true + }, + "throughput_change_pct": -1.4348033760015033, + "first_sse_change_pct": 1.3699696206915357 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.636839246028103, + "output_tok_s": 8.798849175131433, + "median_first_sse_s": 2.866227310965769, + "median_tpot_est_s": 0.024852504938720696, + "median_e2e_s": 3.6366549640661106, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.7718445679638535, + "output_tok_s": 8.483912691363761, + "median_first_sse_s": 2.9373642350547016, + "median_tpot_est_s": 0.026914485902976123, + "median_e2e_s": 3.7717132980469614, + "all_text_equal": true + }, + "throughput_change_pct": -3.579291763038639, + "first_sse_change_pct": 2.481900992875663 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 128, + "wall_s": 12.187823576037772, + "output_tok_s": 10.502285268688837, + "median_first_sse_s": 7.140909380977973, + "median_tpot_est_s": 0.025063636806613256, + "median_e2e_s": 7.917882121982984, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 128, + "wall_s": 12.398822656017728, + "output_tok_s": 10.323560837276402, + "median_first_sse_s": 7.272774892975576, + "median_tpot_est_s": 0.027120182258752925, + "median_e2e_s": 8.096748902986292, + "all_text_equal": true + }, + "throughput_change_pct": -1.7017670615487757, + "first_sse_change_pct": 1.8466207168076787 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 512, + "wall_s": 46.370749281020835, + "output_tok_s": 11.041443322322104, + "median_first_sse_s": 24.260911067016423, + "median_tpot_est_s": 0.025874566645436588, + "median_e2e_s": 25.080011933052447, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 512, + "wall_s": 45.94501227198634, + "output_tok_s": 11.143755865577978, + "median_first_sse_s": 24.065952128032222, + "median_tpot_est_s": 0.02582315859692772, + "median_e2e_s": 24.863611125038005, + "all_text_equal": true + }, + "throughput_change_pct": 0.9266229085198585, + "first_sse_change_pct": -0.8035928182814733 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.158095913007855, + "output_tok_s": 20.78564572689154, + "median_first_sse_s": 2.877790122060105, + "median_tpot_est_s": 0.025827755779394602, + "median_e2e_s": 6.15791510604322, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.160433033946902, + "output_tok_s": 20.777760150083513, + "median_first_sse_s": 2.8113285440485924, + "median_tpot_est_s": 0.02636973762162411, + "median_e2e_s": 6.1602852219948545, + "all_text_equal": true + }, + "throughput_change_pct": -0.03793760805721469, + "first_sse_change_pct": -2.3094657773005123 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 512, + "wall_s": 14.883136463933624, + "output_tok_s": 34.4013509007817, + "median_first_sse_s": 7.182273587503005, + "median_tpot_est_s": 0.026408043138198847, + "median_e2e_s": 10.536095066054258, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 512, + "wall_s": 14.768116903025657, + "output_tok_s": 34.66928135537054, + "median_first_sse_s": 7.088801535486709, + "median_tpot_est_s": 0.026913238858426447, + "median_e2e_s": 10.556075934495311, + "all_text_equal": true + }, + "throughput_change_pct": 0.7788370153299917, + "first_sse_change_pct": -1.3014270603522338 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.92640683695208, + "output_tok_s": 41.85878613209811, + "median_first_sse_s": 24.215235407522414, + "median_tpot_est_s": 0.026753554602362596, + "median_e2e_s": 27.697570833028294, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.03478913893923, + "output_tok_s": 41.76626505310398, + "median_first_sse_s": 24.075821354053915, + "median_tpot_est_s": 0.026899172204883963, + "median_e2e_s": 27.450733256991953, + "all_text_equal": true + }, + "throughput_change_pct": -0.22103144296194976, + "first_sse_change_pct": -0.5757286729708655 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.843274821061641, + "output_tok_s": 8.326232572449902, + "median_first_sse_s": 3.003903826000169, + "median_tpot_est_s": 0.027072391161064226, + "median_e2e_s": 3.84314795199316, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.945928778965026, + "output_tok_s": 8.109624322310564, + "median_first_sse_s": 3.141313388943672, + "median_tpot_est_s": 0.025950086614747923, + "median_e2e_s": 3.9457660740008578, + "all_text_equal": true + }, + "throughput_change_pct": -2.60151573060855, + "first_sse_change_pct": 4.574366254810158 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 128, + "wall_s": 12.749716062913649, + "output_tok_s": 10.039439260324093, + "median_first_sse_s": 7.466653063020203, + "median_tpot_est_s": 0.025596863032513927, + "median_e2e_s": 8.284569455485325, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 128, + "wall_s": 12.172303085098974, + "output_tok_s": 10.515676376535051, + "median_first_sse_s": 7.091708452033345, + "median_tpot_est_s": 0.0255083825792216, + "median_e2e_s": 7.890375397459138, + "all_text_equal": true + }, + "throughput_change_pct": 4.743662508055091, + "first_sse_change_pct": -5.021588760348761 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 512, + "wall_s": 46.98791066301055, + "output_tok_s": 10.896419797678993, + "median_first_sse_s": 24.6336495334981, + "median_tpot_est_s": 0.0250324510477273, + "median_e2e_s": 25.404889553028625, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 512, + "wall_s": 46.20221433893312, + "output_tok_s": 11.08171994190664, + "median_first_sse_s": 24.13236670056358, + "median_tpot_est_s": 0.025335372613382437, + "median_e2e_s": 24.96728817099938, + "all_text_equal": true + }, + "throughput_change_pct": 1.700559887267894, + "first_sse_change_pct": -2.034951549720032 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.079132216051221, + "output_tok_s": 21.055636799941826, + "median_first_sse_s": 2.9262310749618337, + "median_tpot_est_s": 0.024825054519685117, + "median_e2e_s": 6.0790129989618436, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.266925232950598, + "output_tok_s": 20.42468918042843, + "median_first_sse_s": 2.8313267999328673, + "median_tpot_est_s": 0.027050363503975838, + "median_e2e_s": 6.266722964937799, + "all_text_equal": true + }, + "throughput_change_pct": -2.9965734378318065, + "first_sse_change_pct": -3.2432255894283446 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 512, + "wall_s": 14.522591115906835, + "output_tok_s": 35.255416606696166, + "median_first_sse_s": 6.977572105941363, + "median_tpot_est_s": 0.02599662249196526, + "median_e2e_s": 10.286594730452634, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 512, + "wall_s": 14.83509479695931, + "output_tok_s": 34.512755530550606, + "median_first_sse_s": 7.151992013503332, + "median_tpot_est_s": 0.027155561228722744, + "median_e2e_s": 10.588076097541489, + "all_text_equal": true + }, + "throughput_change_pct": -2.10651623956275, + "first_sse_change_pct": 2.4997220367447115 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.192781265010126, + "output_tok_s": 41.63212461940433, + "median_first_sse_s": 24.612169308995362, + "median_tpot_est_s": 0.026611734634012306, + "median_e2e_s": 27.96816182450857, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.54142292402685, + "output_tok_s": 41.33914367257204, + "median_first_sse_s": 24.450606751954183, + "median_tpot_est_s": 0.02662571351590207, + "median_e2e_s": 27.830910447984934, + "all_text_equal": true + }, + "throughput_change_pct": -0.7037376773601722, + "first_sse_change_pct": -0.6564336325369347 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 32, + "wall_s": 4.199458156013861, + "output_tok_s": 7.620030682809446, + "median_first_sse_s": 3.3747121680062264, + "median_tpot_est_s": 0.02660094451669964, + "median_e2e_s": 4.199341448023915, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.643375781015493, + "output_tok_s": 8.78306326971325, + "median_first_sse_s": 2.857726918067783, + "median_tpot_est_s": 0.025337460514668737, + "median_e2e_s": 3.643188194022514, + "all_text_equal": true + }, + "throughput_change_pct": 15.262833383697117, + "first_sse_change_pct": -15.319387971504462 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 128, + "wall_s": 13.205650482908823, + "output_tok_s": 9.692820521462515, + "median_first_sse_s": 8.0374622924719, + "median_tpot_est_s": 0.026171039097056155, + "median_e2e_s": 8.848766844486818, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 128, + "wall_s": 12.517737936927006, + "output_tok_s": 10.225489672730989, + "median_first_sse_s": 7.47981611196883, + "median_tpot_est_s": 0.026645917452931885, + "median_e2e_s": 8.298069035517983, + "all_text_equal": true + }, + "throughput_change_pct": 5.4955020583430825, + "first_sse_change_pct": -6.938087672590099 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 512, + "wall_s": 51.270903732045554, + "output_tok_s": 9.986170766090625, + "median_first_sse_s": 26.803447068086825, + "median_tpot_est_s": 0.1057412954021035, + "median_e2e_s": 30.457469970046077, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 512, + "wall_s": 46.68368743208703, + "output_tok_s": 10.96742841372226, + "median_first_sse_s": 25.86334280553274, + "median_tpot_est_s": 0.028045082806549486, + "median_e2e_s": 26.712535800528713, + "all_text_equal": true + }, + "throughput_change_pct": 9.826165310167001, + "first_sse_change_pct": -3.5074005972664968 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.657406355952844, + "output_tok_s": 19.226706791834392, + "median_first_sse_s": 3.4630271260393783, + "median_tpot_est_s": 0.02515140170869454, + "median_e2e_s": 6.657255143043585, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.16653712594416, + "output_tok_s": 20.75719279487868, + "median_first_sse_s": 2.874457805068232, + "median_tpot_est_s": 0.025920808952353075, + "median_e2e_s": 6.166400542017072, + "all_text_equal": true + }, + "throughput_change_pct": 7.960208784659306, + "first_sse_change_pct": -16.995804524473524 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 512, + "wall_s": 16.016934926970862, + "output_tok_s": 31.966165957123604, + "median_first_sse_s": 8.493684782530181, + "median_tpot_est_s": 0.02901831886632089, + "median_e2e_s": 12.179011278552935, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 512, + "wall_s": 14.728964269976132, + "output_tok_s": 34.76143947498555, + "median_first_sse_s": 7.113750016025733, + "median_tpot_est_s": 0.027268773952213034, + "median_e2e_s": 10.576884307956789, + "all_text_equal": true + }, + "throughput_change_pct": 8.744475398179642, + "first_sse_change_pct": -16.24659734656858 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 53.753593918983825, + "output_tok_s": 38.09977809272992, + "median_first_sse_s": 27.78725230850978, + "median_tpot_est_s": 0.030791497555117703, + "median_e2e_s": 32.433394862979185, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.11081874195952, + "output_tok_s": 41.701605724813966, + "median_first_sse_s": 24.4042886030511, + "median_tpot_est_s": 0.027804161051625576, + "median_e2e_s": 28.926480908005033, + "all_text_equal": true + }, + "throughput_change_pct": 9.453670893614309, + "first_sse_change_pct": -12.17451681763675 + } + ], + "limits": "Single ordered off/on pair per condition; no statistical speedup guarantee. Logical transferred tokens are not PCIe byte measurements. Producer still computes full prompt." +} diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md new file mode 100644 index 00000000..96c3b4df --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md @@ -0,0 +1,112 @@ +# Preserve HTTP connection bursts while accept is delayed (#267) + +The common serving listener inherited Python3.10 ThreadingHTTPServer's backlog5. Sixteen clients connecting during a brief accept pause can exhaust it. A real-socket regression holds the accept thread before it starts: the original listener times out; the candidate accepts all16 connections. The candidate sets backlog128 before bind/listen without changing engine admission,request timeouts or request processing. + +## Verification + +```bash +OMP_NUM_THREADS=2 MKL_NUM_THREADS=2 TORCH_DEVICE_BACKEND_AUTOLOAD=0 PYTHONPATH=opt_prime python -m pytest -q opt_prime/tests/test_http_listener_backlog.py opt_prime/tests/test_pd_http_cancel.py opt_prime/tests/test_pd_stage_metrics.py opt_prime/tests/test_pd_llmd.py +``` + +15tests passed. The live Qwen3-4B RNGD-to-GPU whole-KV cache-off control originally reset one connection at2048input/128output/c16 during correctness checks. Server remained alive;host listen drops increased36. Candidate retry completed all24conditions/344responses with exact same-route token/text checks;host listen drops increased0. First attempt retained8warmups/110completed token responses and no timed batches;it remains a failed run. + +Runtime base:f52c41305f6bb2ee65d3c0fe8166acbca51dcc72 plus this HTTP-only candidate. Source fingerprint,raw result fingerprints,exact geometry,versions and complete timing rows are in ../evidence267/qwen4-backlog-controls.json. One A10080GB decoder plus one RNGD research prefiller,BF16,4decode slots,context9216,page256/chunk256,producerwidth8192,both cachesoff. + +## Hardware reproduction + +Use the HTTP harness and diagnostics published with PR279 at experiments/cache-history-212/full_grid_http.py and http267_diagnostics.py. The published harness adds argument/output-file guards to the measured benchmark;the HTTP requests and grid logic are the same. Configure one serve_batched.py server with: + +```text +--model Qwen/Qwen3-4B --device cuda:0 --dtype bfloat16 --slots 4 +--max-context 9216 --max-output 128 --kv paged --page-size 256 --chunk 256 +--port {port} --pd-prefill-python +--pd-prefill-args "--engine factory --engine-factory streaming_factories:qwen --width 8192 --chip 0" +``` + +Use the model/producer setup from PR266;the wrapper must load the Furiosa environment and matching rngd_pd engines. Server-config metadata:gpus1,rngd1,atom0,slots4,producer_cache=false,decoder_cache=false,producer_chunk8192,local_chunk256,context9216(using the harness physical_* and slots_per_decoder key names). Run: + +```bash +OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 TORCH_DEVICE_BACKEND_AUTOLOAD=0 python full_grid_http.py --model Qwen/Qwen3-4B --mode pd --server-config qwen4-whole-off-config.json --lengths 128 512 2048 8192 --outputs 32 128 --concurrency 1 4 16 --repeats 1 --startup-timeout 1800 --request-timeout 180 --output qwen4-whole-off.json +``` + +Use distinct server-config and result paths. Compare fresh server processes on the original/candidate llmd.py. Do not overwrite failed results or retry individual failed requests inside a measured batch. + +## Limits + +One successful hardware retry supports the backlog hypothesis but does not establish it as the sole reset cause. Counters are host-wide. No before/after throughput gain is claimed because the original did not finish timed measurements. ATOM Qwen4 also reset underc16 on the original listener with no OOM;its native candidate retry passed as detailed below; remaining ATOM and GPU model/routes are pending. Engine concurrency,KV-cache correctness and resource limits are unchanged. Related#267 and#201;not closing either issue. + +## ATOM Qwen3-4B cache-on retry + +The original native ATOM run reset one SSE connection at input 512, output 128, concurrency 16. The server remained alive and cgroup OOM counters were zero. All 168 token checks completed, but only 11 timed batches completed; the original remains a failed run. Host-wide listen drops increased 82. + +With the identical backlog candidate, the retry completed all 24 conditions and 344 responses, matching retained same-route reference outputs. Cache hits were 867456 of 935680 queried tokens; listen drop/overflow increments were both zero. This uses one ATOM execution lane, B1/k3/context9216, prefix caching enabled. At 8192/128/c4, throughput was 23.0725 tok/s and median first SSE 8.43916 s; this is a single warm sample, not a paired cache speedup. Exact raw hashes, package versions and timings are in ../evidence267/qwen4-atom-backlog-controls.json. + +The independent retry auditor passed 13 corruption/accounting selfchecks. Combined with the earlier GPU retry, two previously failing configurations now completed, but broader validation remains in progress. No merge or universal reliability claim. + +## Qwen3-4B whole-KV paired cache control + +Both fresh off/on processes using the same HTTP candidate completed 24 conditions and 344 responses each (48 conditions, 688 responses total), with identical token references and matching streamed text. Host listen drops/overflows increased zero in both runs. All 24 paired timing rows and raw SHA256 fingerprints are retained in ../evidence267/qwen4-whole-cache-pair.json. + +At input 8192, output 128, concurrency 4, cache off/on throughput was 31.9662/34.7614 tok/s and median first SSE was 8.49368/7.11375 s. Across all 24 conditions, throughput increased in only 11; changes ranged from -3.72% to +15.26%. These are single ordered off/on samples, not evidence of a universal or repeatable cache speedup, and not a backlog performance comparison. + +Both runs queried 935680 prefix tokens. Cache-on hit 850688 tokens, reducing logical transferred tokens from 935680 to 84992. The producer still performs full prefill and copies tensors to CPU before slicing. These counters therefore do not establish reduced producer compute or physical PCIe traffic. Streamed-KV controls remain in progress. + +## Qwen3-4B streamed KV cache-off retry + +Streamed KV cache-off completed all 24 conditions and 344 responses with exact same-route outputs. All eight prompt/output references also equal the whole-KV cache-off control. Resource metadata is identical; the only added server argument is `--pd-prefill-stream-layers`. Both runs use the same backlog candidate; host listen drops/overflows increased zero. Exact raw/config/source fingerprints, package versions and all 24 paired rows are in ../evidence267/qwen4-off-route-comparison.json. Reproduce with the whole/off command above and add that server argument; keep both caches disabled and use distinct result files. + +Throughput increased in 24/24 conditions in this single ordered whole/stream pair, ranging from +4.33% to +30.38%. At 8192 input, 128 output, concurrency 4: 31.9662 to 37.5449 tok/s (+17.45%); median first SSE 8.49368 to 6.51176 s (-23.33%); median end-to-end 12.1790 to 10.2213 s. These compare transfer modes, not the backlog patch. One sample per condition does not establish repeatability; timing alone does not establish physical DMA overlap. Cache-on streaming, remaining GPU models and ATOM PD are still running. + +## Qwen3-4B four GPU PD configurations complete + +Whole/streamed KV crossed with cache off/on completed all four runs: 96 conditions, 1376 responses, zero same-route mismatches. All four prompt/reference sets are identical. Every run has zero host listen drop/overflow increment. The complete measurements, raw/config/source SHA256 fingerprints and package versions are in ../evidence267/qwen4-full-pd-cache-summary.json. Cache-on reproduction uses the command above with `--prefix-cache` in server arguments and `decoder_cache=true` metadata; pass the fresh same-route off result as `--reference`. Producer cache stays off. + +At 8192 input, 128 output, concurrency 4: + +| Configuration | Output tok/s | Median first SSE (s) | +|---|---:|---:| +| Whole, cache off | 31.9662 | 8.49368 | +| Whole, cache on | 34.7614 | 7.11375 | +| Stream, cache off | 37.5449 | 6.51176 | +| Stream, cache on | 42.2561 | 5.45906 | + +Stream cache-on versus off improves throughput in 14/24 conditions, with changes from -5.92% to +17.39%. Stream versus whole with cache-on improves in 24/24 conditions, from +8.57% to +32.33%. These are single ordered runs, not repeated stability or universal cache-gain evidence. Cache-on logically transfers 84992 of 935680 queried tokens; producer full-prefill/CPU-copy limitations remain. No native equivalence or physical overlap certification. + +The GPU queue advanced to Gemma whole/cache-off. ATOM Qwen3-4B whole/cache-on and later streamed KV remain in progress; the four-model goal remains incomplete. + +## Qwen3-4B RNGD-to-ATOM whole-KV retry + +The whole-KV cache-on retry completed 24 conditions/344 responses, matching retained same-route cache-off references. Host listen drop/overflow increments were zero. Prefix hits were 867456 of 935680 queried tokens. The native and whole-PD results, all 48 timing rows and fingerprints are in ../evidence267/qwen4-atom-native-whole-summary.json. One ATOM execution lane, B1/k3/context9216, with one additional RNGD for PD; concurrent clients are queued, not hardware batch size. + +At 8192 input, 128 output, concurrency 4, native versus whole PD delivered 23.0725 versus 15.0415 tok/s, with median first SSE 8.43916 versus 15.8125 s. This single ordered comparison does not show a PD advantage; it is not a fresh cache off/on speedup measurement. + +Native/PD reference outputs match 6/8 cases. At 512/128 and 2048/128, they diverge at generated-token indices 50 and 61, respectively, before EOS. The same divergent arrays already occur in the archived cache-off runs; all four historical raw hashes and same-route reference arrays were verified. See ../evidence267/qwen4-atom-divergence-history.json. These cases are not new cache-on regressions, but their numerical root cause remains unresolved. Do not label cross-engine parity as passing. Streamed ATOM KV is still being measured. + +## Gemma3-1B GPU PD configurations complete + +The exact `unsloth/gemma-3-1b-it` whole/stream x cache off/on grid completed all four runs: 96 conditions, 1376 responses, zero same-route mismatches. All four prompt/reference sets equal; host listen drop/overflow increments are zero in every run. Full 96 timing rows, raw/config/source fingerprints and package versions are in ../evidence267/gemma-full-pd-cache-summary.json. Use the GPU reproduction commands above with this model and `streaming_factories:gemma`; retain page/chunk256, context9216, producerwidth8192, four GPU slots, BF16, one A10080GB and one RNGD. Add `--pd-prefill-stream-layers` for stream mode and `--prefix-cache` plus decoder-cache metadata for cache-on, referencing the fresh same-route off result. Producer cache remains off. + +At 8192 input, 128 output, concurrency 4: + +| Configuration | Output tok/s | Median first SSE (s) | +|---|---:|---:| +| Whole, cache off | 55.1567 | 3.93112 | +| Whole, cache on | 55.7948 | 3.77659 | +| Stream, cache off | 56.6012 | 3.76451 | +| Stream, cache on | 57.0211 | 3.72432 | + +Cache-on improves streamed throughput in only 10/24 conditions, with changes from -4.93% to +2.58%. At cache-on, stream beats whole in 20/24 conditions (-3.34% to +4.95%). These small differences in single ordered samples do not establish stable or universal gains. Same-route outputs do not certify native-engine equivalence or answer quality; logical transferred tokens do not measure physical PCIe traffic. KONI GPU and Qwen4 ATOM streaming remain in progress. + +## Qwen3-4B ATOM three cache-on routes complete + +Native, whole-KV PD and streamed-KV PD all completed: 72 conditions, 1032 responses matching retained same-route references. Each has zero host listen drop/overflow increment and 867456 hits of 935680 prefix queries. Whole/stream reference outputs match 8/8; native/PD matches 6/8, retaining the two documented pre-EOS differences. Full 72 timing rows, source/config/raw fingerprints, package versions and reference comparisons are in ../evidence267/qwen4-atom-full-summary.json. + +At 8192 input, 128 output, concurrency 4: + +| Route | Output tok/s | Median first SSE (s) | +|---|---:|---:| +| Native ATOM | 23.0725 | 8.43916 | +| RNGD to ATOM, whole KV | 15.0415 | 15.8125 | +| RNGD to ATOM, streamed KV | 16.5199 | 13.9411 | + +Streaming improves this condition versus whole KV but remains below native ATOM. These are single ordered cache-on runs, not a fresh cache-off/on timing comparison. One decoder execution lane, B1/k3/context9216; PD adds an RNGD research producer with width8192. Queued concurrency is not hardware batching. No universal native/PD equality or repeatable speedup claim. The coordinator exited zero, deleted its owned Pod with UID verification, and fresh namespace absence was checked before the separate KONI TP2 allocation. Remaining KONI GPU and TP2 tests continue; the full goal is incomplete. diff --git a/opt_prime/opt_prime/accel/serve/llmd.py b/opt_prime/opt_prime/accel/serve/llmd.py index 35fd4208..fd4ee03e 100644 --- a/opt_prime/opt_prime/accel/serve/llmd.py +++ b/opt_prime/opt_prime/accel/serve/llmd.py @@ -679,7 +679,11 @@ def text_delta(final=None): send(chunk(usage=response["usage"])) send("[DONE]") - server = ThreadingHTTPServer(("127.0.0.1", port), Handler) + class Server(ThreadingHTTPServer): + # Keep connection bursts queued while the accept thread is briefly delayed. + request_queue_size = 128 + + server = Server(("127.0.0.1", port), Handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() return server, thread diff --git a/opt_prime/tests/test_http_listener_backlog.py b/opt_prime/tests/test_http_listener_backlog.py new file mode 100644 index 00000000..1aba3726 --- /dev/null +++ b/opt_prime/tests/test_http_listener_backlog.py @@ -0,0 +1,18 @@ +"""A serving burst must survive a brief pause before the accept loop runs.""" +import socket +import threading + +from opt_prime.accel.serve.llmd import serve_stage + + +def test_sixteen_clients_connect_before_accept_loop_runs(monkeypatch): + monkeypatch.setattr(threading.Thread, 'start', lambda self: None) + server, _ = serve_stage(None, 'decode', 0) + clients = [] + try: + for _ in range(16): + clients.append(socket.create_connection(server.server_address, timeout=0.1)) + finally: + for client in clients: + client.close() + server.server_close()