From 91500dd144a76852f2a052144d67a4041b935048 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 30 Oct 2026 14:15:00 +0900 Subject: [PATCH 1/8] fix: queue serving connection bursts before accept (#267) --- .../pd/http_matrix/.research/events.jsonl | 1 + opt_prime/examples/pd/http_matrix/STATUS.md | 4 + .../evidence267/qwen4-backlog-controls.json | 401 ++++++++++++++++++ .../experiments/2026-10-30-http-backlog267.md | 36 ++ opt_prime/opt_prime/accel/serve/llmd.py | 6 +- opt_prime/tests/test_http_listener_backlog.py | 18 + 6 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 opt_prime/examples/pd/http_matrix/evidence267/qwen4-backlog-controls.json create mode 100644 opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md create mode 100644 opt_prime/tests/test_http_listener_backlog.py diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index 42c6a658..61fdbdfe 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -29,3 +29,4 @@ {"ts": "2026-10-24T05:13:10.885088+00:00", "task": "aicomp #265 / #267", "result": "KONI whole diagnostic264 responses pass;8K/128 native7.7847->whole9.3113tok/s; native/whole token parity6/8. Stream pending, different physical ATOM allocation.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-24-stream265-atom-http-progress.md"], "runtime_commit": "33d6cb08ec806f693dbecfd7d630331a3bd1bb2f"} {"ts": "2026-10-24T05:56:15.922056+00:00", "task": "aicomp #265 / #267", "result": "KONI all3 routes264 responses each audited; whole/stream8/8 exact.8K128 throughput7.7847/9.3113/10.8298tok/s. Native parity6/8; single batch and different physical card limits retained.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-24-stream265-atom-http-progress.md"], "runtime_commit": "33d6cb08ec806f693dbecfd7d630331a3bd1bb2f"} {"ts": "2026-10-30T04:24:12.257264+00:00", "task": "aicomp #201 / #265", "result": "ATOM short/long final-HTTP sweep:12 arms96conditions528responses audited;whole/stream16/16 references exact. Native/PD Qwen0.6/Qwen4=4/4,Gemma2/4(other2afterEOS),KONI2/4(preEOS). Eleven selfchecks pass;coordinator/cleanup0,fresh owned-Pod absence verified.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-atom-edge201-final.md"], "runtime_scope": "historical stream265 B1 engine,final HTTP SHA256e3449979b88d03edb6d5bf0d402a708e5d98054fdf203e22b480f3d7e6fd759a"} +{"ts": "2026-10-30T05:14:21.740281+00:00", "task": "aicomp #267 / #201", "result": "HTTP backlog128 candidate:15CPUtests pass;Qwen4 whole/cacheoff24conditions344responses exact in one retry after originalreset. Remaining hardware verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index d2880622..68f1c66c 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -146,3 +146,7 @@ Four models, whole/stream, input128/8192, output32/128, c1/c4:64 conditions352 r ## ATOM short/long controls complete (2026-10-30) All four models complete native/whole/stream at128/8192 input,32/128 output,c1/c4:96conditions528responses. Whole/stream16/16 reference pairs match. Native/PD equality is Qwen0.6/Qwen4=4/4,Gemma2/4(other2equal through EOS),KONI2/4(pre-EOS differences retained). Single-batch8K/128/c4 stream throughput44.24/14.47/43.87/10.44tok/s; one ATOM lane,extra RNGD for PD. Eleven independent auditor selfchecks pass;all exits0 and owned Pod absence verified. Cache and broader goal work remain active. See experiments/2026-10-30-atom-edge201-final.md. + +## HTTP listener backlog candidate (#267) + +Backlog128 handles a16connection accept-pause regression;15HTTPtests pass. Qwen4 RNGD-GPU whole/cacheoff retry24conditions344responses exact after originalc16 connection reset;listen drops+36before/+0retry(host-wide). One retry only;ATOM/remaining GPU verification pending. See experiments/2026-10-30-http-backlog267.md. diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-backlog-controls.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-backlog-controls.json new file mode 100644 index 00000000..894efb83 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-backlog-controls.json @@ -0,0 +1,401 @@ +{ + "model": "Qwen/Qwen3-4B", + "base_commit": "f52c41305f6bb2ee65d3c0fe8166acbca51dcc72", + "candidate_http_sha256": "9f3aaf0902f7b1958f32be12b1f276384f20cc3e0070571af10bb7bce148c1f0", + "hardware": "A10080GB GPU decoder plus one RNGD research prefiller", + "decoder": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "producer": { + "torch": "2.10.0+cpu", + "transformers": "5.8.1", + "furiosa-torch": "2026.4.0" + }, + "configuration": { + "inputs": [ + 128, + 512, + 2048, + 8192 + ], + "outputs": [ + 32, + 128 + ], + "concurrency": [ + 1, + 4, + 16 + ], + "gpu_slots": 4, + "context": 9216, + "page_size": 256, + "local_chunk": 256, + "producer_width": 8192, + "producer_cache": false, + "decoder_cache": false, + "repeats": 1 + }, + "before": { + "status": "failed_not_passed", + "raw_sha256": "e8c4e2fd391bbab5ae82c9d65bfbcab4fc922e9f5cd208c4979695ca31caa134", + "error": "ConnectionResetError(104, 'Connection reset by peer')", + "failed_condition": { + "input_tokens": 2048, + "output_tokens": 128, + "concurrency": 16, + "stream": false, + "elapsed_s": 47.106650558067486 + }, + "completed_warmups": 8, + "completed_token_batches": 17, + "completed_token_responses": 110, + "completed_token_responses_equal": true, + "timed_batches": 0, + "server_alive_at_failure": true, + "host_listen_counter_delta": { + "ListenOverflows": 36, + "ListenDrops": 36 + }, + "limits": "Host-wide listen counters do not identify a particular socket. Candidate backlog fix requires live retry; reset root cause not proven." + }, + "after": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 0.0, + "transferred_tokens": 935680.0 + }, + "raw_sha256": "c8100535bfc02fe0b532c39323cb23d8362026fa48cedea19684a277565fcf18", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.5938640710664913, + "output_tok_s": 8.90406519757546, + "median_first_sse_s": 2.769333482021466, + "median_tpot_est_s": 0.02659360800058611, + "median_e2e_s": 3.5937353300396353, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 11.845253351028077, + "output_tok_s": 10.80601623340463, + "median_first_sse_s": 6.894638849480543, + "median_tpot_est_s": 0.026379545144135912, + "median_e2e_s": 7.7222023899666965, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.374117970932275, + "output_tok_s": 11.040641254264422, + "median_first_sse_s": 24.28165605297545, + "median_tpot_est_s": 0.026395979968290177, + "median_e2e_s": 25.099912968988065, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.118984114029445, + "output_tok_s": 20.918505035259855, + "median_first_sse_s": 2.848816549987532, + "median_tpot_est_s": 0.025747966134048058, + "median_e2e_s": 6.118808249011636, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.784803000977263, + "output_tok_s": 34.630153676458, + "median_first_sse_s": 7.131008746451698, + "median_tpot_est_s": 0.027017266000071028, + "median_e2e_s": 10.569683398469351, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.64825740107335, + "output_tok_s": 42.0981163439333, + "median_first_sse_s": 24.06720970547758, + "median_tpot_est_s": 0.026488267799645047, + "median_e2e_s": 27.375394922448322, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.636839246028103, + "output_tok_s": 8.798849175131433, + "median_first_sse_s": 2.866227310965769, + "median_tpot_est_s": 0.024852504938720696, + "median_e2e_s": 3.6366549640661106, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.187823576037772, + "output_tok_s": 10.502285268688837, + "median_first_sse_s": 7.140909380977973, + "median_tpot_est_s": 0.025063636806613256, + "median_e2e_s": 7.917882121982984, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.370749281020835, + "output_tok_s": 11.041443322322104, + "median_first_sse_s": 24.260911067016423, + "median_tpot_est_s": 0.025874566645436588, + "median_e2e_s": 25.080011933052447, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.158095913007855, + "output_tok_s": 20.78564572689154, + "median_first_sse_s": 2.877790122060105, + "median_tpot_est_s": 0.025827755779394602, + "median_e2e_s": 6.15791510604322, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.883136463933624, + "output_tok_s": 34.4013509007817, + "median_first_sse_s": 7.182273587503005, + "median_tpot_est_s": 0.026408043138198847, + "median_e2e_s": 10.536095066054258, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.92640683695208, + "output_tok_s": 41.85878613209811, + "median_first_sse_s": 24.215235407522414, + "median_tpot_est_s": 0.026753554602362596, + "median_e2e_s": 27.697570833028294, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.843274821061641, + "output_tok_s": 8.326232572449902, + "median_first_sse_s": 3.003903826000169, + "median_tpot_est_s": 0.027072391161064226, + "median_e2e_s": 3.84314795199316, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.749716062913649, + "output_tok_s": 10.039439260324093, + "median_first_sse_s": 7.466653063020203, + "median_tpot_est_s": 0.025596863032513927, + "median_e2e_s": 8.284569455485325, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.98791066301055, + "output_tok_s": 10.896419797678993, + "median_first_sse_s": 24.6336495334981, + "median_tpot_est_s": 0.0250324510477273, + "median_e2e_s": 25.404889553028625, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.079132216051221, + "output_tok_s": 21.055636799941826, + "median_first_sse_s": 2.9262310749618337, + "median_tpot_est_s": 0.024825054519685117, + "median_e2e_s": 6.0790129989618436, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.522591115906835, + "output_tok_s": 35.255416606696166, + "median_first_sse_s": 6.977572105941363, + "median_tpot_est_s": 0.02599662249196526, + "median_e2e_s": 10.286594730452634, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.192781265010126, + "output_tok_s": 41.63212461940433, + "median_first_sse_s": 24.612169308995362, + "median_tpot_est_s": 0.026611734634012306, + "median_e2e_s": 27.96816182450857, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.199458156013861, + "output_tok_s": 7.620030682809446, + "median_first_sse_s": 3.3747121680062264, + "median_tpot_est_s": 0.02660094451669964, + "median_e2e_s": 4.199341448023915, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 13.205650482908823, + "output_tok_s": 9.692820521462515, + "median_first_sse_s": 8.0374622924719, + "median_tpot_est_s": 0.026171039097056155, + "median_e2e_s": 8.848766844486818, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 51.270903732045554, + "output_tok_s": 9.986170766090625, + "median_first_sse_s": 26.803447068086825, + "median_tpot_est_s": 0.1057412954021035, + "median_e2e_s": 30.457469970046077, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.657406355952844, + "output_tok_s": 19.226706791834392, + "median_first_sse_s": 3.4630271260393783, + "median_tpot_est_s": 0.02515140170869454, + "median_e2e_s": 6.657255143043585, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 16.016934926970862, + "output_tok_s": 31.966165957123604, + "median_first_sse_s": 8.493684782530181, + "median_tpot_est_s": 0.02901831886632089, + "median_e2e_s": 12.179011278552935, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 53.753593918983825, + "output_tok_s": 38.09977809272992, + "median_first_sse_s": 27.78725230850978, + "median_tpot_est_s": 0.030791497555117703, + "median_e2e_s": 32.433394862979185, + "all_text_equal": true + } + ] + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + }, + "limitations": [ + "One hardware retry,not proof all resets are eliminated.", + "Listen counters are host-wide,not socket-specific.", + "ATOM retry and remaining GPU routes/models still running.", + "Failed original has no timed batches;no throughput speedup is claimed." + ] +} diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md new file mode 100644 index 00000000..e29e7236 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md @@ -0,0 +1,36 @@ +# Preserve HTTP connection bursts while accept is delayed (#267) + +The common serving listener inherited Python3.10 ThreadingHTTPServer's backlog5. Sixteen clients connecting during a brief accept pause can exhaust it. A real-socket regression holds the accept thread before it starts: the original listener times out; the candidate accepts all16 connections. The candidate sets backlog128 before bind/listen without changing engine admission,request timeouts or request processing. + +## Verification + +```bash +OMP_NUM_THREADS=2 MKL_NUM_THREADS=2 TORCH_DEVICE_BACKEND_AUTOLOAD=0 PYTHONPATH=opt_prime python -m pytest -q opt_prime/tests/test_http_listener_backlog.py opt_prime/tests/test_pd_http_cancel.py opt_prime/tests/test_pd_stage_metrics.py opt_prime/tests/test_pd_llmd.py +``` + +15tests passed. The live Qwen3-4B RNGD-to-GPU whole-KV cache-off control originally reset one connection at2048input/128output/c16 during correctness checks. Server remained alive;host listen drops increased36. Candidate retry completed all24conditions/344responses with exact same-route token/text checks;host listen drops increased0. First attempt retained8warmups/110completed token responses and no timed batches;it remains a failed run. + +Runtime base:f52c41305f6bb2ee65d3c0fe8166acbca51dcc72 plus this HTTP-only candidate. Source fingerprint,raw result fingerprints,exact geometry,versions and complete timing rows are in ../evidence267/qwen4-backlog-controls.json. One A10080GB decoder plus one RNGD research prefiller,BF16,4decode slots,context9216,page256/chunk256,producerwidth8192,both cachesoff. + +## Hardware reproduction + +Use the HTTP harness and diagnostics published with PR279 at experiments/cache-history-212/full_grid_http.py and http267_diagnostics.py. The published harness adds argument/output-file guards to the measured benchmark;the HTTP requests and grid logic are the same. Configure one serve_batched.py server with: + +```text +--model Qwen/Qwen3-4B --device cuda:0 --dtype bfloat16 --slots 4 +--max-context 9216 --max-output 128 --kv paged --page-size 256 --chunk 256 +--port {port} --pd-prefill-python +--pd-prefill-args "--engine factory --engine-factory streaming_factories:qwen --width 8192 --chip 0" +``` + +Use the model/producer setup from PR266;the wrapper must load the Furiosa environment and matching rngd_pd engines. Server-config metadata:gpus1,rngd1,atom0,slots4,producer_cache=false,decoder_cache=false,producer_chunk8192,local_chunk256,context9216(using the harness physical_* and slots_per_decoder key names). Run: + +```bash +OMP_NUM_THREADS=4 MKL_NUM_THREADS=4 TORCH_DEVICE_BACKEND_AUTOLOAD=0 python full_grid_http.py --model Qwen/Qwen3-4B --mode pd --server-config qwen4-whole-off-config.json --lengths 128 512 2048 8192 --outputs 32 128 --concurrency 1 4 16 --repeats 1 --startup-timeout 1800 --request-timeout 180 --output qwen4-whole-off.json +``` + +Use distinct server-config and result paths. Compare fresh server processes on the original/candidate llmd.py. Do not overwrite failed results or retry individual failed requests inside a measured batch. + +## Limits + +One successful hardware retry supports the backlog hypothesis but does not establish it as the sole reset cause. Counters are host-wide. No before/after throughput gain is claimed because the original did not finish timed measurements. ATOM Qwen4 also reset underc16 on the original listener with no OOM;its candidate retry and remaining GPU model/routes are pending. Engine concurrency,KV-cache correctness and resource limits are unchanged. Related#267 and#201;not closing either issue. diff --git a/opt_prime/opt_prime/accel/serve/llmd.py b/opt_prime/opt_prime/accel/serve/llmd.py index 35fd4208..fd4ee03e 100644 --- a/opt_prime/opt_prime/accel/serve/llmd.py +++ b/opt_prime/opt_prime/accel/serve/llmd.py @@ -679,7 +679,11 @@ def text_delta(final=None): send(chunk(usage=response["usage"])) send("[DONE]") - server = ThreadingHTTPServer(("127.0.0.1", port), Handler) + class Server(ThreadingHTTPServer): + # Keep connection bursts queued while the accept thread is briefly delayed. + request_queue_size = 128 + + server = Server(("127.0.0.1", port), Handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() return server, thread diff --git a/opt_prime/tests/test_http_listener_backlog.py b/opt_prime/tests/test_http_listener_backlog.py new file mode 100644 index 00000000..1aba3726 --- /dev/null +++ b/opt_prime/tests/test_http_listener_backlog.py @@ -0,0 +1,18 @@ +"""A serving burst must survive a brief pause before the accept loop runs.""" +import socket +import threading + +from opt_prime.accel.serve.llmd import serve_stage + + +def test_sixteen_clients_connect_before_accept_loop_runs(monkeypatch): + monkeypatch.setattr(threading.Thread, 'start', lambda self: None) + server, _ = serve_stage(None, 'decode', 0) + clients = [] + try: + for _ in range(16): + clients.append(socket.create_connection(server.server_address, timeout=0.1)) + finally: + for client in clients: + client.close() + server.server_close() From 6e8c6687ec121b7b5b901b769d29a344ee67f154 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 30 Oct 2026 14:26:15 +0900 Subject: [PATCH 2/8] docs: record ATOM backlog retry and correctness evidence --- .../pd/http_matrix/.research/events.jsonl | 1 + opt_prime/examples/pd/http_matrix/STATUS.md | 2 + .../qwen4-atom-backlog-controls.json | 395 ++++++++++++++++++ .../experiments/2026-10-30-http-backlog267.md | 8 + 4 files changed, 406 insertions(+) create mode 100644 opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-backlog-controls.json diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index 61fdbdfe..40adaf92 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -30,3 +30,4 @@ {"ts": "2026-10-24T05:56:15.922056+00:00", "task": "aicomp #265 / #267", "result": "KONI all3 routes264 responses each audited; whole/stream8/8 exact.8K128 throughput7.7847/9.3113/10.8298tok/s. Native parity6/8; single batch and different physical card limits retained.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-24-stream265-atom-http-progress.md"], "runtime_commit": "33d6cb08ec806f693dbecfd7d630331a3bd1bb2f"} {"ts": "2026-10-30T04:24:12.257264+00:00", "task": "aicomp #201 / #265", "result": "ATOM short/long final-HTTP sweep:12 arms96conditions528responses audited;whole/stream16/16 references exact. Native/PD Qwen0.6/Qwen4=4/4,Gemma2/4(other2afterEOS),KONI2/4(preEOS). Eleven selfchecks pass;coordinator/cleanup0,fresh owned-Pod absence verified.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-atom-edge201-final.md"], "runtime_scope": "historical stream265 B1 engine,final HTTP SHA256e3449979b88d03edb6d5bf0d402a708e5d98054fdf203e22b480f3d7e6fd759a"} {"ts": "2026-10-30T05:14:21.740281+00:00", "task": "aicomp #267 / #201", "result": "HTTP backlog128 candidate:15CPUtests pass;Qwen4 whole/cacheoff24conditions344responses exact in one retry after originalreset. Remaining hardware verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T05:26:15.716445+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 native ATOM cache-on retry:24 conditions,344 responses exact;13 auditor selfchecks pass;host listen drops +82 original/+0 retry. Wider hardware verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index 68f1c66c..b2a9f15b 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -150,3 +150,5 @@ All four models complete native/whole/stream at128/8192 input,32/128 output,c1/c ## HTTP listener backlog candidate (#267) Backlog128 handles a16connection accept-pause regression;15HTTPtests pass. Qwen4 RNGD-GPU whole/cacheoff retry24conditions344responses exact after originalc16 connection reset;listen drops+36before/+0retry(host-wide). One retry only;ATOM/remaining GPU verification pending. See experiments/2026-10-30-http-backlog267.md. + +PR289 follow-up: Qwen4 native ATOM cache-on retry also passes 24 conditions/344 responses; listen drops +82 original/+0 retry, no OOM in original. Both GPU and ATOM have one successful retry; other routes/models remain pending. diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-backlog-controls.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-backlog-controls.json new file mode 100644 index 00000000..1fc3b4b1 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-backlog-controls.json @@ -0,0 +1,395 @@ +{ + "model": "Qwen/Qwen3-4B", + "hardware": "One assigned ATOM decoder; one execution lane, queued client concurrency", + "versions": { + "torch": "2.11.0+cpu", + "transformers": "5.8.1", + "optimum-rbln": "0.10.2", + "rebel-compiler": "0.11.0" + }, + "candidate_http_sha256": "9f3aaf0902f7b1958f32be12b1f276384f20cc3e0070571af10bb7bce148c1f0", + "configuration": { + "inputs": [ + 128, + 512, + 2048, + 8192 + ], + "outputs": [ + 32, + 128 + ], + "concurrency": [ + 1, + 4, + 16 + ], + "context": 9216, + "kv_blocks": 3, + "batch_size": 1, + "decoder_cache": true, + "producer_cache": false, + "repeats": 1 + }, + "before": { + "status": "failed_not_passed", + "raw_sha256": "456652095fdae63a2295f8533217d24f1c6d8df63fb0d0bb84fda0346b34b326", + "error": "ConnectionResetError(104, 'Connection reset by peer')", + "failed_condition": { + "input_tokens": 512, + "output_tokens": 128, + "concurrency": 16, + "stream": true, + "elapsed_s": 72.44087444990873 + }, + "completed_warmups": 8, + "completed_token_batches": 24, + "completed_token_responses": 168, + "completed_token_responses_equal": true, + "completed_timed_batches": 11, + "server_alive_at_failure": true, + "memory_events": "low 0\nhigh 0\nmax 0\noom 0\noom_kill 0\noom_group_kill 0", + "listen_counter_delta": { + "ListenOverflows": 82, + "ListenDrops": 82 + }, + "limits": "Listen counters do not isolate the specific server socket; candidate retry needed. Incomplete result is not a full-grid pass." + }, + "after": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 867456.0 + }, + "raw_sha256": "f6b0ed35f371a37e6460adebb51eca0af569b1b2fb061f199896d63b0d8b91a5", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 1.21979605499655, + "output_tok_s": 26.233893665191847, + "median_first_sse_s": 0.06424786592833698, + "median_tpot_est_s": 0.03727080548302301, + "median_e2e_s": 1.2196428359020501, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 4.865430230041966, + "output_tok_s": 26.308053748187437, + "median_first_sse_s": 1.8862478689989075, + "median_tpot_est_s": 0.03721824728868782, + "median_e2e_s": 3.0406833300366998, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 19.443709495011717, + "output_tok_s": 26.332423868570633, + "median_first_sse_s": 9.170724456431344, + "median_tpot_est_s": 0.03721861124053717, + "median_e2e_s": 10.32500390149653, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.779329141136259, + "output_tok_s": 26.782001452523673, + "median_first_sse_s": 0.06421606685034931, + "median_tpot_est_s": 0.03712567288289213, + "median_e2e_s": 4.77917652297765, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.179921305039898, + "output_tok_s": 26.69458293686857, + "median_first_sse_s": 7.2513526930706576, + "median_tpot_est_s": 0.03726597728691702, + "median_e2e_s": 11.985176004120149, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 76.49607840902172, + "output_tok_s": 26.772614264608695, + "median_first_sse_s": 35.89145322609693, + "median_tpot_est_s": 0.0371851845594915, + "median_e2e_s": 40.606709040584974, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 1.409332315903157, + "output_tok_s": 22.705787441972554, + "median_first_sse_s": 0.248691565124318, + "median_tpot_est_s": 0.0374359464828646, + "median_e2e_s": 1.4092059060931206, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 4.901245624991134, + "output_tok_s": 26.115810100872377, + "median_first_sse_s": 1.9010033400263637, + "median_tpot_est_s": 0.03745433188704474, + "median_e2e_s": 3.062635939917527, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 19.62891486589797, + "output_tok_s": 26.08396865022408, + "median_first_sse_s": 9.260896492982283, + "median_tpot_est_s": 0.03751262429080182, + "median_e2e_s": 10.424593756091781, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.838200076017529, + "output_tok_s": 26.456119628967624, + "median_first_sse_s": 0.06642191391438246, + "median_tpot_est_s": 0.0375715390867077, + "median_e2e_s": 4.83800737792626, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.333293949021026, + "output_tok_s": 26.482812569346258, + "median_first_sse_s": 7.3223171149147674, + "median_tpot_est_s": 0.03758554547607136, + "median_e2e_s": 12.085917299962603, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 77.24892303696834, + "output_tok_s": 26.51169646753401, + "median_first_sse_s": 36.27493576845154, + "median_tpot_est_s": 0.03750518164927448, + "median_e2e_s": 41.03906974289566, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.0401992758270353, + "output_tok_s": 15.684742357840591, + "median_first_sse_s": 0.8407348680775613, + "median_tpot_est_s": 0.03868695670708774, + "median_e2e_s": 2.040030525997281, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 5.082977191079408, + "output_tok_s": 25.182092145650223, + "median_first_sse_s": 1.9748041100101545, + "median_tpot_est_s": 0.038678585807041775, + "median_e2e_s": 3.1738402700284496, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 20.373157310998067, + "output_tok_s": 25.13110718109492, + "median_first_sse_s": 9.618916884530336, + "median_tpot_est_s": 0.03872891262759484, + "median_e2e_s": 10.823435940546915, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.998283860972151, + "output_tok_s": 25.608789648674414, + "median_first_sse_s": 0.07471933006308973, + "median_tpot_est_s": 0.038767074181617595, + "median_e2e_s": 4.9981377511285245, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.97290690196678, + "output_tok_s": 25.63472620750974, + "median_first_sse_s": 7.558827344560996, + "median_tpot_est_s": 0.038741515790786975, + "median_e2e_s": 12.481202929164283, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 79.82538939104415, + "output_tok_s": 25.655997617091124, + "median_first_sse_s": 37.48764609743375, + "median_tpot_est_s": 0.0387145414293971, + "median_e2e_s": 42.39941652864218, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 5.352265110006556, + "output_tok_s": 5.978777086391523, + "median_first_sse_s": 4.025504268007353, + "median_tpot_est_s": 0.04279371299931119, + "median_e2e_s": 5.352109370986, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 5.7538805468939245, + "output_tok_s": 22.24585633240115, + "median_first_sse_s": 2.2709930365672335, + "median_tpot_est_s": 0.04280944161056992, + "median_e2e_s": 3.5957872320432216, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 23.044004359981045, + "output_tok_s": 22.218360663442486, + "median_first_sse_s": 10.923830434447154, + "median_tpot_est_s": 0.042838023904140196, + "median_e2e_s": 12.250851112417877, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.550273383036256, + "output_tok_s": 23.06192707393777, + "median_first_sse_s": 0.11678028688766062, + "median_tpot_est_s": 0.04278182463104448, + "median_e2e_s": 5.55007201503031, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 22.190961834043264, + "output_tok_s": 23.07245642748744, + "median_first_sse_s": 8.439156876062043, + "median_tpot_est_s": 0.0427968583077075, + "median_e2e_s": 13.874486317043193, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 88.77933540102094, + "output_tok_s": 23.068431304977402, + "median_first_sse_s": 41.72694445645902, + "median_tpot_est_s": 0.04281781166131923, + "median_e2e_s": 47.15564054495189, + "all_text_equal": true + } + ] + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + }, + "limitations": [ + "One successful retry; not proof that every connection reset is resolved.", + "Host-wide listen counters do not identify the individual socket.", + "Output reference is retained same-route cache-off evidence, not a fresh paired off/on speedup.", + "RNGD-to-ATOM whole/stream and remaining GPU model/routes still running." + ] +} diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md index e29e7236..801366c7 100644 --- a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md @@ -34,3 +34,11 @@ Use distinct server-config and result paths. Compare fresh server processes on t ## Limits One successful hardware retry supports the backlog hypothesis but does not establish it as the sole reset cause. Counters are host-wide. No before/after throughput gain is claimed because the original did not finish timed measurements. ATOM Qwen4 also reset underc16 on the original listener with no OOM;its candidate retry and remaining GPU model/routes are pending. Engine concurrency,KV-cache correctness and resource limits are unchanged. Related#267 and#201;not closing either issue. + +## ATOM Qwen3-4B cache-on retry + +The original native ATOM run reset one SSE connection at input 512, output 128, concurrency 16. The server remained alive and cgroup OOM counters were zero. All 168 token checks completed, but only 11 timed batches completed; the original remains a failed run. Host-wide listen drops increased 82. + +With the identical backlog candidate, the retry completed all 24 conditions and 344 responses, matching retained same-route reference outputs. Cache hits were 867456 of 935680 queried tokens; listen drop/overflow increments were both zero. This uses one ATOM execution lane, B1/k3/context9216, prefix caching enabled. At 8192/128/c4, throughput was 23.0725 tok/s and median first SSE 8.43916 s; this is a single warm sample, not a paired cache speedup. Exact raw hashes, package versions and timings are in ../evidence267/qwen4-atom-backlog-controls.json. + +The independent retry auditor passed 13 corruption/accounting selfchecks. Combined with the earlier GPU retry, two previously failing configurations now completed, but broader validation remains in progress. No merge or universal reliability claim. From 6adfcb0d1300fe086c897e48627e4e95a32b9ad3 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 30 Oct 2026 14:36:10 +0900 Subject: [PATCH 3/8] docs: retain full Qwen4 cache paired controls for HTTP retry --- .../pd/http_matrix/.research/events.jsonl | 1 + opt_prime/examples/pd/http_matrix/STATUS.md | 2 + .../evidence267/qwen4-whole-cache-pair.json | 660 ++++++++++++++++++ .../experiments/2026-10-30-http-backlog267.md | 10 +- 4 files changed, 672 insertions(+), 1 deletion(-) create mode 100644 opt_prime/examples/pd/http_matrix/evidence267/qwen4-whole-cache-pair.json diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index 40adaf92..30fe6ca0 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -31,3 +31,4 @@ {"ts": "2026-10-30T04:24:12.257264+00:00", "task": "aicomp #201 / #265", "result": "ATOM short/long final-HTTP sweep:12 arms96conditions528responses audited;whole/stream16/16 references exact. Native/PD Qwen0.6/Qwen4=4/4,Gemma2/4(other2afterEOS),KONI2/4(preEOS). Eleven selfchecks pass;coordinator/cleanup0,fresh owned-Pod absence verified.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-atom-edge201-final.md"], "runtime_scope": "historical stream265 B1 engine,final HTTP SHA256e3449979b88d03edb6d5bf0d402a708e5d98054fdf203e22b480f3d7e6fd759a"} {"ts": "2026-10-30T05:14:21.740281+00:00", "task": "aicomp #267 / #201", "result": "HTTP backlog128 candidate:15CPUtests pass;Qwen4 whole/cacheoff24conditions344responses exact in one retry after originalreset. Remaining hardware verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} {"ts": "2026-10-30T05:26:15.716445+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 native ATOM cache-on retry:24 conditions,344 responses exact;13 auditor selfchecks pass;host listen drops +82 original/+0 retry. Wider hardware verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T05:36:34.408243+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 RNGD-GPU whole KV paired cache off/on:48 conditions,688 responses exact. Across24 conditions throughput improves11,range -3.72% to +15.26%;single ordered pair. Full paired evidence retained;remaining hardware runs pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index b2a9f15b..e2cf2b3a 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -152,3 +152,5 @@ All four models complete native/whole/stream at128/8192 input,32/128 output,c1/c Backlog128 handles a16connection accept-pause regression;15HTTPtests pass. Qwen4 RNGD-GPU whole/cacheoff retry24conditions344responses exact after originalc16 connection reset;listen drops+36before/+0retry(host-wide). One retry only;ATOM/remaining GPU verification pending. See experiments/2026-10-30-http-backlog267.md. PR289 follow-up: Qwen4 native ATOM cache-on retry also passes 24 conditions/344 responses; listen drops +82 original/+0 retry, no OOM in original. Both GPU and ATOM have one successful retry; other routes/models remain pending. + +PR289 paired-cache follow-up: Qwen4 RNGD-GPU whole KV paired cache off/on:48 conditions,688 responses exact. Across24 conditions throughput improves11,range -3.72% to +15.26%;single ordered pair. Full paired evidence retained;remaining hardware runs pending. diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-whole-cache-pair.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-whole-cache-pair.json new file mode 100644 index 00000000..a2c55251 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-whole-cache-pair.json @@ -0,0 +1,660 @@ +{ + "model": "Qwen3-4B", + "route": "RNGD-GPU whole", + "raw_sha256": { + "off": "c8100535bfc02fe0b532c39323cb23d8362026fa48cedea19684a277565fcf18", + "on": "06fae33330c3d3499a0fa80e56c9494117f077e64bb6e4220dded2f591cc9e72" + }, + "reference_tokens_equal": true, + "conditions": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.5938640710664913, + "output_tok_s": 8.90406519757546, + "median_first_sse_s": 2.769333482021466, + "median_tpot_est_s": 0.02659360800058611, + "median_e2e_s": 3.5937353300396353, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.66532118502073, + "output_tok_s": 8.730476371559513, + "median_first_sse_s": 2.918039128999226, + "median_tpot_est_s": 0.024100132226463285, + "median_e2e_s": 3.6651432280195877, + "all_text_equal": true + }, + "throughput_change_pct": -1.9495457654916226, + "first_sse_change_pct": 5.369726973770339 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 128, + "wall_s": 11.845253351028077, + "output_tok_s": 10.80601623340463, + "median_first_sse_s": 6.894638849480543, + "median_tpot_est_s": 0.026379545144135912, + "median_e2e_s": 7.7222023899666965, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 128, + "wall_s": 12.30348469002638, + "output_tok_s": 10.403556652836828, + "median_first_sse_s": 7.266616121982224, + "median_tpot_est_s": 0.02600571070985508, + "median_e2e_s": 8.072793153987732, + "all_text_equal": true + }, + "throughput_change_pct": -3.7244028870110246, + "first_sse_change_pct": 5.3951668915871664 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 512, + "wall_s": 46.374117970932275, + "output_tok_s": 11.040641254264422, + "median_first_sse_s": 24.28165605297545, + "median_tpot_est_s": 0.026395979968290177, + "median_e2e_s": 25.099912968988065, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 512, + "wall_s": 46.18980112497229, + "output_tok_s": 11.084698083343548, + "median_first_sse_s": 24.03905806154944, + "median_tpot_est_s": 0.026055202466405688, + "median_e2e_s": 24.958345417049713, + "all_text_equal": true + }, + "throughput_change_pct": 0.3990423025665324, + "first_sse_change_pct": -0.9990998591559497 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.118984114029445, + "output_tok_s": 20.918505035259855, + "median_first_sse_s": 2.848816549987532, + "median_tpot_est_s": 0.025747966134048058, + "median_e2e_s": 6.118808249011636, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.242904701037332, + "output_tok_s": 20.50327630001004, + "median_first_sse_s": 2.869474889943376, + "median_tpot_est_s": 0.026561346441309343, + "median_e2e_s": 6.242765887989663, + "all_text_equal": true + }, + "throughput_change_pct": -1.9849828395954394, + "first_sse_change_pct": 0.7251551510375265 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 512, + "wall_s": 14.784803000977263, + "output_tok_s": 34.630153676458, + "median_first_sse_s": 7.131008746451698, + "median_tpot_est_s": 0.027017266000071028, + "median_e2e_s": 10.569683398469351, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 512, + "wall_s": 15.31299539806787, + "output_tok_s": 33.43565296601618, + "median_first_sse_s": 7.409371640475001, + "median_tpot_est_s": 0.026311591090671133, + "median_e2e_s": 10.784471123944968, + "all_text_equal": true + }, + "throughput_change_pct": -3.4493081422675265, + "first_sse_change_pct": 3.903555638770073 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.64825740107335, + "output_tok_s": 42.0981163439333, + "median_first_sse_s": 24.06720970547758, + "median_tpot_est_s": 0.026488267799645047, + "median_e2e_s": 27.375394922448322, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.35642505402211, + "output_tok_s": 41.494091149397505, + "median_first_sse_s": 24.396923166990746, + "median_tpot_est_s": 0.026859728673080674, + "median_e2e_s": 27.779408978007268, + "all_text_equal": true + }, + "throughput_change_pct": -1.4348033760015033, + "first_sse_change_pct": 1.3699696206915357 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.636839246028103, + "output_tok_s": 8.798849175131433, + "median_first_sse_s": 2.866227310965769, + "median_tpot_est_s": 0.024852504938720696, + "median_e2e_s": 3.6366549640661106, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.7718445679638535, + "output_tok_s": 8.483912691363761, + "median_first_sse_s": 2.9373642350547016, + "median_tpot_est_s": 0.026914485902976123, + "median_e2e_s": 3.7717132980469614, + "all_text_equal": true + }, + "throughput_change_pct": -3.579291763038639, + "first_sse_change_pct": 2.481900992875663 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 128, + "wall_s": 12.187823576037772, + "output_tok_s": 10.502285268688837, + "median_first_sse_s": 7.140909380977973, + "median_tpot_est_s": 0.025063636806613256, + "median_e2e_s": 7.917882121982984, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 128, + "wall_s": 12.398822656017728, + "output_tok_s": 10.323560837276402, + "median_first_sse_s": 7.272774892975576, + "median_tpot_est_s": 0.027120182258752925, + "median_e2e_s": 8.096748902986292, + "all_text_equal": true + }, + "throughput_change_pct": -1.7017670615487757, + "first_sse_change_pct": 1.8466207168076787 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 512, + "wall_s": 46.370749281020835, + "output_tok_s": 11.041443322322104, + "median_first_sse_s": 24.260911067016423, + "median_tpot_est_s": 0.025874566645436588, + "median_e2e_s": 25.080011933052447, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 512, + "wall_s": 45.94501227198634, + "output_tok_s": 11.143755865577978, + "median_first_sse_s": 24.065952128032222, + "median_tpot_est_s": 0.02582315859692772, + "median_e2e_s": 24.863611125038005, + "all_text_equal": true + }, + "throughput_change_pct": 0.9266229085198585, + "first_sse_change_pct": -0.8035928182814733 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.158095913007855, + "output_tok_s": 20.78564572689154, + "median_first_sse_s": 2.877790122060105, + "median_tpot_est_s": 0.025827755779394602, + "median_e2e_s": 6.15791510604322, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.160433033946902, + "output_tok_s": 20.777760150083513, + "median_first_sse_s": 2.8113285440485924, + "median_tpot_est_s": 0.02636973762162411, + "median_e2e_s": 6.1602852219948545, + "all_text_equal": true + }, + "throughput_change_pct": -0.03793760805721469, + "first_sse_change_pct": -2.3094657773005123 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 512, + "wall_s": 14.883136463933624, + "output_tok_s": 34.4013509007817, + "median_first_sse_s": 7.182273587503005, + "median_tpot_est_s": 0.026408043138198847, + "median_e2e_s": 10.536095066054258, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 512, + "wall_s": 14.768116903025657, + "output_tok_s": 34.66928135537054, + "median_first_sse_s": 7.088801535486709, + "median_tpot_est_s": 0.026913238858426447, + "median_e2e_s": 10.556075934495311, + "all_text_equal": true + }, + "throughput_change_pct": 0.7788370153299917, + "first_sse_change_pct": -1.3014270603522338 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.92640683695208, + "output_tok_s": 41.85878613209811, + "median_first_sse_s": 24.215235407522414, + "median_tpot_est_s": 0.026753554602362596, + "median_e2e_s": 27.697570833028294, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.03478913893923, + "output_tok_s": 41.76626505310398, + "median_first_sse_s": 24.075821354053915, + "median_tpot_est_s": 0.026899172204883963, + "median_e2e_s": 27.450733256991953, + "all_text_equal": true + }, + "throughput_change_pct": -0.22103144296194976, + "first_sse_change_pct": -0.5757286729708655 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.843274821061641, + "output_tok_s": 8.326232572449902, + "median_first_sse_s": 3.003903826000169, + "median_tpot_est_s": 0.027072391161064226, + "median_e2e_s": 3.84314795199316, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.945928778965026, + "output_tok_s": 8.109624322310564, + "median_first_sse_s": 3.141313388943672, + "median_tpot_est_s": 0.025950086614747923, + "median_e2e_s": 3.9457660740008578, + "all_text_equal": true + }, + "throughput_change_pct": -2.60151573060855, + "first_sse_change_pct": 4.574366254810158 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 128, + "wall_s": 12.749716062913649, + "output_tok_s": 10.039439260324093, + "median_first_sse_s": 7.466653063020203, + "median_tpot_est_s": 0.025596863032513927, + "median_e2e_s": 8.284569455485325, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 128, + "wall_s": 12.172303085098974, + "output_tok_s": 10.515676376535051, + "median_first_sse_s": 7.091708452033345, + "median_tpot_est_s": 0.0255083825792216, + "median_e2e_s": 7.890375397459138, + "all_text_equal": true + }, + "throughput_change_pct": 4.743662508055091, + "first_sse_change_pct": -5.021588760348761 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 512, + "wall_s": 46.98791066301055, + "output_tok_s": 10.896419797678993, + "median_first_sse_s": 24.6336495334981, + "median_tpot_est_s": 0.0250324510477273, + "median_e2e_s": 25.404889553028625, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 512, + "wall_s": 46.20221433893312, + "output_tok_s": 11.08171994190664, + "median_first_sse_s": 24.13236670056358, + "median_tpot_est_s": 0.025335372613382437, + "median_e2e_s": 24.96728817099938, + "all_text_equal": true + }, + "throughput_change_pct": 1.700559887267894, + "first_sse_change_pct": -2.034951549720032 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.079132216051221, + "output_tok_s": 21.055636799941826, + "median_first_sse_s": 2.9262310749618337, + "median_tpot_est_s": 0.024825054519685117, + "median_e2e_s": 6.0790129989618436, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.266925232950598, + "output_tok_s": 20.42468918042843, + "median_first_sse_s": 2.8313267999328673, + "median_tpot_est_s": 0.027050363503975838, + "median_e2e_s": 6.266722964937799, + "all_text_equal": true + }, + "throughput_change_pct": -2.9965734378318065, + "first_sse_change_pct": -3.2432255894283446 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 512, + "wall_s": 14.522591115906835, + "output_tok_s": 35.255416606696166, + "median_first_sse_s": 6.977572105941363, + "median_tpot_est_s": 0.02599662249196526, + "median_e2e_s": 10.286594730452634, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 512, + "wall_s": 14.83509479695931, + "output_tok_s": 34.512755530550606, + "median_first_sse_s": 7.151992013503332, + "median_tpot_est_s": 0.027155561228722744, + "median_e2e_s": 10.588076097541489, + "all_text_equal": true + }, + "throughput_change_pct": -2.10651623956275, + "first_sse_change_pct": 2.4997220367447115 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.192781265010126, + "output_tok_s": 41.63212461940433, + "median_first_sse_s": 24.612169308995362, + "median_tpot_est_s": 0.026611734634012306, + "median_e2e_s": 27.96816182450857, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.54142292402685, + "output_tok_s": 41.33914367257204, + "median_first_sse_s": 24.450606751954183, + "median_tpot_est_s": 0.02662571351590207, + "median_e2e_s": 27.830910447984934, + "all_text_equal": true + }, + "throughput_change_pct": -0.7037376773601722, + "first_sse_change_pct": -0.6564336325369347 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 32, + "wall_s": 4.199458156013861, + "output_tok_s": 7.620030682809446, + "median_first_sse_s": 3.3747121680062264, + "median_tpot_est_s": 0.02660094451669964, + "median_e2e_s": 4.199341448023915, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 32, + "wall_s": 3.643375781015493, + "output_tok_s": 8.78306326971325, + "median_first_sse_s": 2.857726918067783, + "median_tpot_est_s": 0.025337460514668737, + "median_e2e_s": 3.643188194022514, + "all_text_equal": true + }, + "throughput_change_pct": 15.262833383697117, + "first_sse_change_pct": -15.319387971504462 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 128, + "wall_s": 13.205650482908823, + "output_tok_s": 9.692820521462515, + "median_first_sse_s": 8.0374622924719, + "median_tpot_est_s": 0.026171039097056155, + "median_e2e_s": 8.848766844486818, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 128, + "wall_s": 12.517737936927006, + "output_tok_s": 10.225489672730989, + "median_first_sse_s": 7.47981611196883, + "median_tpot_est_s": 0.026645917452931885, + "median_e2e_s": 8.298069035517983, + "all_text_equal": true + }, + "throughput_change_pct": 5.4955020583430825, + "first_sse_change_pct": -6.938087672590099 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 512, + "wall_s": 51.270903732045554, + "output_tok_s": 9.986170766090625, + "median_first_sse_s": 26.803447068086825, + "median_tpot_est_s": 0.1057412954021035, + "median_e2e_s": 30.457469970046077, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 512, + "wall_s": 46.68368743208703, + "output_tok_s": 10.96742841372226, + "median_first_sse_s": 25.86334280553274, + "median_tpot_est_s": 0.028045082806549486, + "median_e2e_s": 26.712535800528713, + "all_text_equal": true + }, + "throughput_change_pct": 9.826165310167001, + "first_sse_change_pct": -3.5074005972664968 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "off": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.657406355952844, + "output_tok_s": 19.226706791834392, + "median_first_sse_s": 3.4630271260393783, + "median_tpot_est_s": 0.02515140170869454, + "median_e2e_s": 6.657255143043585, + "all_text_equal": true + }, + "on": { + "requests": 1, + "output_tokens": 128, + "wall_s": 6.16653712594416, + "output_tok_s": 20.75719279487868, + "median_first_sse_s": 2.874457805068232, + "median_tpot_est_s": 0.025920808952353075, + "median_e2e_s": 6.166400542017072, + "all_text_equal": true + }, + "throughput_change_pct": 7.960208784659306, + "first_sse_change_pct": -16.995804524473524 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "off": { + "requests": 4, + "output_tokens": 512, + "wall_s": 16.016934926970862, + "output_tok_s": 31.966165957123604, + "median_first_sse_s": 8.493684782530181, + "median_tpot_est_s": 0.02901831886632089, + "median_e2e_s": 12.179011278552935, + "all_text_equal": true + }, + "on": { + "requests": 4, + "output_tokens": 512, + "wall_s": 14.728964269976132, + "output_tok_s": 34.76143947498555, + "median_first_sse_s": 7.113750016025733, + "median_tpot_est_s": 0.027268773952213034, + "median_e2e_s": 10.576884307956789, + "all_text_equal": true + }, + "throughput_change_pct": 8.744475398179642, + "first_sse_change_pct": -16.24659734656858 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "off": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 53.753593918983825, + "output_tok_s": 38.09977809272992, + "median_first_sse_s": 27.78725230850978, + "median_tpot_est_s": 0.030791497555117703, + "median_e2e_s": 32.433394862979185, + "all_text_equal": true + }, + "on": { + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.11081874195952, + "output_tok_s": 41.701605724813966, + "median_first_sse_s": 24.4042886030511, + "median_tpot_est_s": 0.027804161051625576, + "median_e2e_s": 28.926480908005033, + "all_text_equal": true + }, + "throughput_change_pct": 9.453670893614309, + "first_sse_change_pct": -12.17451681763675 + } + ], + "limits": "Single ordered off/on pair per condition; no statistical speedup guarantee. Logical transferred tokens are not PCIe byte measurements. Producer still computes full prompt." +} diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md index 801366c7..4a03c277 100644 --- a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md @@ -33,7 +33,7 @@ Use distinct server-config and result paths. Compare fresh server processes on t ## Limits -One successful hardware retry supports the backlog hypothesis but does not establish it as the sole reset cause. Counters are host-wide. No before/after throughput gain is claimed because the original did not finish timed measurements. ATOM Qwen4 also reset underc16 on the original listener with no OOM;its candidate retry and remaining GPU model/routes are pending. Engine concurrency,KV-cache correctness and resource limits are unchanged. Related#267 and#201;not closing either issue. +One successful hardware retry supports the backlog hypothesis but does not establish it as the sole reset cause. Counters are host-wide. No before/after throughput gain is claimed because the original did not finish timed measurements. ATOM Qwen4 also reset underc16 on the original listener with no OOM;its native candidate retry passed as detailed below; remaining ATOM and GPU model/routes are pending. Engine concurrency,KV-cache correctness and resource limits are unchanged. Related#267 and#201;not closing either issue. ## ATOM Qwen3-4B cache-on retry @@ -42,3 +42,11 @@ The original native ATOM run reset one SSE connection at input 512, output 128, With the identical backlog candidate, the retry completed all 24 conditions and 344 responses, matching retained same-route reference outputs. Cache hits were 867456 of 935680 queried tokens; listen drop/overflow increments were both zero. This uses one ATOM execution lane, B1/k3/context9216, prefix caching enabled. At 8192/128/c4, throughput was 23.0725 tok/s and median first SSE 8.43916 s; this is a single warm sample, not a paired cache speedup. Exact raw hashes, package versions and timings are in ../evidence267/qwen4-atom-backlog-controls.json. The independent retry auditor passed 13 corruption/accounting selfchecks. Combined with the earlier GPU retry, two previously failing configurations now completed, but broader validation remains in progress. No merge or universal reliability claim. + +## Qwen3-4B whole-KV paired cache control + +Both fresh off/on processes using the same HTTP candidate completed 24 conditions and 344 responses each (48 conditions, 688 responses total), with identical token references and matching streamed text. Host listen drops/overflows increased zero in both runs. All 24 paired timing rows and raw SHA256 fingerprints are retained in ../evidence267/qwen4-whole-cache-pair.json. + +At input 8192, output 128, concurrency 4, cache off/on throughput was 31.9662/34.7614 tok/s and median first SSE was 8.49368/7.11375 s. Across all 24 conditions, throughput increased in only 11; changes ranged from -3.72% to +15.26%. These are single ordered off/on samples, not evidence of a universal or repeatable cache speedup, and not a backlog performance comparison. + +Both runs queried 935680 prefix tokens. Cache-on hit 850688 tokens, reducing logical transferred tokens from 935680 to 84992. The producer still performs full prefill and copies tensors to CPU before slicing. These counters therefore do not establish reduced producer compute or physical PCIe traffic. Streamed-KV controls remain in progress. From cddc511d4eafd0559f90b614136c8e3969048462 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 30 Oct 2026 14:48:24 +0900 Subject: [PATCH 4/8] docs: retain Qwen4 streamed KV retry comparison --- .../pd/http_matrix/.research/events.jsonl | 1 + opt_prime/examples/pd/http_matrix/STATUS.md | 2 + .../qwen4-off-route-comparison.json | 853 ++++++++++++++++++ .../experiments/2026-10-30-http-backlog267.md | 6 + 4 files changed, 862 insertions(+) create mode 100644 opt_prime/examples/pd/http_matrix/evidence267/qwen4-off-route-comparison.json diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index 30fe6ca0..1dbedcac 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -32,3 +32,4 @@ {"ts": "2026-10-30T05:14:21.740281+00:00", "task": "aicomp #267 / #201", "result": "HTTP backlog128 candidate:15CPUtests pass;Qwen4 whole/cacheoff24conditions344responses exact in one retry after originalreset. Remaining hardware verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} {"ts": "2026-10-30T05:26:15.716445+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 native ATOM cache-on retry:24 conditions,344 responses exact;13 auditor selfchecks pass;host listen drops +82 original/+0 retry. Wider hardware verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} {"ts": "2026-10-30T05:36:34.408243+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 RNGD-GPU whole KV paired cache off/on:48 conditions,688 responses exact. Across24 conditions throughput improves11,range -3.72% to +15.26%;single ordered pair. Full paired evidence retained;remaining hardware runs pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T05:48:16.366880+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 GPU stream/off24conditions344responses exact;whole/stream references8/8equal. Same candidate,only stream flag differs. Single ordered pair all24 throughput increases(+4.33..30.38%);8K128c4 31.97->37.54tok/s. Remaining verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index e2cf2b3a..192da76d 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -154,3 +154,5 @@ Backlog128 handles a16connection accept-pause regression;15HTTPtests pass. Qwen4 PR289 follow-up: Qwen4 native ATOM cache-on retry also passes 24 conditions/344 responses; listen drops +82 original/+0 retry, no OOM in original. Both GPU and ATOM have one successful retry; other routes/models remain pending. PR289 paired-cache follow-up: Qwen4 RNGD-GPU whole KV paired cache off/on:48 conditions,688 responses exact. Across24 conditions throughput improves11,range -3.72% to +15.26%;single ordered pair. Full paired evidence retained;remaining hardware runs pending. + +PR289 streamed-control follow-up: Qwen4 GPU stream/off24conditions344responses exact;whole/stream references8/8equal. Same candidate,only stream flag differs. Single ordered pair all24 throughput increases(+4.33..30.38%);8K128c4 31.97->37.54tok/s. Remaining verification pending. diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-off-route-comparison.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-off-route-comparison.json new file mode 100644 index 00000000..fd901e5d --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-off-route-comparison.json @@ -0,0 +1,853 @@ +{ + "model": "Qwen3-4B", + "comparison": "RNGD-GPU whole versus streamed KV, decoder/producer cache off", + "runtime_base": "f52c41305f6bb2ee65d3c0fe8166acbca51dcc72", + "raw_sha256": { + "whole": "c8100535bfc02fe0b532c39323cb23d8362026fa48cedea19684a277565fcf18", + "stream": "3ddd7794442c6e6e5a7904871ca859c521738c8768433d35e8d20399420493a6" + }, + "reference_tokens_equal": true, + "conditions": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "whole": { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.5938640710664913, + "output_tok_s": 8.90406519757546, + "median_first_sse_s": 2.769333482021466, + "median_tpot_est_s": 0.02659360800058611, + "median_e2e_s": 3.5937353300396353, + "all_text_equal": true + }, + "stream": { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.0344492259901017, + "output_tok_s": 10.545571079561832, + "median_first_sse_s": 2.222173067042604, + "median_tpot_est_s": 0.026198500128943595, + "median_e2e_s": 3.0343265710398555, + "all_text_equal": true + }, + "throughput_change_pct": 18.435465661609808, + "first_sse_change_pct": -19.757837708280036 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "whole": { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 11.845253351028077, + "output_tok_s": 10.80601623340463, + "median_first_sse_s": 6.894638849480543, + "median_tpot_est_s": 0.026379545144135912, + "median_e2e_s": 7.7222023899666965, + "all_text_equal": true + }, + "stream": { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.291457520914264, + "output_tok_s": 13.776094838929534, + "median_first_sse_s": 5.330548713565804, + "median_tpot_est_s": 0.025170327402696376, + "median_e2e_s": 6.114778548071627, + "all_text_equal": true + }, + "throughput_change_pct": 27.48541683977397, + "first_sse_change_pct": -22.685599203395277 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "whole": { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.374117970932275, + "output_tok_s": 11.040641254264422, + "median_first_sse_s": 24.28165605297545, + "median_tpot_est_s": 0.026395979968290177, + "median_e2e_s": 25.099912968988065, + "all_text_equal": true + }, + "stream": { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.56748351501301, + "output_tok_s": 14.395170796491271, + "median_first_sse_s": 18.518849209474865, + "median_tpot_est_s": 0.026875704419874254, + "median_e2e_s": 19.44195634953212, + "all_text_equal": true + }, + "throughput_change_pct": 30.38346654847761, + "first_sse_change_pct": -23.733170550343974 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "whole": { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.118984114029445, + "output_tok_s": 20.918505035259855, + "median_first_sse_s": 2.848816549987532, + "median_tpot_est_s": 0.025747966134048058, + "median_e2e_s": 6.118808249011636, + "all_text_equal": true + }, + "stream": { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.373530109995045, + "output_tok_s": 23.820467621817798, + "median_first_sse_s": 2.137613946921192, + "median_tpot_est_s": 0.025478321228425686, + "median_e2e_s": 5.373360742931254, + "all_text_equal": true + }, + "throughput_change_pct": 13.872705442699873, + "first_sse_change_pct": -24.964843842593254 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "whole": { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.784803000977263, + "output_tok_s": 34.630153676458, + "median_first_sse_s": 7.131008746451698, + "median_tpot_est_s": 0.027017266000071028, + "median_e2e_s": 10.569683398469351, + "all_text_equal": true + }, + "stream": { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 11.927924883086234, + "output_tok_s": 42.924482256424554, + "median_first_sse_s": 5.350217982544564, + "median_tpot_est_s": 0.027437909999880647, + "median_e2e_s": 8.846565436047968, + "all_text_equal": true + }, + "throughput_change_pct": 23.951174625035353, + "first_sse_change_pct": -24.972494456597516 + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "whole": { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.64825740107335, + "output_tok_s": 42.0981163439333, + "median_first_sse_s": 24.06720970547758, + "median_tpot_est_s": 0.026488267799645047, + "median_e2e_s": 27.375394922448322, + "all_text_equal": true + }, + "stream": { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 37.57740541594103, + "output_tok_s": 54.50083573708366, + "median_first_sse_s": 18.221833322022576, + "median_tpot_est_s": 0.02667981534270482, + "median_e2e_s": 21.64534808701137, + "all_text_equal": true + }, + "throughput_change_pct": 29.46145925347965, + "first_sse_change_pct": -24.287719494647632 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "whole": { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.636839246028103, + "output_tok_s": 8.798849175131433, + "median_first_sse_s": 2.866227310965769, + "median_tpot_est_s": 0.024852504938720696, + "median_e2e_s": 3.6366549640661106, + "all_text_equal": true + }, + "stream": { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.211695746984333, + "output_tok_s": 9.963583888681502, + "median_first_sse_s": 2.3731549710500985, + "median_tpot_est_s": 0.02704614116209409, + "median_e2e_s": 3.2115853470750153, + "all_text_equal": true + }, + "throughput_change_pct": 13.237352867031849, + "first_sse_change_pct": -17.202834472661934 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "whole": { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.187823576037772, + "output_tok_s": 10.502285268688837, + "median_first_sse_s": 7.140909380977973, + "median_tpot_est_s": 0.025063636806613256, + "median_e2e_s": 7.917882121982984, + "all_text_equal": true + }, + "stream": { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.43295849103015, + "output_tok_s": 13.5694437881515, + "median_first_sse_s": 5.399655768938828, + "median_tpot_est_s": 0.026163732065939374, + "median_e2e_s": 6.210731462982949, + "all_text_equal": true + }, + "throughput_change_pct": 29.204677277306356, + "first_sse_change_pct": -24.38419981462745 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "whole": { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.370749281020835, + "output_tok_s": 11.041443322322104, + "median_first_sse_s": 24.260911067016423, + "median_tpot_est_s": 0.025874566645436588, + "median_e2e_s": 25.080011933052447, + "all_text_equal": true + }, + "stream": { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.71247306501027, + "output_tok_s": 14.336727648851577, + "median_first_sse_s": 18.574257490516175, + "median_tpot_est_s": 0.02652731508151778, + "median_e2e_s": 19.40594940056326, + "all_text_equal": true + }, + "throughput_change_pct": 29.84468814749526, + "first_sse_change_pct": -23.43957141919315 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "whole": { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.158095913007855, + "output_tok_s": 20.78564572689154, + "median_first_sse_s": 2.877790122060105, + "median_tpot_est_s": 0.025827755779394602, + "median_e2e_s": 6.15791510604322, + "all_text_equal": true + }, + "stream": { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.338220724021085, + "output_tok_s": 23.97802687776131, + "median_first_sse_s": 2.165918902028352, + "median_tpot_est_s": 0.024977886991568027, + "median_e2e_s": 5.3381105499574915, + "all_text_equal": true + }, + "throughput_change_pct": 15.358585404636283, + "first_sse_change_pct": -24.736731653041833 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "whole": { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.883136463933624, + "output_tok_s": 34.4013509007817, + "median_first_sse_s": 7.182273587503005, + "median_tpot_est_s": 0.026408043138198847, + "median_e2e_s": 10.536095066054258, + "all_text_equal": true + }, + "stream": { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 11.92513715790119, + "output_tok_s": 42.934516661786674, + "median_first_sse_s": 5.504016306484118, + "median_tpot_est_s": 0.02519130685841518, + "median_e2e_s": 8.703312277502846, + "all_text_equal": true + }, + "throughput_change_pct": 24.80474033015685, + "first_sse_change_pct": -23.366657654743438 + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "whole": { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.92640683695208, + "output_tok_s": 41.85878613209811, + "median_first_sse_s": 24.215235407522414, + "median_tpot_est_s": 0.026753554602362596, + "median_e2e_s": 27.697570833028294, + "all_text_equal": true + }, + "stream": { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 37.93958489596844, + "output_tok_s": 53.980558975953, + "median_first_sse_s": 18.479511058481876, + "median_tpot_est_s": 0.0253564650866599, + "median_e2e_s": 21.677008820464835, + "all_text_equal": true + }, + "throughput_change_pct": 28.95872996794737, + "first_sse_change_pct": -23.686428203207754 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "whole": { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.843274821061641, + "output_tok_s": 8.326232572449902, + "median_first_sse_s": 3.003903826000169, + "median_tpot_est_s": 0.027072391161064226, + "median_e2e_s": 3.84314795199316, + "all_text_equal": true + }, + "stream": { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.058836486074142, + "output_tok_s": 10.461494148407501, + "median_first_sse_s": 2.2814225980546325, + "median_tpot_est_s": 0.02507314690193462, + "median_e2e_s": 3.0586901520146057, + "all_text_equal": true + }, + "throughput_change_pct": 25.64499078518201, + "first_sse_change_pct": -24.051410091499235 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "whole": { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.749716062913649, + "output_tok_s": 10.039439260324093, + "median_first_sse_s": 7.466653063020203, + "median_tpot_est_s": 0.025596863032513927, + "median_e2e_s": 8.284569455485325, + "all_text_equal": true + }, + "stream": { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.86284664098639, + "output_tok_s": 12.97799759636115, + "median_first_sse_s": 5.68974005500786, + "median_tpot_est_s": 0.025835917564108968, + "median_e2e_s": 6.488205989997368, + "all_text_equal": true + }, + "throughput_change_pct": 29.270144077172233, + "first_sse_change_pct": -23.797985429546607 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "whole": { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.98791066301055, + "output_tok_s": 10.896419797678993, + "median_first_sse_s": 24.6336495334981, + "median_tpot_est_s": 0.0250324510477273, + "median_e2e_s": 25.404889553028625, + "all_text_equal": true + }, + "stream": { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 37.1058676840039, + "output_tok_s": 13.798356754792179, + "median_first_sse_s": 19.3962357249693, + "median_tpot_est_s": 0.029022938258300025, + "median_e2e_s": 20.305324282497168, + "all_text_equal": true + }, + "throughput_change_pct": 26.632022361430295, + "first_sse_change_pct": -21.26121751227602 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "whole": { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.079132216051221, + "output_tok_s": 21.055636799941826, + "median_first_sse_s": 2.9262310749618337, + "median_tpot_est_s": 0.024825054519685117, + "median_e2e_s": 6.0790129989618436, + "all_text_equal": true + }, + "stream": { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.826587219024077, + "output_tok_s": 21.968262927923583, + "median_first_sse_s": 2.3179203160107136, + "median_tpot_est_s": 0.027626068841878647, + "median_e2e_s": 5.826431058929302, + "all_text_equal": true + }, + "throughput_change_pct": 4.334355387362487, + "first_sse_change_pct": -20.78819967965292 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "whole": { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.522591115906835, + "output_tok_s": 35.255416606696166, + "median_first_sse_s": 6.977572105941363, + "median_tpot_est_s": 0.02599662249196526, + "median_e2e_s": 10.286594730452634, + "all_text_equal": true + }, + "stream": { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 12.11718301300425, + "output_tok_s": 42.254045304962204, + "median_first_sse_s": 5.6174396550050005, + "median_tpot_est_s": 0.026623807287422984, + "median_e2e_s": 9.04161945101805, + "all_text_equal": true + }, + "throughput_change_pct": 19.85121542128301, + "first_sse_change_pct": -19.49291860098754 + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "whole": { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.192781265010126, + "output_tok_s": 41.63212461940433, + "median_first_sse_s": 24.612169308995362, + "median_tpot_est_s": 0.026611734634012306, + "median_e2e_s": 27.96816182450857, + "all_text_equal": true + }, + "stream": { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 39.340143382083625, + "output_tok_s": 52.058783317315125, + "median_first_sse_s": 19.44411149097141, + "median_tpot_est_s": 0.02653768402386794, + "median_e2e_s": 22.713961433502845, + "all_text_equal": true + }, + "throughput_change_pct": 25.044743195861386, + "first_sse_change_pct": -20.997977679826498 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "whole": { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.199458156013861, + "output_tok_s": 7.620030682809446, + "median_first_sse_s": 3.3747121680062264, + "median_tpot_est_s": 0.02660094451669964, + "median_e2e_s": 4.199341448023915, + "all_text_equal": true + }, + "stream": { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.316174581996165, + "output_tok_s": 9.649672901339729, + "median_first_sse_s": 2.5564451449317858, + "median_tpot_est_s": 0.024503057582243797, + "median_e2e_s": 3.3160399299813434, + "all_text_equal": true + }, + "throughput_change_pct": 26.63561740124083, + "first_sse_change_pct": -24.247016703586645 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "whole": { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 13.205650482908823, + "output_tok_s": 9.692820521462515, + "median_first_sse_s": 8.0374622924719, + "median_tpot_est_s": 0.026171039097056155, + "median_e2e_s": 8.848766844486818, + "all_text_equal": true + }, + "stream": { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 11.041767364018597, + "output_tok_s": 11.592347110763166, + "median_first_sse_s": 6.5184954924625345, + "median_tpot_est_s": 0.028367914580514714, + "median_e2e_s": 7.396435436501633, + "all_text_equal": true + }, + "throughput_change_pct": 19.59725329788773, + "first_sse_change_pct": -18.898586951158336 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "whole": { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 51.270903732045554, + "output_tok_s": 9.986170766090625, + "median_first_sse_s": 26.803447068086825, + "median_tpot_est_s": 0.1057412954021035, + "median_e2e_s": 30.457469970046077, + "all_text_equal": true + }, + "stream": { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 41.80970816500485, + "output_tok_s": 12.245959669925398, + "median_first_sse_s": 23.20848668902181, + "median_tpot_est_s": 0.03088053917482255, + "median_e2e_s": 24.188921280030627, + "all_text_equal": true + }, + "throughput_change_pct": 22.62918346547995, + "first_sse_change_pct": -13.412306148283836 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "whole": { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.657406355952844, + "output_tok_s": 19.226706791834392, + "median_first_sse_s": 3.4630271260393783, + "median_tpot_est_s": 0.02515140170869454, + "median_e2e_s": 6.657255143043585, + "all_text_equal": true + }, + "stream": { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.676418976043351, + "output_tok_s": 22.549427824163214, + "median_first_sse_s": 2.630975637002848, + "median_tpot_est_s": 0.02397870525211741, + "median_e2e_s": 5.676271204021759, + "all_text_equal": true + }, + "throughput_change_pct": 17.281800093503197, + "first_sse_change_pct": -24.02671012248574 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "whole": { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 16.016934926970862, + "output_tok_s": 31.966165957123604, + "median_first_sse_s": 8.493684782530181, + "median_tpot_est_s": 0.02901831886632089, + "median_e2e_s": 12.179011278552935, + "all_text_equal": true + }, + "stream": { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 13.63700374995824, + "output_tok_s": 37.54490424640148, + "median_first_sse_s": 6.511755163548514, + "median_tpot_est_s": 0.029209394330584157, + "median_e2e_s": 10.221348243532702, + "all_text_equal": true + }, + "throughput_change_pct": 17.452009405071188, + "first_sse_change_pct": -23.334155548815538 + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "whole": { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 53.753593918983825, + "output_tok_s": 38.09977809272992, + "median_first_sse_s": 27.78725230850978, + "median_tpot_est_s": 0.030791497555117703, + "median_e2e_s": 32.433394862979185, + "all_text_equal": true + }, + "stream": { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 44.1992259939434, + "output_tok_s": 46.33565303339557, + "median_first_sse_s": 22.067861926509067, + "median_tpot_est_s": 0.03135937557869231, + "median_e2e_s": 26.46022372151492, + "all_text_equal": true + }, + "throughput_change_pct": 21.616595562894368, + "first_sse_change_pct": -20.582784934979557 + } + ], + "limits": [ + "One ordered whole/stream pair, not repeated ABBA or a stable speedup estimate.", + "Both routes use one A10080GB plus one RNGD research producer, four GPU slots, BF16, context9216, producer width8192, page/chunk256.", + "No proof of physical DMA overlap or reduced producer compute from timing alone.", + "Same-route and whole/stream parity here does not establish native-decoder equivalence or answer quality." + ], + "aggregate_scope": { + "conditions": 24, + "responses": 688, + "throughput_increased_conditions": 24, + "min_throughput_change_pct": 4.334355387362487, + "max_throughput_change_pct": 30.38346654847761 + }, + "config_comparison": { + "metadata_equal": true, + "only_added_server_argument": "--pd-prefill-stream-layers" + }, + "listen_counter_delta": { + "whole": { + "ListenOverflows": 0, + "ListenDrops": 0 + }, + "stream": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "source_sha256": { + "benchmark201_native_http.py": "e1ea87a51343d1af83c40aaa9c41ae2a20e9864aa8b53bd475d047d175c41433", + "llmd.py": "9f3aaf0902f7b1958f32be12b1f276384f20cc3e0070571af10bb7bce148c1f0", + "serve_batched.py": "cf509e2c392a39aab7b47d7296447ff62cc8237e193e42d7d9cd76fdc47d9c30", + "streaming_factories.py": "77745350973cc54bf508e0c8b473efc442afef20b1c64f7c4edc179ea1471389", + "stream265-python.sh": "1a417cc6dde8ec6837362153406e618b006169513851131f8d324ff46aa96d9a", + "fsdk.py": "2cf093e93fbc2a98c1ce242563e65d7feaaabff468a91654392412c74df98bb6" + }, + "config_sha256": { + "whole": "a0dea0a54b95edcb9677ba473ae7cc831d3ee6cfa5de6f7a4175ccbda532fe12", + "stream": "73c2047141df251249a8c9fdd60ef6ce1cfcf6838c507b923f50a9bbde6b8db9" + }, + "packages": { + "whole": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "stream": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + } + } +} diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md index 4a03c277..2ca0fdf2 100644 --- a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md @@ -50,3 +50,9 @@ Both fresh off/on processes using the same HTTP candidate completed 24 condition At input 8192, output 128, concurrency 4, cache off/on throughput was 31.9662/34.7614 tok/s and median first SSE was 8.49368/7.11375 s. Across all 24 conditions, throughput increased in only 11; changes ranged from -3.72% to +15.26%. These are single ordered off/on samples, not evidence of a universal or repeatable cache speedup, and not a backlog performance comparison. Both runs queried 935680 prefix tokens. Cache-on hit 850688 tokens, reducing logical transferred tokens from 935680 to 84992. The producer still performs full prefill and copies tensors to CPU before slicing. These counters therefore do not establish reduced producer compute or physical PCIe traffic. Streamed-KV controls remain in progress. + +## Qwen3-4B streamed KV cache-off retry + +Streamed KV cache-off completed all 24 conditions and 344 responses with exact same-route outputs. All eight prompt/output references also equal the whole-KV cache-off control. Resource metadata is identical; the only added server argument is `--pd-prefill-stream-layers`. Both runs use the same backlog candidate; host listen drops/overflows increased zero. Exact raw/config/source fingerprints, package versions and all 24 paired rows are in ../evidence267/qwen4-off-route-comparison.json. Reproduce with the whole/off command above and add that server argument; keep both caches disabled and use distinct result files. + +Throughput increased in 24/24 conditions in this single ordered whole/stream pair, ranging from +4.33% to +30.38%. At 8192 input, 128 output, concurrency 4: 31.9662 to 37.5449 tok/s (+17.45%); median first SSE 8.49368 to 6.51176 s (-23.33%); median end-to-end 12.1790 to 10.2213 s. These compare transfer modes, not the backlog patch. One sample per condition does not establish repeatability; timing alone does not establish physical DMA overlap. Cache-on streaming, remaining GPU models and ATOM PD are still running. From 1e9177b1dbf668d169ad5c2d2aa8d045b718c308 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 30 Oct 2026 15:03:22 +0900 Subject: [PATCH 5/8] docs: complete Qwen4 GPU PD cache evidence --- .../pd/http_matrix/.research/events.jsonl | 1 + opt_prime/examples/pd/http_matrix/STATUS.md | 2 + .../qwen4-full-pd-cache-summary.json | 1392 +++++++++++++++++ .../experiments/2026-10-30-http-backlog267.md | 17 + 4 files changed, 1412 insertions(+) create mode 100644 opt_prime/examples/pd/http_matrix/evidence267/qwen4-full-pd-cache-summary.json diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index 1dbedcac..ef08cc2b 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -33,3 +33,4 @@ {"ts": "2026-10-30T05:26:15.716445+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 native ATOM cache-on retry:24 conditions,344 responses exact;13 auditor selfchecks pass;host listen drops +82 original/+0 retry. Wider hardware verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} {"ts": "2026-10-30T05:36:34.408243+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 RNGD-GPU whole KV paired cache off/on:48 conditions,688 responses exact. Across24 conditions throughput improves11,range -3.72% to +15.26%;single ordered pair. Full paired evidence retained;remaining hardware runs pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} {"ts": "2026-10-30T05:48:16.366880+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 GPU stream/off24conditions344responses exact;whole/stream references8/8equal. Same candidate,only stream flag differs. Single ordered pair all24 throughput increases(+4.33..30.38%);8K128c4 31.97->37.54tok/s. Remaining verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T06:03:17.316059+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 GPU PD four configurations complete96conditions1376responses exact;all4reference sets equal,listen deltas0. Full grid provenance/timings retained;single ordered samples only. Gemma GPU and Qwen4 ATOM pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index 192da76d..aff2fd02 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -156,3 +156,5 @@ PR289 follow-up: Qwen4 native ATOM cache-on retry also passes 24 conditions/344 PR289 paired-cache follow-up: Qwen4 RNGD-GPU whole KV paired cache off/on:48 conditions,688 responses exact. Across24 conditions throughput improves11,range -3.72% to +15.26%;single ordered pair. Full paired evidence retained;remaining hardware runs pending. PR289 streamed-control follow-up: Qwen4 GPU stream/off24conditions344responses exact;whole/stream references8/8equal. Same candidate,only stream flag differs. Single ordered pair all24 throughput increases(+4.33..30.38%);8K128c4 31.97->37.54tok/s. Remaining verification pending. + +PR289 Qwen4 GPU subset: Qwen4 GPU PD four configurations complete96conditions1376responses exact;all4reference sets equal,listen deltas0. Full grid provenance/timings retained;single ordered samples only. Gemma GPU and Qwen4 ATOM pending. diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-full-pd-cache-summary.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-full-pd-cache-summary.json new file mode 100644 index 00000000..44a3cf2e --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-full-pd-cache-summary.json @@ -0,0 +1,1392 @@ +{ + "model": "Qwen3-4B", + "runtime_base": "f52c41305f6bb2ee65d3c0fe8166acbca51dcc72", + "source_sha256": { + "benchmark201_native_http.py": "e1ea87a51343d1af83c40aaa9c41ae2a20e9864aa8b53bd475d047d175c41433", + "llmd.py": "9f3aaf0902f7b1958f32be12b1f276384f20cc3e0070571af10bb7bce148c1f0", + "serve_batched.py": "cf509e2c392a39aab7b47d7296447ff62cc8237e193e42d7d9cd76fdc47d9c30", + "streaming_factories.py": "77745350973cc54bf508e0c8b473efc442afef20b1c64f7c4edc179ea1471389", + "stream265-python.sh": "1a417cc6dde8ec6837362153406e618b006169513851131f8d324ff46aa96d9a", + "fsdk.py": "2cf093e93fbc2a98c1ce242563e65d7feaaabff468a91654392412c74df98bb6" + }, + "arms": { + "whole/off": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 0.0, + "transferred_tokens": 935680.0 + }, + "raw_sha256": "c8100535bfc02fe0b532c39323cb23d8362026fa48cedea19684a277565fcf18", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.5938640710664913, + "output_tok_s": 8.90406519757546, + "median_first_sse_s": 2.769333482021466, + "median_tpot_est_s": 0.02659360800058611, + "median_e2e_s": 3.5937353300396353, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 11.845253351028077, + "output_tok_s": 10.80601623340463, + "median_first_sse_s": 6.894638849480543, + "median_tpot_est_s": 0.026379545144135912, + "median_e2e_s": 7.7222023899666965, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.374117970932275, + "output_tok_s": 11.040641254264422, + "median_first_sse_s": 24.28165605297545, + "median_tpot_est_s": 0.026395979968290177, + "median_e2e_s": 25.099912968988065, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.118984114029445, + "output_tok_s": 20.918505035259855, + "median_first_sse_s": 2.848816549987532, + "median_tpot_est_s": 0.025747966134048058, + "median_e2e_s": 6.118808249011636, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.784803000977263, + "output_tok_s": 34.630153676458, + "median_first_sse_s": 7.131008746451698, + "median_tpot_est_s": 0.027017266000071028, + "median_e2e_s": 10.569683398469351, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.64825740107335, + "output_tok_s": 42.0981163439333, + "median_first_sse_s": 24.06720970547758, + "median_tpot_est_s": 0.026488267799645047, + "median_e2e_s": 27.375394922448322, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.636839246028103, + "output_tok_s": 8.798849175131433, + "median_first_sse_s": 2.866227310965769, + "median_tpot_est_s": 0.024852504938720696, + "median_e2e_s": 3.6366549640661106, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.187823576037772, + "output_tok_s": 10.502285268688837, + "median_first_sse_s": 7.140909380977973, + "median_tpot_est_s": 0.025063636806613256, + "median_e2e_s": 7.917882121982984, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.370749281020835, + "output_tok_s": 11.041443322322104, + "median_first_sse_s": 24.260911067016423, + "median_tpot_est_s": 0.025874566645436588, + "median_e2e_s": 25.080011933052447, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.158095913007855, + "output_tok_s": 20.78564572689154, + "median_first_sse_s": 2.877790122060105, + "median_tpot_est_s": 0.025827755779394602, + "median_e2e_s": 6.15791510604322, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.883136463933624, + "output_tok_s": 34.4013509007817, + "median_first_sse_s": 7.182273587503005, + "median_tpot_est_s": 0.026408043138198847, + "median_e2e_s": 10.536095066054258, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 48.92640683695208, + "output_tok_s": 41.85878613209811, + "median_first_sse_s": 24.215235407522414, + "median_tpot_est_s": 0.026753554602362596, + "median_e2e_s": 27.697570833028294, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.843274821061641, + "output_tok_s": 8.326232572449902, + "median_first_sse_s": 3.003903826000169, + "median_tpot_est_s": 0.027072391161064226, + "median_e2e_s": 3.84314795199316, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.749716062913649, + "output_tok_s": 10.039439260324093, + "median_first_sse_s": 7.466653063020203, + "median_tpot_est_s": 0.025596863032513927, + "median_e2e_s": 8.284569455485325, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.98791066301055, + "output_tok_s": 10.896419797678993, + "median_first_sse_s": 24.6336495334981, + "median_tpot_est_s": 0.0250324510477273, + "median_e2e_s": 25.404889553028625, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.079132216051221, + "output_tok_s": 21.055636799941826, + "median_first_sse_s": 2.9262310749618337, + "median_tpot_est_s": 0.024825054519685117, + "median_e2e_s": 6.0790129989618436, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.522591115906835, + "output_tok_s": 35.255416606696166, + "median_first_sse_s": 6.977572105941363, + "median_tpot_est_s": 0.02599662249196526, + "median_e2e_s": 10.286594730452634, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.192781265010126, + "output_tok_s": 41.63212461940433, + "median_first_sse_s": 24.612169308995362, + "median_tpot_est_s": 0.026611734634012306, + "median_e2e_s": 27.96816182450857, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.199458156013861, + "output_tok_s": 7.620030682809446, + "median_first_sse_s": 3.3747121680062264, + "median_tpot_est_s": 0.02660094451669964, + "median_e2e_s": 4.199341448023915, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 13.205650482908823, + "output_tok_s": 9.692820521462515, + "median_first_sse_s": 8.0374622924719, + "median_tpot_est_s": 0.026171039097056155, + "median_e2e_s": 8.848766844486818, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 51.270903732045554, + "output_tok_s": 9.986170766090625, + "median_first_sse_s": 26.803447068086825, + "median_tpot_est_s": 0.1057412954021035, + "median_e2e_s": 30.457469970046077, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.657406355952844, + "output_tok_s": 19.226706791834392, + "median_first_sse_s": 3.4630271260393783, + "median_tpot_est_s": 0.02515140170869454, + "median_e2e_s": 6.657255143043585, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 16.016934926970862, + "output_tok_s": 31.966165957123604, + "median_first_sse_s": 8.493684782530181, + "median_tpot_est_s": 0.02901831886632089, + "median_e2e_s": 12.179011278552935, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 53.753593918983825, + "output_tok_s": 38.09977809272992, + "median_first_sse_s": 27.78725230850978, + "median_tpot_est_s": 0.030791497555117703, + "median_e2e_s": 32.433394862979185, + "all_text_equal": true + } + ], + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "whole/on": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 850688.0, + "transferred_tokens": 84992.0 + }, + "raw_sha256": "06fae33330c3d3499a0fa80e56c9494117f077e64bb6e4220dded2f591cc9e72", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.66532118502073, + "output_tok_s": 8.730476371559513, + "median_first_sse_s": 2.918039128999226, + "median_tpot_est_s": 0.024100132226463285, + "median_e2e_s": 3.6651432280195877, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.30348469002638, + "output_tok_s": 10.403556652836828, + "median_first_sse_s": 7.266616121982224, + "median_tpot_est_s": 0.02600571070985508, + "median_e2e_s": 8.072793153987732, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.18980112497229, + "output_tok_s": 11.084698083343548, + "median_first_sse_s": 24.03905806154944, + "median_tpot_est_s": 0.026055202466405688, + "median_e2e_s": 24.958345417049713, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.242904701037332, + "output_tok_s": 20.50327630001004, + "median_first_sse_s": 2.869474889943376, + "median_tpot_est_s": 0.026561346441309343, + "median_e2e_s": 6.242765887989663, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 15.31299539806787, + "output_tok_s": 33.43565296601618, + "median_first_sse_s": 7.409371640475001, + "median_tpot_est_s": 0.026311591090671133, + "median_e2e_s": 10.784471123944968, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.35642505402211, + "output_tok_s": 41.494091149397505, + "median_first_sse_s": 24.396923166990746, + "median_tpot_est_s": 0.026859728673080674, + "median_e2e_s": 27.779408978007268, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.7718445679638535, + "output_tok_s": 8.483912691363761, + "median_first_sse_s": 2.9373642350547016, + "median_tpot_est_s": 0.026914485902976123, + "median_e2e_s": 3.7717132980469614, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.398822656017728, + "output_tok_s": 10.323560837276402, + "median_first_sse_s": 7.272774892975576, + "median_tpot_est_s": 0.027120182258752925, + "median_e2e_s": 8.096748902986292, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 45.94501227198634, + "output_tok_s": 11.143755865577978, + "median_first_sse_s": 24.065952128032222, + "median_tpot_est_s": 0.02582315859692772, + "median_e2e_s": 24.863611125038005, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.160433033946902, + "output_tok_s": 20.777760150083513, + "median_first_sse_s": 2.8113285440485924, + "median_tpot_est_s": 0.02636973762162411, + "median_e2e_s": 6.1602852219948545, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.768116903025657, + "output_tok_s": 34.66928135537054, + "median_first_sse_s": 7.088801535486709, + "median_tpot_est_s": 0.026913238858426447, + "median_e2e_s": 10.556075934495311, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.03478913893923, + "output_tok_s": 41.76626505310398, + "median_first_sse_s": 24.075821354053915, + "median_tpot_est_s": 0.026899172204883963, + "median_e2e_s": 27.450733256991953, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.945928778965026, + "output_tok_s": 8.109624322310564, + "median_first_sse_s": 3.141313388943672, + "median_tpot_est_s": 0.025950086614747923, + "median_e2e_s": 3.9457660740008578, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.172303085098974, + "output_tok_s": 10.515676376535051, + "median_first_sse_s": 7.091708452033345, + "median_tpot_est_s": 0.0255083825792216, + "median_e2e_s": 7.890375397459138, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.20221433893312, + "output_tok_s": 11.08171994190664, + "median_first_sse_s": 24.13236670056358, + "median_tpot_est_s": 0.025335372613382437, + "median_e2e_s": 24.96728817099938, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.266925232950598, + "output_tok_s": 20.42468918042843, + "median_first_sse_s": 2.8313267999328673, + "median_tpot_est_s": 0.027050363503975838, + "median_e2e_s": 6.266722964937799, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.83509479695931, + "output_tok_s": 34.512755530550606, + "median_first_sse_s": 7.151992013503332, + "median_tpot_est_s": 0.027155561228722744, + "median_e2e_s": 10.588076097541489, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.54142292402685, + "output_tok_s": 41.33914367257204, + "median_first_sse_s": 24.450606751954183, + "median_tpot_est_s": 0.02662571351590207, + "median_e2e_s": 27.830910447984934, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.643375781015493, + "output_tok_s": 8.78306326971325, + "median_first_sse_s": 2.857726918067783, + "median_tpot_est_s": 0.025337460514668737, + "median_e2e_s": 3.643188194022514, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 12.517737936927006, + "output_tok_s": 10.225489672730989, + "median_first_sse_s": 7.47981611196883, + "median_tpot_est_s": 0.026645917452931885, + "median_e2e_s": 8.298069035517983, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 46.68368743208703, + "output_tok_s": 10.96742841372226, + "median_first_sse_s": 25.86334280553274, + "median_tpot_est_s": 0.028045082806549486, + "median_e2e_s": 26.712535800528713, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 6.16653712594416, + "output_tok_s": 20.75719279487868, + "median_first_sse_s": 2.874457805068232, + "median_tpot_est_s": 0.025920808952353075, + "median_e2e_s": 6.166400542017072, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 14.728964269976132, + "output_tok_s": 34.76143947498555, + "median_first_sse_s": 7.113750016025733, + "median_tpot_est_s": 0.027268773952213034, + "median_e2e_s": 10.576884307956789, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 49.11081874195952, + "output_tok_s": 41.701605724813966, + "median_first_sse_s": 24.4042886030511, + "median_tpot_est_s": 0.027804161051625576, + "median_e2e_s": 28.926480908005033, + "all_text_equal": true + } + ], + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "stream/off": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 0.0, + "transferred_tokens": 935680.0 + }, + "raw_sha256": "3ddd7794442c6e6e5a7904871ca859c521738c8768433d35e8d20399420493a6", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.0344492259901017, + "output_tok_s": 10.545571079561832, + "median_first_sse_s": 2.222173067042604, + "median_tpot_est_s": 0.026198500128943595, + "median_e2e_s": 3.0343265710398555, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.291457520914264, + "output_tok_s": 13.776094838929534, + "median_first_sse_s": 5.330548713565804, + "median_tpot_est_s": 0.025170327402696376, + "median_e2e_s": 6.114778548071627, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.56748351501301, + "output_tok_s": 14.395170796491271, + "median_first_sse_s": 18.518849209474865, + "median_tpot_est_s": 0.026875704419874254, + "median_e2e_s": 19.44195634953212, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.373530109995045, + "output_tok_s": 23.820467621817798, + "median_first_sse_s": 2.137613946921192, + "median_tpot_est_s": 0.025478321228425686, + "median_e2e_s": 5.373360742931254, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 11.927924883086234, + "output_tok_s": 42.924482256424554, + "median_first_sse_s": 5.350217982544564, + "median_tpot_est_s": 0.027437909999880647, + "median_e2e_s": 8.846565436047968, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 37.57740541594103, + "output_tok_s": 54.50083573708366, + "median_first_sse_s": 18.221833322022576, + "median_tpot_est_s": 0.02667981534270482, + "median_e2e_s": 21.64534808701137, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.211695746984333, + "output_tok_s": 9.963583888681502, + "median_first_sse_s": 2.3731549710500985, + "median_tpot_est_s": 0.02704614116209409, + "median_e2e_s": 3.2115853470750153, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.43295849103015, + "output_tok_s": 13.5694437881515, + "median_first_sse_s": 5.399655768938828, + "median_tpot_est_s": 0.026163732065939374, + "median_e2e_s": 6.210731462982949, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.71247306501027, + "output_tok_s": 14.336727648851577, + "median_first_sse_s": 18.574257490516175, + "median_tpot_est_s": 0.02652731508151778, + "median_e2e_s": 19.40594940056326, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.338220724021085, + "output_tok_s": 23.97802687776131, + "median_first_sse_s": 2.165918902028352, + "median_tpot_est_s": 0.024977886991568027, + "median_e2e_s": 5.3381105499574915, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 11.92513715790119, + "output_tok_s": 42.934516661786674, + "median_first_sse_s": 5.504016306484118, + "median_tpot_est_s": 0.02519130685841518, + "median_e2e_s": 8.703312277502846, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 37.93958489596844, + "output_tok_s": 53.980558975953, + "median_first_sse_s": 18.479511058481876, + "median_tpot_est_s": 0.0253564650866599, + "median_e2e_s": 21.677008820464835, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.058836486074142, + "output_tok_s": 10.461494148407501, + "median_first_sse_s": 2.2814225980546325, + "median_tpot_est_s": 0.02507314690193462, + "median_e2e_s": 3.0586901520146057, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.86284664098639, + "output_tok_s": 12.97799759636115, + "median_first_sse_s": 5.68974005500786, + "median_tpot_est_s": 0.025835917564108968, + "median_e2e_s": 6.488205989997368, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 37.1058676840039, + "output_tok_s": 13.798356754792179, + "median_first_sse_s": 19.3962357249693, + "median_tpot_est_s": 0.029022938258300025, + "median_e2e_s": 20.305324282497168, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.826587219024077, + "output_tok_s": 21.968262927923583, + "median_first_sse_s": 2.3179203160107136, + "median_tpot_est_s": 0.027626068841878647, + "median_e2e_s": 5.826431058929302, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 12.11718301300425, + "output_tok_s": 42.254045304962204, + "median_first_sse_s": 5.6174396550050005, + "median_tpot_est_s": 0.026623807287422984, + "median_e2e_s": 9.04161945101805, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 39.340143382083625, + "output_tok_s": 52.058783317315125, + "median_first_sse_s": 19.44411149097141, + "median_tpot_est_s": 0.02653768402386794, + "median_e2e_s": 22.713961433502845, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.316174581996165, + "output_tok_s": 9.649672901339729, + "median_first_sse_s": 2.5564451449317858, + "median_tpot_est_s": 0.024503057582243797, + "median_e2e_s": 3.3160399299813434, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 11.041767364018597, + "output_tok_s": 11.592347110763166, + "median_first_sse_s": 6.5184954924625345, + "median_tpot_est_s": 0.028367914580514714, + "median_e2e_s": 7.396435436501633, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 41.80970816500485, + "output_tok_s": 12.245959669925398, + "median_first_sse_s": 23.20848668902181, + "median_tpot_est_s": 0.03088053917482255, + "median_e2e_s": 24.188921280030627, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.676418976043351, + "output_tok_s": 22.549427824163214, + "median_first_sse_s": 2.630975637002848, + "median_tpot_est_s": 0.02397870525211741, + "median_e2e_s": 5.676271204021759, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 13.63700374995824, + "output_tok_s": 37.54490424640148, + "median_first_sse_s": 6.511755163548514, + "median_tpot_est_s": 0.029209394330584157, + "median_e2e_s": 10.221348243532702, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 44.1992259939434, + "output_tok_s": 46.33565303339557, + "median_first_sse_s": 22.067861926509067, + "median_tpot_est_s": 0.03135937557869231, + "median_e2e_s": 26.46022372151492, + "all_text_equal": true + } + ], + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "stream/on": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 850688.0, + "transferred_tokens": 84992.0 + }, + "raw_sha256": "1c2c47cc3a9cdae054096f95ec26653cd9ac8594dfd9be13e3a98ade28c0822c", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.9660799669800326, + "output_tok_s": 10.788650459947434, + "median_first_sse_s": 2.138459959998727, + "median_tpot_est_s": 0.026691501002548444, + "median_e2e_s": 2.9658964910777286, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.432780474075116, + "output_tok_s": 13.569699872884023, + "median_first_sse_s": 5.423826438491233, + "median_tpot_est_s": 0.027401641080711758, + "median_e2e_s": 6.273733754991554, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 36.13144242495764, + "output_tok_s": 14.170483258823294, + "median_first_sse_s": 18.93030613096198, + "median_tpot_est_s": 0.027004231435937748, + "median_e2e_s": 19.77994070999557, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.6312565330881625, + "output_tok_s": 22.7302732965364, + "median_first_sse_s": 2.1788647409994155, + "median_tpot_est_s": 0.027182801669705513, + "median_e2e_s": 5.631080553052016, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 11.93878390698228, + "output_tok_s": 42.885439923287485, + "median_first_sse_s": 5.39331592497183, + "median_tpot_est_s": 0.026827251126139064, + "median_e2e_s": 8.800376817991491, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 38.001310175983235, + "output_tok_s": 53.892878706438196, + "median_first_sse_s": 18.469030371517874, + "median_tpot_est_s": 0.027872965275584978, + "median_e2e_s": 22.035761670034844, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.0288188609993085, + "output_tok_s": 10.565174567567944, + "median_first_sse_s": 2.203755221911706, + "median_tpot_est_s": 0.02660922609990643, + "median_e2e_s": 3.0286412310088053, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.58583946689032, + "output_tok_s": 13.353029793803092, + "median_first_sse_s": 5.536463976022787, + "median_tpot_est_s": 0.02809929470681856, + "median_e2e_s": 6.408132624987047, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.72137267398648, + "output_tok_s": 14.333155802068486, + "median_first_sse_s": 18.58372621802846, + "median_tpot_est_s": 0.027761704613259365, + "median_e2e_s": 19.43553564354079, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.674119274015538, + "output_tok_s": 22.55856703368437, + "median_first_sse_s": 2.39965068094898, + "median_tpot_est_s": 0.0257820155357898, + "median_e2e_s": 5.673966653994285, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 11.962467286037281, + "output_tok_s": 42.80053501986265, + "median_first_sse_s": 5.4564229375100695, + "median_tpot_est_s": 0.026840349098330173, + "median_e2e_s": 8.865147272998001, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 37.98896909598261, + "output_tok_s": 53.91038632360726, + "median_first_sse_s": 18.385367062990554, + "median_tpot_est_s": 0.027335515952995413, + "median_e2e_s": 21.945599335012957, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.027376718004234, + "output_tok_s": 10.570207470279966, + "median_first_sse_s": 2.1692160370294005, + "median_tpot_est_s": 0.027678460581228137, + "median_e2e_s": 3.0272483150474727, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.446687264018692, + "output_tok_s": 13.549723455706719, + "median_first_sse_s": 5.366796983988024, + "median_tpot_est_s": 0.027426751824321166, + "median_e2e_s": 6.208029180474114, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.83456855302211, + "output_tok_s": 14.287879571995585, + "median_first_sse_s": 18.475361039978452, + "median_tpot_est_s": 0.02735608807885118, + "median_e2e_s": 19.322635578981135, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.534334080060944, + "output_tok_s": 23.128347177514527, + "median_first_sse_s": 2.167296726955101, + "median_tpot_est_s": 0.02651036960644809, + "median_e2e_s": 5.534113666974008, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 12.09984776400961, + "output_tok_s": 42.31458196713171, + "median_first_sse_s": 5.475183577975258, + "median_tpot_est_s": 0.02788927299236351, + "median_e2e_s": 9.052137463004328, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 38.07431497902144, + "output_tok_s": 53.78954292752023, + "median_first_sse_s": 18.418084530567285, + "median_tpot_est_s": 0.02799524525574621, + "median_e2e_s": 21.927415508020204, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.001718234969303, + "output_tok_s": 10.66056088383234, + "median_first_sse_s": 2.1595299169421196, + "median_tpot_est_s": 0.027163157968090906, + "median_e2e_s": 3.0015878139529377, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 9.459372852928936, + "output_tok_s": 13.531552460200038, + "median_first_sse_s": 5.385697565565351, + "median_tpot_est_s": 0.02743237182280169, + "median_e2e_s": 6.236101092072204, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 35.61464168399107, + "output_tok_s": 14.376109818624013, + "median_first_sse_s": 19.533553374931216, + "median_tpot_est_s": 0.027933973081094486, + "median_e2e_s": 20.4053214424639, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.434419253957458, + "output_tok_s": 23.55357472774809, + "median_first_sse_s": 2.1673672610195354, + "median_tpot_est_s": 0.02572301870915951, + "median_e2e_s": 5.434190637082793, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 12.116605716058984, + "output_tok_s": 42.256058503365395, + "median_first_sse_s": 5.459057881555054, + "median_tpot_est_s": 0.02775848159011028, + "median_e2e_s": 8.98438504349906, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 38.63029872998595, + "output_tok_s": 53.01538086243903, + "median_first_sse_s": 18.56699193548411, + "median_tpot_est_s": 0.02884565194515057, + "median_e2e_s": 22.505755685968325, + "all_text_equal": true + } + ], + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + } + }, + "comparisons": { + "stream/off -> stream/on": { + "increased_conditions": 14, + "total_conditions": 24, + "min_throughput_change_pct": -5.919835903568171, + "max_throughput_change_pct": 17.394717981392738 + }, + "whole/on -> stream/on": { + "increased_conditions": 24, + "total_conditions": 24, + "min_throughput_change_pct": 8.570735588136525, + "max_throughput_change_pct": 32.331584044190585 + } + }, + "scope": { + "conditions": 96, + "responses": 1376, + "repeats_per_condition": 1 + }, + "limits": [ + "Single ordered runs, no statistical repeatability claim.", + "Same-reference parity is not native-engine equivalence or answer quality.", + "One A10080GB plus one RNGD research producer; four GPU slots, BF16, context9216,page/chunk256,width8192.", + "Cache counters count logical KV tokens; full producer prefill and CPU copy remain. No physical DMA overlap or PCIe-byte reduction proof." + ], + "all_four_reference_sets_equal": true, + "config_sha256": { + "whole/off": "a0dea0a54b95edcb9677ba473ae7cc831d3ee6cfa5de6f7a4175ccbda532fe12", + "whole/on": "2e26fe1f9af3b001621c3455d3ce668457b294d5ad9951478df500a62041929b", + "stream/off": "73c2047141df251249a8c9fdd60ef6ce1cfcf6838c507b923f50a9bbde6b8db9", + "stream/on": "74f4db840b0fa3c2ac6b1d32326365281055067beb185bb1f351f0b44b5336a2" + }, + "packages": { + "whole/off": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "whole/on": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "stream/off": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "stream/on": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + } + } +} diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md index 2ca0fdf2..c030875b 100644 --- a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md @@ -56,3 +56,20 @@ Both runs queried 935680 prefix tokens. Cache-on hit 850688 tokens, reducing log Streamed KV cache-off completed all 24 conditions and 344 responses with exact same-route outputs. All eight prompt/output references also equal the whole-KV cache-off control. Resource metadata is identical; the only added server argument is `--pd-prefill-stream-layers`. Both runs use the same backlog candidate; host listen drops/overflows increased zero. Exact raw/config/source fingerprints, package versions and all 24 paired rows are in ../evidence267/qwen4-off-route-comparison.json. Reproduce with the whole/off command above and add that server argument; keep both caches disabled and use distinct result files. Throughput increased in 24/24 conditions in this single ordered whole/stream pair, ranging from +4.33% to +30.38%. At 8192 input, 128 output, concurrency 4: 31.9662 to 37.5449 tok/s (+17.45%); median first SSE 8.49368 to 6.51176 s (-23.33%); median end-to-end 12.1790 to 10.2213 s. These compare transfer modes, not the backlog patch. One sample per condition does not establish repeatability; timing alone does not establish physical DMA overlap. Cache-on streaming, remaining GPU models and ATOM PD are still running. + +## Qwen3-4B four GPU PD configurations complete + +Whole/streamed KV crossed with cache off/on completed all four runs: 96 conditions, 1376 responses, zero same-route mismatches. All four prompt/reference sets are identical. Every run has zero host listen drop/overflow increment. The complete measurements, raw/config/source SHA256 fingerprints and package versions are in ../evidence267/qwen4-full-pd-cache-summary.json. Cache-on reproduction uses the command above with `--prefix-cache` in server arguments and `decoder_cache=true` metadata; pass the fresh same-route off result as `--reference`. Producer cache stays off. + +At 8192 input, 128 output, concurrency 4: + +| Configuration | Output tok/s | Median first SSE (s) | +|---|---:|---:| +| Whole, cache off | 31.9662 | 8.49368 | +| Whole, cache on | 34.7614 | 7.11375 | +| Stream, cache off | 37.5449 | 6.51176 | +| Stream, cache on | 42.2561 | 5.45906 | + +Stream cache-on versus off improves throughput in 14/24 conditions, with changes from -5.92% to +17.39%. Stream versus whole with cache-on improves in 24/24 conditions, from +8.57% to +32.33%. These are single ordered runs, not repeated stability or universal cache-gain evidence. Cache-on logically transfers 84992 of 935680 queried tokens; producer full-prefill/CPU-copy limitations remain. No native equivalence or physical overlap certification. + +The GPU queue advanced to Gemma whole/cache-off. ATOM Qwen3-4B whole/cache-on and later streamed KV remain in progress; the four-model goal remains incomplete. From 61cd8085148a765f8ccf950cba2d99a4331f487a Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 30 Oct 2026 15:38:48 +0900 Subject: [PATCH 6/8] docs: retain ATOM PD retry and native divergence evidence --- .../pd/http_matrix/.research/events.jsonl | 1 + opt_prime/examples/pd/http_matrix/STATUS.md | 2 + .../qwen4-atom-divergence-history.json | 27 + .../qwen4-atom-native-whole-summary.json | 2046 +++++++++++++++++ .../experiments/2026-10-30-http-backlog267.md | 8 + 5 files changed, 2084 insertions(+) create mode 100644 opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-divergence-history.json create mode 100644 opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-native-whole-summary.json diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index ef08cc2b..6c678fc1 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -34,3 +34,4 @@ {"ts": "2026-10-30T05:36:34.408243+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 RNGD-GPU whole KV paired cache off/on:48 conditions,688 responses exact. Across24 conditions throughput improves11,range -3.72% to +15.26%;single ordered pair. Full paired evidence retained;remaining hardware runs pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} {"ts": "2026-10-30T05:48:16.366880+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 GPU stream/off24conditions344responses exact;whole/stream references8/8equal. Same candidate,only stream flag differs. Single ordered pair all24 throughput increases(+4.33..30.38%);8K128c4 31.97->37.54tok/s. Remaining verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} {"ts": "2026-10-30T06:03:17.316059+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 GPU PD four configurations complete96conditions1376responses exact;all4reference sets equal,listen deltas0. Full grid provenance/timings retained;single ordered samples only. Gemma GPU and Qwen4 ATOM pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T06:38:39.539907+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 ATOM whole/on24conditions344responses same-route exact,listen deltas0. Native/PD6/8refs equal;2preEOS differences also present in historical cacheoff. At8K128c4 native23.07vsPD15.04tok/s. Streaming pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index aff2fd02..20688fa5 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -158,3 +158,5 @@ PR289 paired-cache follow-up: Qwen4 RNGD-GPU whole KV paired cache off/on:48 con PR289 streamed-control follow-up: Qwen4 GPU stream/off24conditions344responses exact;whole/stream references8/8equal. Same candidate,only stream flag differs. Single ordered pair all24 throughput increases(+4.33..30.38%);8K128c4 31.97->37.54tok/s. Remaining verification pending. PR289 Qwen4 GPU subset: Qwen4 GPU PD four configurations complete96conditions1376responses exact;all4reference sets equal,listen deltas0. Full grid provenance/timings retained;single ordered samples only. Gemma GPU and Qwen4 ATOM pending. + +PR289 ATOM PD follow-up: Qwen4 ATOM whole/on24conditions344responses same-route exact,listen deltas0. Native/PD6/8refs equal;2preEOS differences also present in historical cacheoff. At8K128c4 native23.07vsPD15.04tok/s. Streaming pending. diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-divergence-history.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-divergence-history.json new file mode 100644 index 00000000..954b1fd8 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-divergence-history.json @@ -0,0 +1,27 @@ +{ + "model": "Qwen3-4B", + "kind": "Historical-reference comparison, not new hardware experiment", + "sources": { + "mid201-atom-final-qwen4b-native.json": "a21a9292094540a8a66c7056a6aa32e4b4832c839b20acf3523e3182e3971261", + "edge201-atom-final-qwen4b-native.json": "03254958b234ac5a00d6f6839129f4fc4c8fdd3a7bd54174f191ea751ab7cda5", + "mid201-atom-final-qwen4b-whole.json": "55278b0eb5c9a03b1808cdd3a84db0150326904f54504a9c4894d2aeaf750732", + "edge201-atom-final-qwen4b-whole.json": "f82833eeea8da4de03276192aada64eb73af5e6294d6d5e507a0830a248da4f6" + }, + "conditions": { + "512:128": { + "first_divergent_generated_index": 50, + "same_native_tokens_off_and_on": true, + "same_whole_tokens_off_and_on": true, + "native_token_at_divergence": 4829, + "whole_token_at_divergence": 2299 + }, + "2048:128": { + "first_divergent_generated_index": 61, + "same_native_tokens_off_and_on": true, + "same_whole_tokens_off_and_on": true, + "native_token_at_divergence": 1366, + "whole_token_at_divergence": 5230 + } + }, + "conclusion": "The same native versus whole-PD token divergence is already present in archived cache-off references. Current cache-on outputs equal those same-route references. This does not identify the numerical root cause or prove all inputs are cache invariant." +} diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-native-whole-summary.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-native-whole-summary.json new file mode 100644 index 00000000..79d15ed9 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-native-whole-summary.json @@ -0,0 +1,2046 @@ +{ + "model": "Qwen3-4B", + "decoder_cache": true, + "arms": { + "native": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 867456.0 + }, + "raw_sha256": "f6b0ed35f371a37e6460adebb51eca0af569b1b2fb061f199896d63b0d8b91a5", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 1.21979605499655, + "output_tok_s": 26.233893665191847, + "median_first_sse_s": 0.06424786592833698, + "median_tpot_est_s": 0.03727080548302301, + "median_e2e_s": 1.2196428359020501, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 4.865430230041966, + "output_tok_s": 26.308053748187437, + "median_first_sse_s": 1.8862478689989075, + "median_tpot_est_s": 0.03721824728868782, + "median_e2e_s": 3.0406833300366998, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 19.443709495011717, + "output_tok_s": 26.332423868570633, + "median_first_sse_s": 9.170724456431344, + "median_tpot_est_s": 0.03721861124053717, + "median_e2e_s": 10.32500390149653, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.779329141136259, + "output_tok_s": 26.782001452523673, + "median_first_sse_s": 0.06421606685034931, + "median_tpot_est_s": 0.03712567288289213, + "median_e2e_s": 4.77917652297765, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.179921305039898, + "output_tok_s": 26.69458293686857, + "median_first_sse_s": 7.2513526930706576, + "median_tpot_est_s": 0.03726597728691702, + "median_e2e_s": 11.985176004120149, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 76.49607840902172, + "output_tok_s": 26.772614264608695, + "median_first_sse_s": 35.89145322609693, + "median_tpot_est_s": 0.0371851845594915, + "median_e2e_s": 40.606709040584974, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 1.409332315903157, + "output_tok_s": 22.705787441972554, + "median_first_sse_s": 0.248691565124318, + "median_tpot_est_s": 0.0374359464828646, + "median_e2e_s": 1.4092059060931206, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 4.901245624991134, + "output_tok_s": 26.115810100872377, + "median_first_sse_s": 1.9010033400263637, + "median_tpot_est_s": 0.03745433188704474, + "median_e2e_s": 3.062635939917527, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 19.62891486589797, + "output_tok_s": 26.08396865022408, + "median_first_sse_s": 9.260896492982283, + "median_tpot_est_s": 0.03751262429080182, + "median_e2e_s": 10.424593756091781, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.838200076017529, + "output_tok_s": 26.456119628967624, + "median_first_sse_s": 0.06642191391438246, + "median_tpot_est_s": 0.0375715390867077, + "median_e2e_s": 4.83800737792626, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.333293949021026, + "output_tok_s": 26.482812569346258, + "median_first_sse_s": 7.3223171149147674, + "median_tpot_est_s": 0.03758554547607136, + "median_e2e_s": 12.085917299962603, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 77.24892303696834, + "output_tok_s": 26.51169646753401, + "median_first_sse_s": 36.27493576845154, + "median_tpot_est_s": 0.03750518164927448, + "median_e2e_s": 41.03906974289566, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.0401992758270353, + "output_tok_s": 15.684742357840591, + "median_first_sse_s": 0.8407348680775613, + "median_tpot_est_s": 0.03868695670708774, + "median_e2e_s": 2.040030525997281, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 5.082977191079408, + "output_tok_s": 25.182092145650223, + "median_first_sse_s": 1.9748041100101545, + "median_tpot_est_s": 0.038678585807041775, + "median_e2e_s": 3.1738402700284496, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 20.373157310998067, + "output_tok_s": 25.13110718109492, + "median_first_sse_s": 9.618916884530336, + "median_tpot_est_s": 0.03872891262759484, + "median_e2e_s": 10.823435940546915, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.998283860972151, + "output_tok_s": 25.608789648674414, + "median_first_sse_s": 0.07471933006308973, + "median_tpot_est_s": 0.038767074181617595, + "median_e2e_s": 4.9981377511285245, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.97290690196678, + "output_tok_s": 25.63472620750974, + "median_first_sse_s": 7.558827344560996, + "median_tpot_est_s": 0.038741515790786975, + "median_e2e_s": 12.481202929164283, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 79.82538939104415, + "output_tok_s": 25.655997617091124, + "median_first_sse_s": 37.48764609743375, + "median_tpot_est_s": 0.0387145414293971, + "median_e2e_s": 42.39941652864218, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 5.352265110006556, + "output_tok_s": 5.978777086391523, + "median_first_sse_s": 4.025504268007353, + "median_tpot_est_s": 0.04279371299931119, + "median_e2e_s": 5.352109370986, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 5.7538805468939245, + "output_tok_s": 22.24585633240115, + "median_first_sse_s": 2.2709930365672335, + "median_tpot_est_s": 0.04280944161056992, + "median_e2e_s": 3.5957872320432216, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 23.044004359981045, + "output_tok_s": 22.218360663442486, + "median_first_sse_s": 10.923830434447154, + "median_tpot_est_s": 0.042838023904140196, + "median_e2e_s": 12.250851112417877, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.550273383036256, + "output_tok_s": 23.06192707393777, + "median_first_sse_s": 0.11678028688766062, + "median_tpot_est_s": 0.04278182463104448, + "median_e2e_s": 5.55007201503031, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 22.190961834043264, + "output_tok_s": 23.07245642748744, + "median_first_sse_s": 8.439156876062043, + "median_tpot_est_s": 0.0427968583077075, + "median_e2e_s": 13.874486317043193, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 88.77933540102094, + "output_tok_s": 23.068431304977402, + "median_first_sse_s": 41.72694445645902, + "median_tpot_est_s": 0.04281781166131923, + "median_e2e_s": 47.15564054495189, + "all_text_equal": true + } + ], + "config_sha256": "fc6780d557b89f162940b8725bf11c65b306cb76c0c117ccb45fcc619d945d46", + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "whole": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 867456.0 + }, + "raw_sha256": "bd0bc2e81c131d9c179268afd97066316bb70bc6904a3002c670f8cf1eef79fa", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.229157143039629, + "output_tok_s": 7.566519502039735, + "median_first_sse_s": 3.07451313897036, + "median_tpot_est_s": 0.03723970416092103, + "median_e2e_s": 4.228943967958912, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 16.919324713060632, + "output_tok_s": 7.565313756357677, + "median_first_sse_s": 9.443536977516487, + "median_tpot_est_s": 0.0372378321612374, + "median_e2e_s": 10.594949356513098, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 67.71203166921623, + "output_tok_s": 7.561433136450541, + "median_first_sse_s": 34.59739960345905, + "median_tpot_est_s": 0.03715300685425679, + "median_e2e_s": 35.75067046692129, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 7.789689174853265, + "output_tok_s": 16.431977852622232, + "median_first_sse_s": 3.0535819258075207, + "median_tpot_est_s": 0.0372907919919221, + "median_e2e_s": 7.789512508781627, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 31.15392838884145, + "output_tok_s": 16.434524519976282, + "median_first_sse_s": 14.739977189921774, + "median_tpot_est_s": 0.03724773100019968, + "median_e2e_s": 19.466690970933996, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 124.65422019199468, + "output_tok_s": 16.429447770365364, + "median_first_sse_s": 61.595142140984535, + "median_tpot_est_s": 0.03722072855521756, + "median_e2e_s": 66.31608892697841, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.327635735040531, + "output_tok_s": 7.39433768440779, + "median_first_sse_s": 3.1801685569807887, + "median_tpot_est_s": 0.03700972913253692, + "median_e2e_s": 4.327470160089433, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 17.268865237943828, + "output_tok_s": 7.412183616949733, + "median_first_sse_s": 9.761258519953117, + "median_tpot_est_s": 0.037525404207107996, + "median_e2e_s": 10.924995203502476, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 67.5510564618744, + "output_tok_s": 7.579452148005578, + "median_first_sse_s": 34.785325877484865, + "median_tpot_est_s": 0.03732400219286641, + "median_e2e_s": 35.948298823903315, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 8.1633458530996, + "output_tok_s": 15.679845286892844, + "median_first_sse_s": 3.39684719289653, + "median_tpot_est_s": 0.03752971129257261, + "median_e2e_s": 8.163120527053252, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 31.40743782185018, + "output_tok_s": 16.30187100597557, + "median_first_sse_s": 14.859845734550618, + "median_tpot_est_s": 0.03752742784178456, + "median_e2e_s": 19.622518819523975, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 125.70304442220367, + "output_tok_s": 16.29236594398862, + "median_first_sse_s": 61.88206681644078, + "median_tpot_est_s": 0.03753838226323815, + "median_e2e_s": 66.65103639895096, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.603710082126781, + "output_tok_s": 6.95091555053287, + "median_first_sse_s": 3.4106448700185865, + "median_tpot_est_s": 0.03847976864105271, + "median_e2e_s": 4.6035176978912205, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 17.03182191797532, + "output_tok_s": 7.5153439612299655, + "median_first_sse_s": 9.448679500841536, + "median_tpot_est_s": 0.03850972472930387, + "median_e2e_s": 10.642480967449956, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 68.56243541510776, + "output_tok_s": 7.467646049912349, + "median_first_sse_s": 35.08746871794574, + "median_tpot_est_s": 0.03867314664662004, + "median_e2e_s": 36.28633626399096, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 8.001783073879778, + "output_tok_s": 15.996434646901442, + "median_first_sse_s": 3.0788507510442287, + "median_tpot_est_s": 0.03876198732338142, + "median_e2e_s": 8.001623141113669, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 32.082207593135536, + "output_tok_s": 15.959001527985562, + "median_first_sse_s": 15.150795516441576, + "median_tpot_est_s": 0.03876264726809101, + "median_e2e_s": 20.078690478461795, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 128.25819011521526, + "output_tok_s": 15.967791204290867, + "median_first_sse_s": 63.30086616508197, + "median_tpot_est_s": 0.038750230264116635, + "median_e2e_s": 68.22345729661174, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 5.651906156912446, + "output_tok_s": 5.661806673994944, + "median_first_sse_s": 4.324762323871255, + "median_tpot_est_s": 0.04280508706916965, + "median_e2e_s": 5.651720023015514, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 17.66353549505584, + "output_tok_s": 7.246567372416932, + "median_first_sse_s": 9.705989120993763, + "median_tpot_est_s": 0.04286090721331176, + "median_e2e_s": 11.035693836514838, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 70.67930217203684, + "output_tok_s": 7.24398776255271, + "median_first_sse_s": 36.408837280469015, + "median_tpot_est_s": 0.042833625098630305, + "median_e2e_s": 37.739714586525224, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 8.460688102990389, + "output_tok_s": 15.128793124374722, + "median_first_sse_s": 3.0657620751298964, + "median_tpot_est_s": 0.04247847278639087, + "median_e2e_s": 8.460528119001538, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 34.039199938997626, + "output_tok_s": 15.041481612892374, + "median_first_sse_s": 15.812518983031623, + "median_tpot_est_s": 0.04263964579825649, + "median_e2e_s": 21.22309566254262, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 137.80537095014006, + "output_tok_s": 14.86153976350454, + "median_first_sse_s": 67.86479224008508, + "median_tpot_est_s": 0.042862537114396984, + "median_e2e_s": 73.3110099316109, + "all_text_equal": true + } + ], + "config_sha256": "12bbf46aa3cfdcf821075371abd139f61397757f90a513617cdc9fac282dcbf5", + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + } + }, + "reference_comparison": { + "512:32": { + "tokens_equal": true, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 1083, + 3897, + 264 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 1083, + 3897, + 264 + ], + "first_different_generated_index": null + }, + "512:128": { + "tokens_equal": false, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 1083, + 3897, + 264, + 15493, + 315, + 11504, + 4004, + 2266, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2938, + 4977, + 45205, + 13, + 10696, + 807, + 4829, + 752, + 311, + 990, + 429, + 2266, + 369, + 279, + 3405, + 30, + 1988, + 279, + 3405, + 572, + 30339, + 13, + 14568, + 358, + 3116, + 2494, + 1939, + 10061, + 752, + 1779, + 1549, + 13, + 576, + 1196, + 594, + 3405, + 572, + 330, + 3838, + 374, + 279, + 6722, + 315, + 9625, + 7521, + 323, + 279, + 4226, + 374, + 12095, + 13, + 576, + 4004, + 2266, + 911, + 14697, + 374, + 4658, + 1101, + 4960, + 1995, + 11, + 7196, + 264, + 1273, + 311, + 1490, + 421, + 358, + 646, + 5244, + 389, + 279, + 5042, + 3405, + 13, + 2521, + 7196, + 432, + 594, + 949, + 315, + 264, + 8131 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 1083, + 3897, + 264, + 15493, + 315, + 11504, + 4004, + 2266, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2938, + 4977, + 45205, + 13, + 10696, + 807, + 2299, + 7497, + 421, + 358, + 646, + 10034, + 39715, + 1995, + 30, + 2521, + 7196, + 1052, + 594, + 264, + 14068, + 1588, + 13, + 6771, + 752, + 1744, + 13, + 576, + 4004, + 2266, + 374, + 1101, + 11504, + 5128, + 911, + 14697, + 11, + 714, + 279, + 3405, + 374, + 911, + 279, + 6722, + 315, + 9625, + 13, + 8704, + 279, + 3405, + 374, + 30339, + 11, + 279, + 4226, + 1265, + 387, + 12095, + 15484, + 315, + 279, + 2266, + 13, + 576, + 14697, + 949, + 374, + 4658, + 1101, + 1052, + 311, + 1490, + 421, + 358, + 633, + 48704, + 13, + 358, + 1265, + 1281, + 2704, + 537, + 311, + 1077 + ], + "first_different_generated_index": 50, + "eos_ids": [ + 151645, + 151643 + ], + "native_first_eos_index": null, + "whole_first_eos_index": null, + "difference_before_both_eos": true + }, + "2048:32": { + "tokens_equal": true, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493 + ], + "first_different_generated_index": null + }, + "2048:128": { + "tokens_equal": false, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493, + 315, + 11504, + 4004, + 2266, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2938, + 4977, + 45205, + 13, + 10696, + 807, + 2299, + 7497, + 421, + 358, + 646, + 10034, + 39715, + 1995, + 30, + 2521, + 8365, + 807, + 1366, + 311, + 1490, + 421, + 358, + 646, + 5244, + 389, + 279, + 5042, + 3405, + 8818, + 279, + 76449, + 13, + 6771, + 752, + 1281, + 2704, + 358, + 3207, + 944, + 3116, + 4113, + 13, + 576, + 4004, + 2266, + 374, + 1101, + 911, + 14697, + 11, + 714, + 279, + 3405, + 374, + 30339, + 13, + 358, + 1265, + 7683, + 429, + 847, + 4226, + 374, + 4396, + 13, + 12095, + 374, + 12824, + 279, + 6722, + 315, + 9625, + 13, + 576, + 14697, + 949, + 374, + 4658, + 1101, + 1052, + 311, + 1490, + 421, + 358 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 369, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 18577, + 12095, + 13, + 1988, + 3783, + 11, + 279, + 1196, + 3897, + 264, + 15493, + 315, + 11504, + 4004, + 2266, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2938, + 4977, + 45205, + 13, + 10696, + 807, + 2299, + 7497, + 421, + 358, + 646, + 10034, + 39715, + 1995, + 30, + 2521, + 8365, + 807, + 5230, + 429, + 438, + 949, + 315, + 264, + 15048, + 1380, + 358, + 1184, + 311, + 5244, + 389, + 3151, + 3565, + 13, + 1988, + 279, + 3405, + 374, + 30339, + 13, + 576, + 4226, + 374, + 12095, + 11, + 15484, + 315, + 279, + 2266, + 13, + 358, + 1265, + 1281, + 2704, + 537, + 311, + 387, + 21815, + 553, + 279, + 11504, + 1995, + 13, + 4599, + 4226, + 279, + 3405, + 3118, + 389, + 4586, + 6540, + 13, + 84194, + 11, + 12095, + 374, + 279, + 6722, + 315, + 9625, + 13, + 2308, + 1184, + 311, + 916 + ], + "first_different_generated_index": 61, + "eos_ids": [ + 151645, + 151643 + ], + "native_first_eos_index": null, + "whole_first_eos_index": null, + "difference_before_both_eos": true + }, + "128:32": { + "tokens_equal": true, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "first_different_generated_index": null + }, + "128:128": { + "tokens_equal": true, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518, + 279, + 2266, + 2661, + 11, + 432, + 594, + 678, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2619, + 594, + 902, + 6286, + 315, + 9625, + 11, + 92999, + 11, + 476, + 894, + 5435, + 1995, + 13, + 4710, + 80022, + 11, + 7196, + 279, + 1196, + 374, + 7497, + 421, + 358, + 646, + 10034, + 279, + 39715, + 2266, + 323, + 4226, + 279, + 5042, + 3405, + 13, + 2521, + 8365, + 807, + 5230, + 279, + 2266, + 553, + 16523, + 13, + 20988, + 1616, + 11, + 279, + 4226, + 1265, + 387, + 12095, + 13, + 576, + 2266, + 911, + 14697, + 3171, + 944, + 7802, + 279, + 3405, + 13, + 358, + 1184, + 311, + 1281, + 2704, + 358, + 1513, + 944, + 633, + 21815, + 553, + 279, + 14590, + 1995, + 323, + 1101, + 5244, + 389, + 279 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518, + 279, + 2266, + 2661, + 11, + 432, + 594, + 678, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2619, + 594, + 902, + 6286, + 315, + 9625, + 11, + 92999, + 11, + 476, + 894, + 5435, + 1995, + 13, + 4710, + 80022, + 11, + 7196, + 279, + 1196, + 374, + 7497, + 421, + 358, + 646, + 10034, + 279, + 39715, + 2266, + 323, + 4226, + 279, + 5042, + 3405, + 13, + 2521, + 8365, + 807, + 5230, + 279, + 2266, + 553, + 16523, + 13, + 20988, + 1616, + 11, + 279, + 4226, + 1265, + 387, + 12095, + 13, + 576, + 2266, + 911, + 14697, + 3171, + 944, + 7802, + 279, + 3405, + 13, + 358, + 1184, + 311, + 1281, + 2704, + 358, + 1513, + 944, + 633, + 21815, + 553, + 279, + 14590, + 1995, + 323, + 1101, + 5244, + 389, + 279 + ], + "first_different_generated_index": null + }, + "8192:32": { + "tokens_equal": true, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518 + ], + "first_different_generated_index": null + }, + "8192:128": { + "tokens_equal": true, + "native_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518, + 279, + 2266, + 2661, + 11, + 432, + 594, + 678, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2619, + 594, + 902, + 6286, + 315, + 9625, + 11, + 92999, + 11, + 476, + 894, + 5435, + 1995, + 13, + 4710, + 80022, + 11, + 279, + 1196, + 2578, + 614, + 3381, + 429, + 279, + 3681, + 2266, + 572, + 9760, + 311, + 862, + 3405, + 11, + 714, + 432, + 594, + 3520, + 45205, + 13, + 10696, + 807, + 1033, + 7497, + 421, + 358, + 646, + 15282, + 429, + 279, + 2266, + 3171, + 944, + 3410, + 894, + 1995, + 911, + 279, + 6722, + 315, + 9625, + 13, + 2521, + 8365, + 807, + 1101, + 4829, + 311, + 1490, + 421, + 358, + 1410, + 4226, + 264, + 30339, + 3405, + 8818, + 279, + 39715, + 2266, + 382, + 40, + 1265, + 1281 + ], + "whole_tokens": [ + 576, + 6722, + 315, + 9625, + 374, + 12095, + 624, + 32313, + 11, + 773, + 279, + 1196, + 4588, + 264, + 3405, + 911, + 279, + 6722, + 315, + 9625, + 11, + 323, + 358, + 3897, + 279, + 4226, + 12095, + 13, + 1988, + 3330, + 1182, + 518, + 279, + 2266, + 2661, + 11, + 432, + 594, + 678, + 911, + 14697, + 8482, + 40676, + 323, + 6467, + 13, + 2619, + 594, + 902, + 6286, + 315, + 9625, + 11, + 92999, + 11, + 476, + 894, + 5435, + 1995, + 13, + 4710, + 80022, + 11, + 279, + 1196, + 2578, + 614, + 3381, + 429, + 279, + 3681, + 2266, + 572, + 9760, + 311, + 862, + 3405, + 11, + 714, + 432, + 594, + 3520, + 45205, + 13, + 10696, + 807, + 1033, + 7497, + 421, + 358, + 646, + 15282, + 429, + 279, + 2266, + 3171, + 944, + 3410, + 894, + 1995, + 911, + 279, + 6722, + 315, + 9625, + 13, + 2521, + 8365, + 807, + 1101, + 4829, + 311, + 1490, + 421, + 358, + 1410, + 4226, + 264, + 30339, + 3405, + 8818, + 279, + 39715, + 2266, + 382, + 40, + 1265, + 1281 + ], + "first_different_generated_index": null + } + }, + "limits": [ + "Each arm uses same-route historical off references; not a fresh cache off/on timing comparison.", + "One ATOM execution lane; concurrency is queued clients, not hardware batch size.", + "PD adds one RNGD research producer to the native single-ATOM configuration; not an equal-resource speedup test.", + "Single ordered runs; no repeatability claim. Same-route correctness does not imply native-PD equivalence." + ], + "packages": { + "native": { + "torch": "2.11.0+cpu", + "transformers": "5.8.1" + }, + "whole": { + "torch": "2.11.0+cpu", + "transformers": "5.8.1" + } + } +} diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md index c030875b..05108044 100644 --- a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md @@ -73,3 +73,11 @@ At 8192 input, 128 output, concurrency 4: Stream cache-on versus off improves throughput in 14/24 conditions, with changes from -5.92% to +17.39%. Stream versus whole with cache-on improves in 24/24 conditions, from +8.57% to +32.33%. These are single ordered runs, not repeated stability or universal cache-gain evidence. Cache-on logically transfers 84992 of 935680 queried tokens; producer full-prefill/CPU-copy limitations remain. No native equivalence or physical overlap certification. The GPU queue advanced to Gemma whole/cache-off. ATOM Qwen3-4B whole/cache-on and later streamed KV remain in progress; the four-model goal remains incomplete. + +## Qwen3-4B RNGD-to-ATOM whole-KV retry + +The whole-KV cache-on retry completed 24 conditions/344 responses, matching retained same-route cache-off references. Host listen drop/overflow increments were zero. Prefix hits were 867456 of 935680 queried tokens. The native and whole-PD results, all 48 timing rows and fingerprints are in ../evidence267/qwen4-atom-native-whole-summary.json. One ATOM execution lane, B1/k3/context9216, with one additional RNGD for PD; concurrent clients are queued, not hardware batch size. + +At 8192 input, 128 output, concurrency 4, native versus whole PD delivered 23.0725 versus 15.0415 tok/s, with median first SSE 8.43916 versus 15.8125 s. This single ordered comparison does not show a PD advantage; it is not a fresh cache off/on speedup measurement. + +Native/PD reference outputs match 6/8 cases. At 512/128 and 2048/128, they diverge at generated-token indices 50 and 61, respectively, before EOS. The same divergent arrays already occur in the archived cache-off runs; all four historical raw hashes and same-route reference arrays were verified. See ../evidence267/qwen4-atom-divergence-history.json. These cases are not new cache-on regressions, but their numerical root cause remains unresolved. Do not label cross-engine parity as passing. Streamed ATOM KV is still being measured. From 0af0af584f86b1d963e942c2c73fa33c87053c1e Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 30 Oct 2026 15:45:59 +0900 Subject: [PATCH 7/8] docs: complete Gemma GPU PD cache evidence --- .../pd/http_matrix/.research/events.jsonl | 1 + opt_prime/examples/pd/http_matrix/STATUS.md | 2 + .../gemma-full-pd-cache-summary.json | 1388 +++++++++++++++++ .../experiments/2026-10-30-http-backlog267.md | 15 + 4 files changed, 1406 insertions(+) create mode 100644 opt_prime/examples/pd/http_matrix/evidence267/gemma-full-pd-cache-summary.json diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index 6c678fc1..77c0e413 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -35,3 +35,4 @@ {"ts": "2026-10-30T05:48:16.366880+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 GPU stream/off24conditions344responses exact;whole/stream references8/8equal. Same candidate,only stream flag differs. Single ordered pair all24 throughput increases(+4.33..30.38%);8K128c4 31.97->37.54tok/s. Remaining verification pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} {"ts": "2026-10-30T06:03:17.316059+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 GPU PD four configurations complete96conditions1376responses exact;all4reference sets equal,listen deltas0. Full grid provenance/timings retained;single ordered samples only. Gemma GPU and Qwen4 ATOM pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} {"ts": "2026-10-30T06:38:39.539907+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 ATOM whole/on24conditions344responses same-route exact,listen deltas0. Native/PD6/8refs equal;2preEOS differences also present in historical cacheoff. At8K128c4 native23.07vsPD15.04tok/s. Streaming pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T06:45:47.350695+00:00", "task": "aicomp #267 / #201", "result": "Gemma GPU PD four configurations96conditions1376responses exact,all4reference sets equal,listen deltas0. Full timing/provenance retained;streamcache gain10/24 only(-4.93..+2.58%),single ordered samples. Remaining model tests pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index 20688fa5..ded34680 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -160,3 +160,5 @@ PR289 streamed-control follow-up: Qwen4 GPU stream/off24conditions344responses e PR289 Qwen4 GPU subset: Qwen4 GPU PD four configurations complete96conditions1376responses exact;all4reference sets equal,listen deltas0. Full grid provenance/timings retained;single ordered samples only. Gemma GPU and Qwen4 ATOM pending. PR289 ATOM PD follow-up: Qwen4 ATOM whole/on24conditions344responses same-route exact,listen deltas0. Native/PD6/8refs equal;2preEOS differences also present in historical cacheoff. At8K128c4 native23.07vsPD15.04tok/s. Streaming pending. + +PR289 Gemma GPU subset: Gemma GPU PD four configurations96conditions1376responses exact,all4reference sets equal,listen deltas0. Full timing/provenance retained;streamcache gain10/24 only(-4.93..+2.58%),single ordered samples. Remaining model tests pending. diff --git a/opt_prime/examples/pd/http_matrix/evidence267/gemma-full-pd-cache-summary.json b/opt_prime/examples/pd/http_matrix/evidence267/gemma-full-pd-cache-summary.json new file mode 100644 index 00000000..f427c1a5 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/gemma-full-pd-cache-summary.json @@ -0,0 +1,1388 @@ +{ + "model": "unsloth/gemma-3-1b-it", + "runtime_base": "f52c41305f6bb2ee65d3c0fe8166acbca51dcc72", + "source_sha256": { + "benchmark201_native_http.py": "e1ea87a51343d1af83c40aaa9c41ae2a20e9864aa8b53bd475d047d175c41433", + "llmd.py": "9f3aaf0902f7b1958f32be12b1f276384f20cc3e0070571af10bb7bce148c1f0", + "serve_batched.py": "cf509e2c392a39aab7b47d7296447ff62cc8237e193e42d7d9cd76fdc47d9c30", + "streaming_factories.py": "77745350973cc54bf508e0c8b473efc442afef20b1c64f7c4edc179ea1471389", + "stream265-python.sh": "1a417cc6dde8ec6837362153406e618b006169513851131f8d324ff46aa96d9a", + "fsdk.py": "2cf093e93fbc2a98c1ce242563e65d7feaaabff468a91654392412c74df98bb6" + }, + "arms": { + "whole/off": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 0.0, + "transferred_tokens": 935680.0 + }, + "raw_sha256": "5f7c0812f6492494248c64ce13100ab298baafe3754364c6d60f3ca781ca09b7", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.3279264480806887, + "output_tok_s": 13.746138769282474, + "median_first_sse_s": 1.5195072460919619, + "median_tpot_est_s": 0.026071898319998815, + "median_e2e_s": 2.327736094011925, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.743163365055807, + "output_tok_s": 18.98218878446832, + "median_first_sse_s": 3.7519505819655024, + "median_tpot_est_s": 0.024586322999018577, + "median_e2e_s": 4.534886106965132, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.738074910943396, + "output_tok_s": 20.696840875581078, + "median_first_sse_s": 12.740061033982784, + "median_tpot_est_s": 0.025022625419942122, + "median_e2e_s": 13.519588774011936, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.590997903025709, + "output_tok_s": 27.88064876170849, + "median_first_sse_s": 1.5005308070685714, + "median_tpot_est_s": 0.02433352762976236, + "median_e2e_s": 4.590888816048391, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.130475593032315, + "output_tok_s": 56.07593983282968, + "median_first_sse_s": 3.7736012175446376, + "median_tpot_est_s": 0.02480497963746687, + "median_e2e_s": 6.946635517524555, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.1226946109673, + "output_tok_s": 75.508721731943, + "median_first_sse_s": 12.762706937966868, + "median_tpot_est_s": 0.025122510413199253, + "median_e2e_s": 16.032839732943103, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.3080955729819834, + "output_tok_s": 13.8642439137202, + "median_first_sse_s": 1.5271887209964916, + "median_tpot_est_s": 0.025187086708272896, + "median_e2e_s": 2.3079884089529514, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.8409073810325935, + "output_tok_s": 18.71096813193211, + "median_first_sse_s": 3.8035742895444855, + "median_tpot_est_s": 0.025399381223495207, + "median_e2e_s": 4.5866546424804255, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.83240737800952, + "output_tok_s": 20.618218451643337, + "median_first_sse_s": 12.765676048526075, + "median_tpot_est_s": 0.024825153613252743, + "median_e2e_s": 13.52388443046948, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.821606518002227, + "output_tok_s": 26.547168360191122, + "median_first_sse_s": 1.5025401849998161, + "median_tpot_est_s": 0.026132927086446288, + "median_e2e_s": 4.821421924978495, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.118290200014599, + "output_tok_s": 56.15087793533707, + "median_first_sse_s": 3.751231240981724, + "median_tpot_est_s": 0.024738553586492944, + "median_e2e_s": 6.915239866473712, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.99413980008103, + "output_tok_s": 75.8683186486962, + "median_first_sse_s": 12.75357724505011, + "median_tpot_est_s": 0.024844022570856238, + "median_e2e_s": 15.9701726125204, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.2997427120571956, + "output_tok_s": 13.914600025571968, + "median_first_sse_s": 1.5408305689925328, + "median_tpot_est_s": 0.024475289095613743, + "median_e2e_s": 2.299564530956559, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.828186889062636, + "output_tok_s": 18.7458255140951, + "median_first_sse_s": 3.8055082780192606, + "median_tpot_est_s": 0.02437940348247667, + "median_e2e_s": 4.566361733479425, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 25.028537103906274, + "output_tok_s": 20.456649059208928, + "median_first_sse_s": 12.85288890299853, + "median_tpot_est_s": 0.023986310467139005, + "median_e2e_s": 13.629948533489369, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.6510499940486625, + "output_tok_s": 27.520667411398453, + "median_first_sse_s": 1.5235844729468226, + "median_tpot_est_s": 0.024624736181196323, + "median_e2e_s": 4.650925967958756, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.157141407951713, + "output_tok_s": 55.91264535408383, + "median_first_sse_s": 3.9284273515222594, + "median_tpot_est_s": 0.023716124732094987, + "median_e2e_s": 6.974470154498704, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.226294361054897, + "output_tok_s": 75.22140078414436, + "median_first_sse_s": 12.845481677446514, + "median_tpot_est_s": 0.02481556302404002, + "median_e2e_s": 16.034130166459363, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.3383831130340695, + "output_tok_s": 13.684669471667439, + "median_first_sse_s": 1.5876917759887874, + "median_tpot_est_s": 0.024212286161679412, + "median_e2e_s": 2.3382726470008492, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.9451680039055645, + "output_tok_s": 18.430079722768426, + "median_first_sse_s": 3.9202036120113917, + "median_tpot_est_s": 0.02514317951675865, + "median_e2e_s": 4.689277389028575, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 25.149196759914048, + "output_tok_s": 20.35850309207847, + "median_first_sse_s": 13.821421672473662, + "median_tpot_est_s": 0.02708717667865717, + "median_e2e_s": 14.623511960497126, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.719465148984455, + "output_tok_s": 27.12171738942565, + "median_first_sse_s": 1.5936326179653406, + "median_tpot_est_s": 0.02461201841777968, + "median_e2e_s": 4.71935895702336, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.28263967903331, + "output_tok_s": 55.15672456364476, + "median_first_sse_s": 3.9311249460442923, + "median_tpot_est_s": 0.0258020139248917, + "median_e2e_s": 7.207980714505538, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.833847583038732, + "output_tok_s": 73.5794788661558, + "median_first_sse_s": 13.436637023987714, + "median_tpot_est_s": 0.025422491287211264, + "median_e2e_s": 16.623036271485034, + "all_text_equal": true + } + ], + "config_sha256": "429f861781991f63e9d31c672c1d612aff45e15e49058377ddd15d2c34029330", + "packages": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "whole/on": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 850688.0, + "transferred_tokens": 84992.0 + }, + "raw_sha256": "44592b0e9d312d2dd5fde84db5ac1f63e51d1ce84980932af6c95b66cfe035d3", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.294777803006582, + "output_tok_s": 13.944705216371755, + "median_first_sse_s": 1.5151921110227704, + "median_tpot_est_s": 0.025143629416913515, + "median_e2e_s": 2.2946446229470894, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.7410926590673625, + "output_tok_s": 18.98801966886314, + "median_first_sse_s": 3.753810322494246, + "median_tpot_est_s": 0.02379378430815714, + "median_e2e_s": 4.495097044564318, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.815822748001665, + "output_tok_s": 20.63199778622007, + "median_first_sse_s": 12.720985402003862, + "median_tpot_est_s": 0.02436743817675198, + "median_e2e_s": 13.510060362983495, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.499732009018771, + "output_tok_s": 28.44613851301606, + "median_first_sse_s": 1.4961800730088726, + "median_tpot_est_s": 0.023648921007979808, + "median_e2e_s": 4.499593041022308, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.146101497928612, + "output_tok_s": 55.980135374176264, + "median_first_sse_s": 3.760683643457014, + "median_tpot_est_s": 0.02449713394505991, + "median_e2e_s": 6.871819654479623, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.23556305700913, + "output_tok_s": 75.19580174322641, + "median_first_sse_s": 12.845750043052249, + "median_tpot_est_s": 0.025506295956300908, + "median_e2e_s": 16.08926992106717, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.284619624959305, + "output_tok_s": 14.006708009684546, + "median_first_sse_s": 1.5479519880609587, + "median_tpot_est_s": 0.023759875837112625, + "median_e2e_s": 2.28450813901145, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.722448652959429, + "output_tok_s": 19.040680949441015, + "median_first_sse_s": 3.7428368200198747, + "median_tpot_est_s": 0.02400586399955735, + "median_e2e_s": 4.482454030017834, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.702370829996653, + "output_tok_s": 20.726755481229628, + "median_first_sse_s": 12.781110248004552, + "median_tpot_est_s": 0.024122127290484643, + "median_e2e_s": 13.536579754494596, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.564453889033757, + "output_tok_s": 28.042785207563163, + "median_first_sse_s": 1.498603621032089, + "median_tpot_est_s": 0.02413911622028622, + "median_e2e_s": 4.564271381008439, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.206287367036566, + "output_tok_s": 55.61416666540672, + "median_first_sse_s": 3.8192548800143413, + "median_tpot_est_s": 0.02545566732654216, + "median_e2e_s": 7.061408449488226, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.19484285300132, + "output_tok_s": 75.30839619372816, + "median_first_sse_s": 12.779843088472262, + "median_tpot_est_s": 0.024819888452738963, + "median_e2e_s": 15.918898725474719, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.294593045953661, + "output_tok_s": 13.945828022284624, + "median_first_sse_s": 1.5489279510220513, + "median_tpot_est_s": 0.024049640159993883, + "median_e2e_s": 2.2944667959818617, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.811993209994398, + "output_tok_s": 18.79038866512688, + "median_first_sse_s": 3.810922817501705, + "median_tpot_est_s": 0.023920489257321723, + "median_e2e_s": 4.5524579844786786, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.577400514041074, + "output_tok_s": 20.832146170523377, + "median_first_sse_s": 12.677245650964323, + "median_tpot_est_s": 0.023498049822650968, + "median_e2e_s": 13.405580043443479, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.6355026679812, + "output_tok_s": 27.61297084006319, + "median_first_sse_s": 1.5174282259540632, + "median_tpot_est_s": 0.02455074081149715, + "median_e2e_s": 4.635372309014201, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.128292449051514, + "output_tok_s": 56.08935108703709, + "median_first_sse_s": 3.852934113529045, + "median_tpot_est_s": 0.02434130883445655, + "median_e2e_s": 6.967866144492291, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.278488956973888, + "output_tok_s": 75.07747233471369, + "median_first_sse_s": 12.975559875019826, + "median_tpot_est_s": 0.02475071167316669, + "median_e2e_s": 16.162191890529357, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.2262181820115075, + "output_tok_s": 14.374152658786699, + "median_first_sse_s": 1.4948794160736725, + "median_tpot_est_s": 0.023585418806291156, + "median_e2e_s": 2.2260273990686983, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.799865473993123, + "output_tok_s": 18.823901809462395, + "median_first_sse_s": 3.758145831001457, + "median_tpot_est_s": 0.02464070370901496, + "median_e2e_s": 4.521373027469963, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.871456557069905, + "output_tok_s": 20.585847026094658, + "median_first_sse_s": 13.620587156503461, + "median_tpot_est_s": 0.02503044254920115, + "median_e2e_s": 14.379491011437494, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.608603984932415, + "output_tok_s": 27.774137334969367, + "median_first_sse_s": 1.5076510719954967, + "median_tpot_est_s": 0.024415457385636163, + "median_e2e_s": 4.608414159971289, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.176488784956746, + "output_tok_s": 55.7947611551964, + "median_first_sse_s": 3.776593458955176, + "median_tpot_est_s": 0.025861740110471376, + "median_e2e_s": 7.061034452985041, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.35276161902584, + "output_tok_s": 74.8736097848148, + "median_first_sse_s": 13.083223339985125, + "median_tpot_est_s": 0.024744039658075714, + "median_e2e_s": 16.22622210800182, + "all_text_equal": true + } + ], + "config_sha256": "def8bcbe12b01fb247db995e81b318cc99b776920ae08a84322baf96a8d9b584", + "packages": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "stream/off": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 0.0, + "transferred_tokens": 935680.0 + }, + "raw_sha256": "b7a5d24446370bc53c93a468cf8368f0096e2b37560f4918b81857a017ff37c9", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.208784915972501, + "output_tok_s": 14.487603464056974, + "median_first_sse_s": 1.4633835150161758, + "median_tpot_est_s": 0.02404057232272481, + "median_e2e_s": 2.208641257020645, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.5753573960391805, + "output_tok_s": 19.466622464826592, + "median_first_sse_s": 3.654461637022905, + "median_tpot_est_s": 0.02462164083901312, + "median_e2e_s": 4.421989984984975, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.097053020028397, + "output_tok_s": 21.247411439666436, + "median_first_sse_s": 12.423074103018735, + "median_tpot_est_s": 0.025114445162246063, + "median_e2e_s": 13.193876743491273, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.487484783050604, + "output_tok_s": 28.523773603302395, + "median_first_sse_s": 1.4645146490074694, + "median_tpot_est_s": 0.023801707173147658, + "median_e2e_s": 4.487331459997222, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 8.898463926045224, + "output_tok_s": 57.5380205230039, + "median_first_sse_s": 3.680733249988407, + "median_tpot_est_s": 0.024517924035349345, + "median_e2e_s": 6.826566770032514, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.336110907024704, + "output_tok_s": 77.76394955314876, + "median_first_sse_s": 12.45146652951371, + "median_tpot_est_s": 0.0250761886338124, + "median_e2e_s": 15.624904347525444, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.1974527299171314, + "output_tok_s": 14.562315523031414, + "median_first_sse_s": 1.4644598189042881, + "median_tpot_est_s": 0.023640203646444265, + "median_e2e_s": 2.1973061319440603, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.654567905003205, + "output_tok_s": 19.234907784735928, + "median_first_sse_s": 3.6605074605322443, + "median_tpot_est_s": 0.02419535599971911, + "median_e2e_s": 4.402268527017441, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.137016809894703, + "output_tok_s": 21.212231985110574, + "median_first_sse_s": 12.423588597972412, + "median_tpot_est_s": 0.02459894633859456, + "median_e2e_s": 13.184217029483989, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.357999384985305, + "output_tok_s": 29.371275370299667, + "median_first_sse_s": 1.4642595020122826, + "median_tpot_est_s": 0.022784263330079088, + "median_e2e_s": 4.357860944932327, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 8.883686052984558, + "output_tok_s": 57.633734121883876, + "median_first_sse_s": 3.698318095470313, + "median_tpot_est_s": 0.024855834641402192, + "median_e2e_s": 6.860209960490465, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.494456356973387, + "output_tok_s": 77.2991894004635, + "median_first_sse_s": 12.526686669967603, + "median_tpot_est_s": 0.024864938393359872, + "median_e2e_s": 15.6774896789575, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.2191090310225263, + "output_tok_s": 14.420201780376228, + "median_first_sse_s": 1.4827205820474774, + "median_tpot_est_s": 0.023748166288339322, + "median_e2e_s": 2.2189137369859964, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.728352272999473, + "output_tok_s": 19.023974192560836, + "median_first_sse_s": 3.7194058959721588, + "median_tpot_est_s": 0.0248676183381899, + "median_e2e_s": 4.490302064456046, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.420636598020792, + "output_tok_s": 20.965874413015744, + "median_first_sse_s": 12.510112286487129, + "median_tpot_est_s": 0.02547007288611043, + "median_e2e_s": 13.306779410515446, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.579361124080606, + "output_tok_s": 27.95149727915342, + "median_first_sse_s": 1.4736862790305167, + "median_tpot_est_s": 0.024453261684751006, + "median_e2e_s": 4.5792505129938945, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.010187053936534, + "output_tok_s": 56.82456944956633, + "median_first_sse_s": 3.7041273544891737, + "median_tpot_est_s": 0.02467996403565777, + "median_e2e_s": 6.847893105470575, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.87396427197382, + "output_tok_s": 76.20758810548125, + "median_first_sse_s": 12.641634211468045, + "median_tpot_est_s": 0.02488390423218155, + "median_e2e_s": 15.758425751526374, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.233825940056704, + "output_tok_s": 14.325198497420844, + "median_first_sse_s": 1.4876205119071528, + "median_tpot_est_s": 0.024065800067487982, + "median_e2e_s": 2.2336603139992803, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.711063691996969, + "output_tok_s": 19.07298244727459, + "median_first_sse_s": 3.7502244460047223, + "median_tpot_est_s": 0.02529637779169265, + "median_e2e_s": 4.5344121575471945, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.589590384042822, + "output_tok_s": 20.82181898939876, + "median_first_sse_s": 13.402657889586408, + "median_tpot_est_s": 0.026513373967440378, + "median_e2e_s": 14.244673553097527, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.601558788912371, + "output_tok_s": 27.816660803817353, + "median_first_sse_s": 1.4871936520794407, + "median_tpot_est_s": 0.024521522700610594, + "median_e2e_s": 4.601427035056986, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.045747432042845, + "output_tok_s": 56.601182361817564, + "median_first_sse_s": 3.764507564541418, + "median_tpot_est_s": 0.025608588775480532, + "median_e2e_s": 7.016798339027446, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.193519476917572, + "output_tok_s": 75.31206108640646, + "median_first_sse_s": 13.009069821040612, + "median_tpot_est_s": 0.02543677709488873, + "median_e2e_s": 16.253884334524628, + "all_text_equal": true + } + ], + "config_sha256": "5e4fbf615f999301c89914edc836f5d60e7056671d7d60f7fff521147c952be3", + "packages": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "stream/on": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 850688.0, + "transferred_tokens": 84992.0 + }, + "raw_sha256": "5d1b70e813e198a43f3182784bb44b33a3bd87e6c1054237eb0f8fc210c98541", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.2193764109397307, + "output_tok_s": 14.418464503031519, + "median_first_sse_s": 1.4607410749886185, + "median_tpot_est_s": 0.02446787642134774, + "median_e2e_s": 2.2192452440503985, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.623916062060744, + "output_tok_s": 19.32391636620141, + "median_first_sse_s": 3.652397355530411, + "median_tpot_est_s": 0.025490233806082077, + "median_e2e_s": 4.4497615519794635, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.252061516977847, + "output_tok_s": 21.111607342805492, + "median_first_sse_s": 12.537051899009384, + "median_tpot_est_s": 0.025534329404528704, + "median_e2e_s": 13.309377963014413, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.588234180002473, + "output_tok_s": 27.897442671492197, + "median_first_sse_s": 1.4684002309804782, + "median_tpot_est_s": 0.024564731629624022, + "median_e2e_s": 4.588121147942729, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 8.714870621915907, + "output_tok_s": 58.75015501807188, + "median_first_sse_s": 3.6713809845387004, + "median_tpot_est_s": 0.023759379680910124, + "median_e2e_s": 6.688822204014286, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.41557551198639, + "output_tok_s": 77.53001629931155, + "median_first_sse_s": 12.573357547051273, + "median_tpot_est_s": 0.02497935570055971, + "median_e2e_s": 15.68049829103984, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.2010550249833614, + "output_tok_s": 14.538482517147385, + "median_first_sse_s": 1.4664556350326166, + "median_tpot_est_s": 0.023691118255467904, + "median_e2e_s": 2.2008803009521216, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.63359681610018, + "output_tok_s": 19.295715966538015, + "median_first_sse_s": 3.6619083679979667, + "median_tpot_est_s": 0.02578308109584595, + "median_e2e_s": 4.465914245462045, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.173571440041997, + "output_tok_s": 21.180155413523394, + "median_first_sse_s": 12.44185278203804, + "median_tpot_est_s": 0.025642615320506476, + "median_e2e_s": 13.242599404067732, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.584160504047759, + "output_tok_s": 27.92223350098181, + "median_first_sse_s": 1.4688922829227522, + "median_tpot_est_s": 0.024528703787987393, + "median_e2e_s": 4.584037663997151, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 8.996187166892923, + "output_tok_s": 56.91299997450287, + "median_first_sse_s": 3.7267173575237393, + "median_tpot_est_s": 0.02494246259837697, + "median_e2e_s": 6.9479099354939535, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.62871236994397, + "output_tok_s": 76.90946417340079, + "median_first_sse_s": 12.502239950466901, + "median_tpot_est_s": 0.02562358440083312, + "median_e2e_s": 15.707677363534458, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.261181575944647, + "output_tok_s": 14.151893125447678, + "median_first_sse_s": 1.4730743109248579, + "median_tpot_est_s": 0.025417398646353714, + "median_e2e_s": 2.261013668961823, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.5867575000738725, + "output_tok_s": 19.432930390797665, + "median_first_sse_s": 3.6612320354324766, + "median_tpot_est_s": 0.024821561806264422, + "median_e2e_s": 4.457853511965368, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.271716309944168, + "output_tok_s": 21.09451154841624, + "median_first_sse_s": 12.441811380034778, + "median_tpot_est_s": 0.025020278642900406, + "median_e2e_s": 13.19704235746758, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.663960383972153, + "output_tok_s": 27.444486972890257, + "median_first_sse_s": 1.461892998078838, + "median_tpot_est_s": 0.025212164652922493, + "median_e2e_s": 4.663837908999994, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 9.092023623059504, + "output_tok_s": 56.313096096830186, + "median_first_sse_s": 3.6667037500301376, + "median_tpot_est_s": 0.025807405223860956, + "median_e2e_s": 6.977219620952383, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 26.680595836951397, + "output_tok_s": 76.75990493299307, + "median_first_sse_s": 12.540620018960908, + "median_tpot_est_s": 0.025764916421236895, + "median_e2e_s": 15.884110663027968, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.3031733550596982, + "output_tok_s": 13.893873828342617, + "median_first_sse_s": 1.5569050030317158, + "median_tpot_est_s": 0.024067208193422805, + "median_e2e_s": 2.3029884570278227, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 6.612298600957729, + "output_tok_s": 19.357867471602145, + "median_first_sse_s": 3.6647086894954555, + "median_tpot_est_s": 0.02475741664473448, + "median_e2e_s": 4.425869460974354, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 24.245720005012117, + "output_tok_s": 21.11712912192991, + "median_first_sse_s": 13.209060697467066, + "median_tpot_est_s": 0.025803333790522187, + "median_e2e_s": 14.002716014510952, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.486034267931245, + "output_tok_s": 28.532996485340664, + "median_first_sse_s": 1.4778158440021798, + "median_tpot_est_s": 0.0236857090160016, + "median_e2e_s": 4.485900889034383, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 8.97912462207023, + "output_tok_s": 57.021148669830254, + "median_first_sse_s": 3.724316539533902, + "median_tpot_est_s": 0.025794144878420598, + "median_e2e_s": 7.000172939093318, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 27.04220046196133, + "output_tok_s": 75.73348192876541, + "median_first_sse_s": 12.852511344943196, + "median_tpot_est_s": 0.025671009602223148, + "median_e2e_s": 16.030342576967087, + "all_text_equal": true + } + ], + "config_sha256": "01c746af74c5dc1dc122c9ebefff5ad0118e245bd8666f0c18c973731503d644", + "packages": { + "torch": "2.13.0+cu130", + "transformers": "5.15.0" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + } + }, + "comparisons": { + "stream/off -> stream/on": { + "increased_conditions": 10, + "total_conditions": 24, + "min_throughput_change_pct": -4.933534043207166, + "max_throughput_change_pct": 2.5752037118164983 + }, + "whole/on -> stream/on": { + "increased_conditions": 20, + "total_conditions": 24, + "min_throughput_change_pct": -3.3412670774057407, + "max_throughput_change_pct": 4.94821890904793 + } + }, + "scope": { + "conditions": 96, + "responses": 1376, + "repeats_per_condition": 1 + }, + "limits": [ + "Single ordered runs, no statistical repeatability claim.", + "Same-reference parity is not native-engine equivalence or answer quality.", + "One A10080GB plus one RNGD research producer;four GPU slots,BF16,context9216,page/chunk256,width8192.", + "Cache counters count logical KV tokens;full producer prefill andCPUcopy remain. No physical DMA overlap orPCIe-byte reduction proof." + ], + "all_four_reference_sets_equal": true +} diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md index 05108044..80162d51 100644 --- a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md @@ -81,3 +81,18 @@ The whole-KV cache-on retry completed 24 conditions/344 responses, matching reta At 8192 input, 128 output, concurrency 4, native versus whole PD delivered 23.0725 versus 15.0415 tok/s, with median first SSE 8.43916 versus 15.8125 s. This single ordered comparison does not show a PD advantage; it is not a fresh cache off/on speedup measurement. Native/PD reference outputs match 6/8 cases. At 512/128 and 2048/128, they diverge at generated-token indices 50 and 61, respectively, before EOS. The same divergent arrays already occur in the archived cache-off runs; all four historical raw hashes and same-route reference arrays were verified. See ../evidence267/qwen4-atom-divergence-history.json. These cases are not new cache-on regressions, but their numerical root cause remains unresolved. Do not label cross-engine parity as passing. Streamed ATOM KV is still being measured. + +## Gemma3-1B GPU PD configurations complete + +The exact `unsloth/gemma-3-1b-it` whole/stream x cache off/on grid completed all four runs: 96 conditions, 1376 responses, zero same-route mismatches. All four prompt/reference sets equal; host listen drop/overflow increments are zero in every run. Full 96 timing rows, raw/config/source fingerprints and package versions are in ../evidence267/gemma-full-pd-cache-summary.json. Use the GPU reproduction commands above with this model and `streaming_factories:gemma`; retain page/chunk256, context9216, producerwidth8192, four GPU slots, BF16, one A10080GB and one RNGD. Add `--pd-prefill-stream-layers` for stream mode and `--prefix-cache` plus decoder-cache metadata for cache-on, referencing the fresh same-route off result. Producer cache remains off. + +At 8192 input, 128 output, concurrency 4: + +| Configuration | Output tok/s | Median first SSE (s) | +|---|---:|---:| +| Whole, cache off | 55.1567 | 3.93112 | +| Whole, cache on | 55.7948 | 3.77659 | +| Stream, cache off | 56.6012 | 3.76451 | +| Stream, cache on | 57.0211 | 3.72432 | + +Cache-on improves streamed throughput in only 10/24 conditions, with changes from -4.93% to +2.58%. At cache-on, stream beats whole in 20/24 conditions (-3.34% to +4.95%). These small differences in single ordered samples do not establish stable or universal gains. Same-route outputs do not certify native-engine equivalence or answer quality; logical transferred tokens do not measure physical PCIe traffic. KONI GPU and Qwen4 ATOM streaming remain in progress. From dc46ce78478f55d5242d43052024138214e29999 Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 30 Oct 2026 15:50:33 +0900 Subject: [PATCH 8/8] docs: complete Qwen4 ATOM cache-on route evidence --- .../pd/http_matrix/.research/events.jsonl | 1 + opt_prime/examples/pd/http_matrix/STATUS.md | 2 + .../evidence267/qwen4-atom-full-summary.json | 1072 +++++++++++++++++ .../experiments/2026-10-30-http-backlog267.md | 14 + 4 files changed, 1089 insertions(+) create mode 100644 opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-full-summary.json diff --git a/opt_prime/examples/pd/http_matrix/.research/events.jsonl b/opt_prime/examples/pd/http_matrix/.research/events.jsonl index 77c0e413..8e816355 100644 --- a/opt_prime/examples/pd/http_matrix/.research/events.jsonl +++ b/opt_prime/examples/pd/http_matrix/.research/events.jsonl @@ -36,3 +36,4 @@ {"ts": "2026-10-30T06:03:17.316059+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 GPU PD four configurations complete96conditions1376responses exact;all4reference sets equal,listen deltas0. Full grid provenance/timings retained;single ordered samples only. Gemma GPU and Qwen4 ATOM pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} {"ts": "2026-10-30T06:38:39.539907+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 ATOM whole/on24conditions344responses same-route exact,listen deltas0. Native/PD6/8refs equal;2preEOS differences also present in historical cacheoff. At8K128c4 native23.07vsPD15.04tok/s. Streaming pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} {"ts": "2026-10-30T06:45:47.350695+00:00", "task": "aicomp #267 / #201", "result": "Gemma GPU PD four configurations96conditions1376responses exact,all4reference sets equal,listen deltas0. Full timing/provenance retained;streamcache gain10/24 only(-4.93..+2.58%),single ordered samples. Remaining model tests pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} +{"ts": "2026-10-30T06:50:26.376967+00:00", "task": "aicomp #267 / #201", "result": "Qwen4 ATOM native/whole/stream72conditions1032responses same-route exact;whole/stream8/8refs,nativePD6/8.8K128c4 native23.07/whole15.04/stream16.52tok/s. Owned cleanup and freshabsence verified. KONI GPU/TP2 pending.", "state": "active", "done": false, "review": true, "experiments": ["experiments/2026-10-30-http-backlog267.md"]} diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index ded34680..15237318 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -162,3 +162,5 @@ PR289 Qwen4 GPU subset: Qwen4 GPU PD four configurations complete96conditions137 PR289 ATOM PD follow-up: Qwen4 ATOM whole/on24conditions344responses same-route exact,listen deltas0. Native/PD6/8refs equal;2preEOS differences also present in historical cacheoff. At8K128c4 native23.07vsPD15.04tok/s. Streaming pending. PR289 Gemma GPU subset: Gemma GPU PD four configurations96conditions1376responses exact,all4reference sets equal,listen deltas0. Full timing/provenance retained;streamcache gain10/24 only(-4.93..+2.58%),single ordered samples. Remaining model tests pending. + +PR289 Qwen4 ATOM subset: Qwen4 ATOM native/whole/stream72conditions1032responses same-route exact;whole/stream8/8refs,nativePD6/8.8K128c4 native23.07/whole15.04/stream16.52tok/s. Owned cleanup and freshabsence verified. KONI GPU/TP2 pending. diff --git a/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-full-summary.json b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-full-summary.json new file mode 100644 index 00000000..36837bfd --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence267/qwen4-atom-full-summary.json @@ -0,0 +1,1072 @@ +{ + "model": "Qwen3-4B", + "scope": { + "conditions": 72, + "responses": 1032, + "cache": "on", + "repeats": 1 + }, + "arms": { + "native": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 867456.0 + }, + "raw_sha256": "f6b0ed35f371a37e6460adebb51eca0af569b1b2fb061f199896d63b0d8b91a5", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 1.21979605499655, + "output_tok_s": 26.233893665191847, + "median_first_sse_s": 0.06424786592833698, + "median_tpot_est_s": 0.03727080548302301, + "median_e2e_s": 1.2196428359020501, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 4.865430230041966, + "output_tok_s": 26.308053748187437, + "median_first_sse_s": 1.8862478689989075, + "median_tpot_est_s": 0.03721824728868782, + "median_e2e_s": 3.0406833300366998, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 19.443709495011717, + "output_tok_s": 26.332423868570633, + "median_first_sse_s": 9.170724456431344, + "median_tpot_est_s": 0.03721861124053717, + "median_e2e_s": 10.32500390149653, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.779329141136259, + "output_tok_s": 26.782001452523673, + "median_first_sse_s": 0.06421606685034931, + "median_tpot_est_s": 0.03712567288289213, + "median_e2e_s": 4.77917652297765, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.179921305039898, + "output_tok_s": 26.69458293686857, + "median_first_sse_s": 7.2513526930706576, + "median_tpot_est_s": 0.03726597728691702, + "median_e2e_s": 11.985176004120149, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 76.49607840902172, + "output_tok_s": 26.772614264608695, + "median_first_sse_s": 35.89145322609693, + "median_tpot_est_s": 0.0371851845594915, + "median_e2e_s": 40.606709040584974, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 1.409332315903157, + "output_tok_s": 22.705787441972554, + "median_first_sse_s": 0.248691565124318, + "median_tpot_est_s": 0.0374359464828646, + "median_e2e_s": 1.4092059060931206, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 4.901245624991134, + "output_tok_s": 26.115810100872377, + "median_first_sse_s": 1.9010033400263637, + "median_tpot_est_s": 0.03745433188704474, + "median_e2e_s": 3.062635939917527, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 19.62891486589797, + "output_tok_s": 26.08396865022408, + "median_first_sse_s": 9.260896492982283, + "median_tpot_est_s": 0.03751262429080182, + "median_e2e_s": 10.424593756091781, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.838200076017529, + "output_tok_s": 26.456119628967624, + "median_first_sse_s": 0.06642191391438246, + "median_tpot_est_s": 0.0375715390867077, + "median_e2e_s": 4.83800737792626, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.333293949021026, + "output_tok_s": 26.482812569346258, + "median_first_sse_s": 7.3223171149147674, + "median_tpot_est_s": 0.03758554547607136, + "median_e2e_s": 12.085917299962603, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 77.24892303696834, + "output_tok_s": 26.51169646753401, + "median_first_sse_s": 36.27493576845154, + "median_tpot_est_s": 0.03750518164927448, + "median_e2e_s": 41.03906974289566, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 2.0401992758270353, + "output_tok_s": 15.684742357840591, + "median_first_sse_s": 0.8407348680775613, + "median_tpot_est_s": 0.03868695670708774, + "median_e2e_s": 2.040030525997281, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 5.082977191079408, + "output_tok_s": 25.182092145650223, + "median_first_sse_s": 1.9748041100101545, + "median_tpot_est_s": 0.038678585807041775, + "median_e2e_s": 3.1738402700284496, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 20.373157310998067, + "output_tok_s": 25.13110718109492, + "median_first_sse_s": 9.618916884530336, + "median_tpot_est_s": 0.03872891262759484, + "median_e2e_s": 10.823435940546915, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 4.998283860972151, + "output_tok_s": 25.608789648674414, + "median_first_sse_s": 0.07471933006308973, + "median_tpot_est_s": 0.038767074181617595, + "median_e2e_s": 4.9981377511285245, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 19.97290690196678, + "output_tok_s": 25.63472620750974, + "median_first_sse_s": 7.558827344560996, + "median_tpot_est_s": 0.038741515790786975, + "median_e2e_s": 12.481202929164283, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 79.82538939104415, + "output_tok_s": 25.655997617091124, + "median_first_sse_s": 37.48764609743375, + "median_tpot_est_s": 0.0387145414293971, + "median_e2e_s": 42.39941652864218, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 5.352265110006556, + "output_tok_s": 5.978777086391523, + "median_first_sse_s": 4.025504268007353, + "median_tpot_est_s": 0.04279371299931119, + "median_e2e_s": 5.352109370986, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 5.7538805468939245, + "output_tok_s": 22.24585633240115, + "median_first_sse_s": 2.2709930365672335, + "median_tpot_est_s": 0.04280944161056992, + "median_e2e_s": 3.5957872320432216, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 23.044004359981045, + "output_tok_s": 22.218360663442486, + "median_first_sse_s": 10.923830434447154, + "median_tpot_est_s": 0.042838023904140196, + "median_e2e_s": 12.250851112417877, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 5.550273383036256, + "output_tok_s": 23.06192707393777, + "median_first_sse_s": 0.11678028688766062, + "median_tpot_est_s": 0.04278182463104448, + "median_e2e_s": 5.55007201503031, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 22.190961834043264, + "output_tok_s": 23.07245642748744, + "median_first_sse_s": 8.439156876062043, + "median_tpot_est_s": 0.0427968583077075, + "median_e2e_s": 13.874486317043193, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 88.77933540102094, + "output_tok_s": 23.068431304977402, + "median_first_sse_s": 41.72694445645902, + "median_tpot_est_s": 0.04281781166131923, + "median_e2e_s": 47.15564054495189, + "all_text_equal": true + } + ], + "config_sha256": "fc6780d557b89f162940b8725bf11c65b306cb76c0c117ccb45fcc619d945d46", + "packages": { + "torch": "2.11.0+cpu", + "transformers": "5.8.1" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "whole": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 867456.0 + }, + "raw_sha256": "bd0bc2e81c131d9c179268afd97066316bb70bc6904a3002c670f8cf1eef79fa", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.229157143039629, + "output_tok_s": 7.566519502039735, + "median_first_sse_s": 3.07451313897036, + "median_tpot_est_s": 0.03723970416092103, + "median_e2e_s": 4.228943967958912, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 16.919324713060632, + "output_tok_s": 7.565313756357677, + "median_first_sse_s": 9.443536977516487, + "median_tpot_est_s": 0.0372378321612374, + "median_e2e_s": 10.594949356513098, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 67.71203166921623, + "output_tok_s": 7.561433136450541, + "median_first_sse_s": 34.59739960345905, + "median_tpot_est_s": 0.03715300685425679, + "median_e2e_s": 35.75067046692129, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 7.789689174853265, + "output_tok_s": 16.431977852622232, + "median_first_sse_s": 3.0535819258075207, + "median_tpot_est_s": 0.0372907919919221, + "median_e2e_s": 7.789512508781627, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 31.15392838884145, + "output_tok_s": 16.434524519976282, + "median_first_sse_s": 14.739977189921774, + "median_tpot_est_s": 0.03724773100019968, + "median_e2e_s": 19.466690970933996, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 124.65422019199468, + "output_tok_s": 16.429447770365364, + "median_first_sse_s": 61.595142140984535, + "median_tpot_est_s": 0.03722072855521756, + "median_e2e_s": 66.31608892697841, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.327635735040531, + "output_tok_s": 7.39433768440779, + "median_first_sse_s": 3.1801685569807887, + "median_tpot_est_s": 0.03700972913253692, + "median_e2e_s": 4.327470160089433, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 17.268865237943828, + "output_tok_s": 7.412183616949733, + "median_first_sse_s": 9.761258519953117, + "median_tpot_est_s": 0.037525404207107996, + "median_e2e_s": 10.924995203502476, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 67.5510564618744, + "output_tok_s": 7.579452148005578, + "median_first_sse_s": 34.785325877484865, + "median_tpot_est_s": 0.03732400219286641, + "median_e2e_s": 35.948298823903315, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 8.1633458530996, + "output_tok_s": 15.679845286892844, + "median_first_sse_s": 3.39684719289653, + "median_tpot_est_s": 0.03752971129257261, + "median_e2e_s": 8.163120527053252, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 31.40743782185018, + "output_tok_s": 16.30187100597557, + "median_first_sse_s": 14.859845734550618, + "median_tpot_est_s": 0.03752742784178456, + "median_e2e_s": 19.622518819523975, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 125.70304442220367, + "output_tok_s": 16.29236594398862, + "median_first_sse_s": 61.88206681644078, + "median_tpot_est_s": 0.03753838226323815, + "median_e2e_s": 66.65103639895096, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.603710082126781, + "output_tok_s": 6.95091555053287, + "median_first_sse_s": 3.4106448700185865, + "median_tpot_est_s": 0.03847976864105271, + "median_e2e_s": 4.6035176978912205, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 17.03182191797532, + "output_tok_s": 7.5153439612299655, + "median_first_sse_s": 9.448679500841536, + "median_tpot_est_s": 0.03850972472930387, + "median_e2e_s": 10.642480967449956, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 68.56243541510776, + "output_tok_s": 7.467646049912349, + "median_first_sse_s": 35.08746871794574, + "median_tpot_est_s": 0.03867314664662004, + "median_e2e_s": 36.28633626399096, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 8.001783073879778, + "output_tok_s": 15.996434646901442, + "median_first_sse_s": 3.0788507510442287, + "median_tpot_est_s": 0.03876198732338142, + "median_e2e_s": 8.001623141113669, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 32.082207593135536, + "output_tok_s": 15.959001527985562, + "median_first_sse_s": 15.150795516441576, + "median_tpot_est_s": 0.03876264726809101, + "median_e2e_s": 20.078690478461795, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 128.25819011521526, + "output_tok_s": 15.967791204290867, + "median_first_sse_s": 63.30086616508197, + "median_tpot_est_s": 0.038750230264116635, + "median_e2e_s": 68.22345729661174, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 5.651906156912446, + "output_tok_s": 5.661806673994944, + "median_first_sse_s": 4.324762323871255, + "median_tpot_est_s": 0.04280508706916965, + "median_e2e_s": 5.651720023015514, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 17.66353549505584, + "output_tok_s": 7.246567372416932, + "median_first_sse_s": 9.705989120993763, + "median_tpot_est_s": 0.04286090721331176, + "median_e2e_s": 11.035693836514838, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 70.67930217203684, + "output_tok_s": 7.24398776255271, + "median_first_sse_s": 36.408837280469015, + "median_tpot_est_s": 0.042833625098630305, + "median_e2e_s": 37.739714586525224, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 8.460688102990389, + "output_tok_s": 15.128793124374722, + "median_first_sse_s": 3.0657620751298964, + "median_tpot_est_s": 0.04247847278639087, + "median_e2e_s": 8.460528119001538, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 34.039199938997626, + "output_tok_s": 15.041481612892374, + "median_first_sse_s": 15.812518983031623, + "median_tpot_est_s": 0.04263964579825649, + "median_e2e_s": 21.22309566254262, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 137.80537095014006, + "output_tok_s": 14.86153976350454, + "median_first_sse_s": 67.86479224008508, + "median_tpot_est_s": 0.042862537114396984, + "median_e2e_s": 73.3110099316109, + "all_text_equal": true + } + ], + "config_sha256": "12bbf46aa3cfdcf821075371abd139f61397757f90a513617cdc9fac282dcbf5", + "packages": { + "torch": "2.11.0+cpu", + "transformers": "5.8.1" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + }, + "stream": { + "cache_counters": { + "query_tokens": 935680.0, + "hit_tokens": 867456.0 + }, + "raw_sha256": "807e9f39105b665672eab4afbe0c35e3c4759c728ad4243cefdc70bffa5c9c6b", + "conditions": 24, + "warmups": 8, + "token_checks": 168, + "sse_checks": 168, + "mismatches": [], + "parity_passed": true, + "measurements": [ + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.5126539489720017, + "output_tok_s": 9.109921006982479, + "median_first_sse_s": 2.357954327017069, + "median_tpot_est_s": 0.037243864131009864, + "median_e2e_s": 3.5125141150783747, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 14.053763817995787, + "output_tok_s": 9.107880398281386, + "median_first_sse_s": 7.634039894444868, + "median_tpot_est_s": 0.03725226656458671, + "median_e2e_s": 8.789965662406757, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 56.79622251703404, + "output_tok_s": 9.014684028439452, + "median_first_sse_s": 28.73849768063519, + "median_tpot_est_s": 0.037373773224141094, + "median_e2e_s": 29.89875908952672, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 7.1622115729842335, + "output_tok_s": 17.871574819545167, + "median_first_sse_s": 2.415262032067403, + "median_tpot_est_s": 0.03737608453544869, + "median_e2e_s": 7.1620247680693865, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 28.278648821869865, + "output_tok_s": 18.105532666187163, + "median_first_sse_s": 12.956028542481363, + "median_tpot_est_s": 0.03734197042603779, + "median_e2e_s": 17.682385004940443, + "all_text_equal": true + }, + { + "input_tokens": 128, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 113.36889456398785, + "output_tok_s": 18.064919904851543, + "median_first_sse_s": 55.33246057049837, + "median_tpot_est_s": 0.03726828140463299, + "median_e2e_s": 60.070664858445525, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.539515095995739, + "output_tok_s": 9.040786416252798, + "median_first_sse_s": 2.375210168072954, + "median_tpot_est_s": 0.03755417258118189, + "median_e2e_s": 3.5393895180895925, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 14.154375656042248, + "output_tok_s": 9.043139952651963, + "median_first_sse_s": 7.684967851499096, + "median_tpot_est_s": 0.03737933471077873, + "median_e2e_s": 8.843727227533236, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 56.62839795812033, + "output_tok_s": 9.041400047704878, + "median_first_sse_s": 29.140717443544418, + "median_tpot_est_s": 0.037329213949100624, + "median_e2e_s": 30.300555902533233, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 7.042883754009381, + "output_tok_s": 18.174373519530548, + "median_first_sse_s": 2.2993753449991345, + "median_tpot_est_s": 0.03734899102343525, + "median_e2e_s": 7.042697204975411, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 28.71817106497474, + "output_tok_s": 17.82843339297625, + "median_first_sse_s": 13.322041620849632, + "median_tpot_est_s": 0.03748940284671922, + "median_e2e_s": 18.08985018741805, + "all_text_equal": true + }, + { + "input_tokens": 512, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 113.7767394981347, + "output_tok_s": 18.00016426058312, + "median_first_sse_s": 55.707091104472056, + "median_tpot_est_s": 0.03747298275615551, + "median_e2e_s": 60.47448480396997, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 3.685936684953049, + "output_tok_s": 8.68164668444586, + "median_first_sse_s": 2.4918664928991348, + "median_tpot_est_s": 0.038512731196298715, + "median_e2e_s": 3.685761159984395, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 14.054411371937022, + "output_tok_s": 9.107460754676818, + "median_first_sse_s": 7.50499695003964, + "median_tpot_est_s": 0.03866276983922768, + "median_e2e_s": 8.704841685015708, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 55.85377869592048, + "output_tok_s": 9.166792506330392, + "median_first_sse_s": 28.488389546517283, + "median_tpot_est_s": 0.03864750609312567, + "median_e2e_s": 29.683132148580626, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 7.289653543150052, + "output_tok_s": 17.55913353663826, + "median_first_sse_s": 2.3522673710249364, + "median_tpot_est_s": 0.03887548847270646, + "median_e2e_s": 7.289454407058656, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 29.20872266800143, + "output_tok_s": 17.5290102829763, + "median_first_sse_s": 13.483660011435859, + "median_tpot_est_s": 0.038700744814943434, + "median_e2e_s": 18.39187702606432, + "all_text_equal": true + }, + { + "input_tokens": 2048, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 115.28838631813414, + "output_tok_s": 17.76414837092626, + "median_first_sse_s": 56.357253332971595, + "median_tpot_est_s": 0.03870382958722455, + "median_e2e_s": 61.261467059957795, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 1, + "requests": 1, + "output_tokens": 32, + "wall_s": 4.428202794166282, + "output_tok_s": 7.226407978007879, + "median_first_sse_s": 3.101593024097383, + "median_tpot_est_s": 0.04278855993893118, + "median_e2e_s": 4.4280383822042495, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 4, + "requests": 4, + "output_tokens": 128, + "wall_s": 14.548575706081465, + "output_tok_s": 8.798112102925277, + "median_first_sse_s": 7.8112124215113, + "median_tpot_est_s": 0.042900833387618825, + "median_e2e_s": 9.141138256527483, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 32, + "concurrency": 16, + "requests": 16, + "output_tokens": 512, + "wall_s": 58.156303505878896, + "output_tok_s": 8.803860787820584, + "median_first_sse_s": 29.476370697491802, + "median_tpot_est_s": 0.04284157726027432, + "median_e2e_s": 30.802679410437122, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 1, + "requests": 1, + "output_tokens": 128, + "wall_s": 7.745887920958921, + "output_tok_s": 16.52489699129986, + "median_first_sse_s": 2.3028251989744604, + "median_tpot_est_s": 0.04285746511057254, + "median_e2e_s": 7.745723268017173, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 4, + "requests": 4, + "output_tokens": 512, + "wall_s": 30.992978358874097, + "output_tok_s": 16.51987085821331, + "median_first_sse_s": 13.941098610986955, + "median_tpot_est_s": 0.04283978970534366, + "median_e2e_s": 19.382869886001572, + "all_text_equal": true + }, + { + "input_tokens": 8192, + "output_tokens_per_request": 128, + "concurrency": 16, + "requests": 16, + "output_tokens": 2048, + "wall_s": 124.24822813295759, + "output_tok_s": 16.48313244200507, + "median_first_sse_s": 60.76766407804098, + "median_tpot_est_s": 0.04279533552288217, + "median_e2e_s": 66.20954955206253, + "all_text_equal": true + } + ], + "config_sha256": "45e0beaeb76c678614cb4e084fbc9d9317827f1e8acc24e3b51d0f5a39742a50", + "packages": { + "torch": "2.11.0+cpu", + "transformers": "5.8.1" + }, + "listen_counter_delta": { + "ListenOverflows": 0, + "ListenDrops": 0 + } + } + }, + "reference_equality": { + "whole/stream": { + "512:32": true, + "512:128": true, + "2048:32": true, + "2048:128": true, + "128:32": true, + "128:128": true, + "8192:32": true, + "8192:128": true + }, + "native/whole": { + "512:32": true, + "512:128": false, + "2048:32": true, + "2048:128": false, + "128:32": true, + "128:128": true, + "8192:32": true, + "8192:128": true + }, + "native/stream": { + "512:32": true, + "512:128": false, + "2048:32": true, + "2048:128": false, + "128:32": true, + "128:128": true, + "8192:32": true, + "8192:128": true + } + }, + "limits": [ + "Historical same-route off references; not a fresh cache off/on performance pair.", + "One ATOM execution lane; concurrency denotes queued clients. PD adds one RNGD.", + "Single ordered runs, not stable speedup estimates.", + "Native/PD pre-EOS differences retained; no broad engine equivalence." + ], + "source_sha256": { + "benchmark267_deadline.py": "c8ebb077a608e3da70a9748aed19c03f22ae87355f0167a2fb1e68b88ec21d79", + "atom_server265_http.py": "f222ebe20de1b7e6d4c233907ff41a248694c01d75382dd8be2bccaad53a7300", + "llmd_mid201_1030.py": "e3449979b88d03edb6d5bf0d402a708e5d98054fdf203e22b480f3d7e6fd759a", + "container_bootstrap.py": "9225577d93008057d4dd7ff93364cd6c6fb37aa8ce40aa819d6f01a5399d054e", + "run-http267-deadline-atom.sh": "a33b0d7643e89eee54b76717b6c85dceedf4ad1d5e7bc2bc5552507826346b3c", + "furiosa_prefill_python.sh": "317036f016a063d4495f8d81af9523b327a95a52bc7d1d740d664e5f197a0edc", + "streaming_factories.py": "77745350973cc54bf508e0c8b473efc442afef20b1c64f7c4edc179ea1471389", + "prefill.py": "53dfd7123d0e7593cfb9073bcd125c92af267204fc480b06ef615cf58f4b5cbd", + "long_width_variant.py": "74849665643d87b6d44f86c64849c4f532c7dedff9880404950a2692ec443c84", + "batching.py": "d98a51d75a36cd8c379a6220adb92528b3607cf80145b85a032270494a429580", + "pd_batching.py": "c827775c1d399bf724a9747c3424ee5ea69f93054493bfbb71d5f0cf08fff65b", + "llmd_backlog267_1030.py": "9f3aaf0902f7b1958f32be12b1f276384f20cc3e0070571af10bb7bce148c1f0", + "atom_cache201_http.py": "e47eb03234fa4d5bac425351f22397e6cd9a5aad730f527a444ed4aac1b241b3" + } +} diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md index 80162d51..96c3b4df 100644 --- a/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-30-http-backlog267.md @@ -96,3 +96,17 @@ At 8192 input, 128 output, concurrency 4: | Stream, cache on | 57.0211 | 3.72432 | Cache-on improves streamed throughput in only 10/24 conditions, with changes from -4.93% to +2.58%. At cache-on, stream beats whole in 20/24 conditions (-3.34% to +4.95%). These small differences in single ordered samples do not establish stable or universal gains. Same-route outputs do not certify native-engine equivalence or answer quality; logical transferred tokens do not measure physical PCIe traffic. KONI GPU and Qwen4 ATOM streaming remain in progress. + +## Qwen3-4B ATOM three cache-on routes complete + +Native, whole-KV PD and streamed-KV PD all completed: 72 conditions, 1032 responses matching retained same-route references. Each has zero host listen drop/overflow increment and 867456 hits of 935680 prefix queries. Whole/stream reference outputs match 8/8; native/PD matches 6/8, retaining the two documented pre-EOS differences. Full 72 timing rows, source/config/raw fingerprints, package versions and reference comparisons are in ../evidence267/qwen4-atom-full-summary.json. + +At 8192 input, 128 output, concurrency 4: + +| Route | Output tok/s | Median first SSE (s) | +|---|---:|---:| +| Native ATOM | 23.0725 | 8.43916 | +| RNGD to ATOM, whole KV | 15.0415 | 15.8125 | +| RNGD to ATOM, streamed KV | 16.5199 | 13.9411 | + +Streaming improves this condition versus whole KV but remains below native ATOM. These are single ordered cache-on runs, not a fresh cache-off/on timing comparison. One decoder execution lane, B1/k3/context9216; PD adds an RNGD research producer with width8192. Queued concurrency is not hardware batching. No universal native/PD equality or repeatable speedup claim. The coordinator exited zero, deleted its owned Pod with UID verification, and fresh namespace absence was checked before the separate KONI TP2 allocation. Remaining KONI GPU and TP2 tests continue; the full goal is incomplete.