Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions opt_prime/examples/pd/http_matrix/.gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
evidence*/*.json linguist-generated=true
1 change: 1 addition & 0 deletions opt_prime/examples/pd/http_matrix/.research/events.jsonl
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@
{"id": "4c7935b757264687b1310fbe1ec6399a", "ts": "2026-10-23T21:26:38+09:00", "task": "Tokenizer EOS in batched serving (aicomp #229)", "result": "Production factory union of generation-config/tokenizer EOS fixes real KONI HTTP turn end: baseline length, fixed stop, ignore_eos length, streaming stop, reuse stop. Baseline factory3fail/3pass ->22related tests pass. Preserved failed initial SSE checker log.", "next": "Publish scoped serving fix; verify NPU HTTP and retain long RNGD semantic validation.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"model": "KONI-8B", "backend": "gpu", "max_tokens": "8"}, "artifacts": [{"path": "analysis/serving-matrix-201/eos229-koni-http-final.json", "size": 5067, "sha256": "d5be9a4a52a6485f"}], "experiments": ["experiments/2026-10-23-serving-eos229.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
{"id": "e4993823527144b5a14e6441b1cd5c01", "ts": "2026-10-23T21:29:56+09:00", "task": "Tokenizer EOS in batched serving (aicomp #229)", "result": "Published production EOS fix and real KONI HTTP evidence in draft PR230 (e5fa087). Final factory/streaming/sampling/batched suite22passed; hardware checker5/5 cases passed; changed files sensitive-clean,4unchanged baseline findings in this branch.", "next": "Verify same HTTP factory on NPU; continue full four-model long semantic and heterogeneous PD validation.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"backend": "gpu"}, "artifacts": [{"path": "analysis/serving-matrix-201/eos229-koni-http-final.json", "size": 5067, "sha256": "d5be9a4a52a6485f"}], "experiments": ["experiments/2026-10-23-serving-eos229.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
{"id": "328b8349e9974a059ddc7e1a8a485e2c", "ts": "2026-10-23T21:32:23+09:00", "task": "Tokenizer EOS in batched serving (aicomp #229)", "result": "Real ATOM KONI HTTP also passes5/5 baseline/fixed/ignore/stream/reuse checks using same production factory: config-only length vs union stop. GPU+ATOM now10/10 expected outcomes; no SDK/artifact rebuild.", "next": "Update PR230 with NPU evidence; finish KONI medium-context RNGD handoff and broader PD work.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {"backend": "atom", "kv_blocks": "1"}, "artifacts": [{"path": "analysis/serving-matrix-201/eos229-koni-atom-http.json", "size": 5199, "sha256": "4735255db379ff00"}], "experiments": ["experiments/2026-10-23-serving-eos229.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
{"id": "2a2897e2be914684b04942a24bf4f220", "ts": "2026-10-24T03:15:52+09:00", "task": "Chunk-copy overlap (aicomp #47)", "result": "Four-model GPU copy-overlap A/B complete: 192 timed batches passed output parity. Long/c8 changes vary by block; Qwen4 B +17.39% is confounded by slower baseline prefill. Keep opt-in.", "next": "Publish reverse-block evidence; finish warm RNGD-to-ATOM prefetch comparison for KONI, Qwen3-4B, Qwen3-0.6B and Gemma3-1B", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [{"path": "analysis/serving-matrix-201/chunk47-four-model-serving-ab-summary.json", "size": 10794, "sha256": "52442216970a0e25"}], "experiments": ["experiments/2026-10-24-chunk-copy47.md"], "labels": ["benchmark"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"}
10 changes: 10 additions & 0 deletions opt_prime/examples/pd/http_matrix/STATUS.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,3 +37,13 @@ A real terminated subprocess producer previously left /ready200 and new requests
## Batched producer timing (#249)

Candidate exports airm_pd_producer_seconds_total{stage="prefill"|"transfer"} from existing accumulated batcher durations. Repeated scrapes do not add observations; missing timing keys are omitted. Tiny real-model admission test covers fixed worker-reported durations and legacy omissions. These counters are worker-reported admitted-handoff sums, not request latency or proof of zero cost for workers that do not measure stages. Full CPU suite:467passed,17skipped,135subtests,1existing warning. See pd-stage-metrics249 experiment.

## Chunk-copy overlap (#47), 2026-10-24

Opt-in Torch GPU token-chunk copies implemented with two pending spans, pinned-host staging, source readiness events and completion before handoff. Focused CUDA correctness/failure checks pass; CPU suite 504 passed, 43 skipped, 253 subtests. Actual CUDA timeline, four-model serving speedup, large-KV curve and RNGD/ATOM layer-wise export remain open. Default remains whole-span transfer. See experiments/2026-10-24-chunk-copy47.md.

Qwen3-0.6B input8192 paired prefill: median997.98 ->955.64 ms (-4.24%), six runs KV/token exact. Separate CUDA traces show 27.91ms DtoH/source-kernel intersection (mostly27.47ms KV index gather), zero GEMM intersection. Both modes transfer939,524,096 bytes. Four-model HTTP A/B remains live; no serving speedup claim.

GPU HTTP blockA96batches all parity;long/c8 chunk-copy throughput changes: Qwen0.6-3.1%,Qwen4+2.5%,KONI+2.4%,Gemma-1.5%. Reverse-order blockB pending. Export-only0.1–4GB curve: chunked slower, but incremental source staging bounded at~67.1MB rather than4GB. Keep opt-in; no general serving speedup claim.

Reverse-order blockB completed: 192 total timed batches passed parity. Long/c8 A/B changes: Qwen0.6 -3.12%/+5.80%, Qwen4 +2.49%/+17.39%, KONI +2.38%/-1.53%, Gemma -1.54%/-0.28%. Qwen4 B baseline prefill slowed while transfer stayed constant, confounding the larger gain. Keep opt-in; no repeatable general throughput improvement. See evidence47/chunk47-four-model-serving-ab-summary.json.
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
"""Whole-span versus bounded chunk export, including gather; no model compute to hide copies behind.
Run only after the owned GPU serving queue has finished and both devices are idle.
"""
import json, math, time, traceback
from pathlib import Path
import torch
from opt_prime.accel.pd.paged import PagedKV
from opt_prime.accel.pd.pd_batching import _ChunkCopy, move_kv

root=Path(__file__).resolve().parent
path=root/'chunk47-size-curve.json'
if path.exists():raise RuntimeError('Refusing overwrite '+str(path))
torch.set_num_threads(4)
result=dict(passed=False,torch=torch.__version__,runtime_commit='0870dd3',chunk=256,
method='Export wall time includes source KV gather and both host-staged DMA legs; excludes validation and warmup. No concurrent model compute. Decimal GB, rounded up to a full 64-token page.',rows=[])

@torch.inference_mode()
def run(kv,n,overlap):
for device in [0,1]:torch.cuda.synchronize(device)
initial={d:torch.cuda.memory_allocated(d) for d in [0,1]}
for device in [0,1]:torch.cuda.reset_peak_memory_stats(device)
start=time.perf_counter()
if overlap:
copier=_ChunkCopy(torch.device('cuda:0'),torch.device('cuda:1'),0,n)
try:
for offset in range(0,n,256):copier.submit(kv,offset,min(n,offset+256))
keys,values=copier.finish()
finally:copier.close()
else:
keys,values=kv.read_span(0,0,n)
torch.cuda.synchronize(0)
keys,values=move_kv(keys,'cuda:1'),move_kv(values,'cuda:1')
torch.cuda.synchronize(1)
elapsed=time.perf_counter()-start
peaks={str(d):torch.cuda.max_memory_allocated(d)-initial[d] for d in [0,1]}
assert keys.shape==(32,8,n,128) and values.shape==keys.shape
assert bool((keys==0.125).all()) and bool((values==0.5).all())
return elapsed,peaks

try:
for gb in [0.1,0.5,1,2,4]:
n=math.ceil(gb*1e9/(2*32*8*128*2*64))*64
kv=PagedKV(32,1,8,128,torch.bfloat16,'cuda:0',pages=n//64,page_size=64,max_context=n,prefix_cache=False)
assert kv.alloc(0,n) is not None
kv.k.fill_(0.125);kv.v.fill_(0.5)
run(kv,n,False);run(kv,n,True)
for repeat in range(3):
for overlap in ([False,True] if repeat%2==0 else [True,False]):
elapsed,peaks=run(kv,n,overlap)
row=dict(requested_gb=gb,kv_bytes=2*32*8*n*128*2,tokens=n,repeat=repeat,copy_overlap=overlap,
elapsed_s=elapsed,incremental_peak_device_bytes=peaks,parity=True)
result['rows'].append(row);print(json.dumps(row),flush=True)
path.write_text(json.dumps(result,indent=2)+'\n')
kv.release(0);del kv
result['passed']=True
except BaseException:
result['error']=traceback.format_exc()
raise
finally:
path.write_text(json.dumps(result,indent=2)+'\n')

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading