diff --git a/opt_prime/examples/pd/http_matrix/README.md b/opt_prime/examples/pd/http_matrix/README.md index 038d7625..b260e35b 100644 --- a/opt_prime/examples/pd/http_matrix/README.md +++ b/opt_prime/examples/pd/http_matrix/README.md @@ -45,3 +45,9 @@ PYTHONPATH=opt_prime python opt_prime/examples/pd/http_matrix/validate.py \ ``` Run from the repository root. Omit the producer arguments for native control. `passed` means lifecycle/count assertions passed; concurrent/serial token equality is reported separately, including failed numerical comparisons. See gpu-live-pd240 for the complete default-policy, FP32 and matching-policy results. + +### Terminal producer availability + +Subprocess producers expose a boolean `ready` property; pools require every configured member that implements it to be ready. Batched `/ready` and `/health` return503 after a known terminal failure/close, and new completions receive retryable503 before admission. A healthy replacement at an idle boundary restores availability without poisoning the decoder. Individual request errors do not make a still-running producer unready. + +No pool member is silently removed and no request is automatically retried. Workers without this optional property retain prior behavior. Existing requests keep their existing completion/error handling. See producer-readiness246 for the real subprocess/HTTP reproduction and regression results. diff --git a/opt_prime/examples/pd/http_matrix/STATUS.md b/opt_prime/examples/pd/http_matrix/STATUS.md index cfc3154f..07288f3b 100644 --- a/opt_prime/examples/pd/http_matrix/STATUS.md +++ b/opt_prime/examples/pd/http_matrix/STATUS.md @@ -29,3 +29,7 @@ New opt-in controls disable producer prefix reuse and select producer chunk size ## Live GPU PD matrix (#240) All four native/PD BF16 lifecycle configurations pass across128/2048/8192tokens. Default PD has35/48concurrent equality versus native48/48, and11/12native-PD serial equality. FP32 restores all divergent models; BF16 explicit cache-off/chunk256 controls#241 restore four-model48/48concurrent and12/12serial equality. Original divergences and validator module-identity failure are retained in evidence240. The self-contained HTTP-candidate loader was fixed and Qwen0.6B rerun passes. No matched speedup or compute/copy overlap claim. + +## Producer-aware readiness (#246) + +A real terminated subprocess producer previously left /ready200 and new requests failed500. The service now reports503 and rejects new admissions with retryable503 until explicit healthy replacement; single request errors preserve readiness. Pools require all configured members because routing still includes them.30focused tests and full468passed/15skipped/135subtests. Legacy workers without the optional readiness property retain their prior behavior. Historical #244 hardware evidence predates this new post-failure503 contract; no automatic restart claim. diff --git a/opt_prime/examples/pd/http_matrix/evidence246/events.jsonl b/opt_prime/examples/pd/http_matrix/evidence246/events.jsonl new file mode 100644 index 00000000..6427b676 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence246/events.jsonl @@ -0,0 +1,2 @@ +{"id": "40c71a4351714f6e9533c71198fef82c", "ts": "2026-10-24T00:29:43+09:00", "task": "PD producer readiness (aicomp #246)", "result": "Confirmed readiness bug with real CPU subprocess:dead producer stillready200/newrequest500. Candidate ready503/newrequest503retryable,explicit replacement restores200/output. Regression3failed->4passed;30focused tests pass.", "next": "Complete full suite and publish fix; keep prior hardware-fault evidence pinned to its500behavior and plan corrected503 integration.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [{"path": "analysis/serving-matrix-201/readiness246-fixed.json", "size": 308, "sha256": "ae8a7873aa27ac40"}], "experiments": ["experiments/2026-10-24-pd-producer-readiness246.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} +{"id": "97c098f411f2418bbdfd2d1558bd63b7", "ts": "2026-10-24T00:33:26+09:00", "task": "PD producer readiness (aicomp #246)", "result": "Final readiness/admission fix:real subprocess200->503->200 across failure/replacement; new requests503retryable before admission.30focused tests; full468passed/15skipped/135subtests. Per-request errors retain readiness.", "next": "Publish fix; proceed to controlled native/DP/PD latency-throughput measurements with matched producer policies.", "state": "active", "done": false, "blocker": "", "metrics": {}, "params": {}, "artifacts": [{"path": "analysis/serving-matrix-201/readiness246-fixed.json", "size": 308, "sha256": "ae8a7873aa27ac40"}], "experiments": ["experiments/2026-10-24-pd-producer-readiness246.md"], "labels": ["bug"], "review": true, "todos": [], "note": "", "agent": "codex", "host": "xeon-max", "branch": "main", "commit": "351d39d-dirty"} diff --git a/opt_prime/examples/pd/http_matrix/evidence246/readiness-baseline.json b/opt_prime/examples/pd/http_matrix/evidence246/readiness-baseline.json new file mode 100644 index 00000000..29bb5853 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence246/readiness-baseline.json @@ -0,0 +1 @@ +{"before": {"status": 200, "body": {"ready": true, "stage": "decode"}}, "after_worker_exit": {"status": 200, "body": {"ready": true, "stage": "decode"}}, "new_request": {"status": 500, "body": {"error": "engine failed; owner retained", "retryable": false}}, "terminal_failure": true, "service_state": "idle"} diff --git a/opt_prime/examples/pd/http_matrix/evidence246/readiness246-fixed.json b/opt_prime/examples/pd/http_matrix/evidence246/readiness246-fixed.json new file mode 100644 index 00000000..d34b58d3 --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/evidence246/readiness246-fixed.json @@ -0,0 +1 @@ +{"before": {"status": 200, "body": {"ready": true, "stage": "decode"}}, "after_worker_exit": {"status": 503, "body": {"ready": false, "stage": "decode"}}, "new_request": {"status": 503, "body": {"error": "prefill producer unavailable", "retryable": true}}, "terminal_failure": true, "service_state": "idle"} diff --git a/opt_prime/examples/pd/http_matrix/experiments/2026-10-24-pd-producer-readiness246.md b/opt_prime/examples/pd/http_matrix/experiments/2026-10-24-pd-producer-readiness246.md new file mode 100644 index 00000000..cecf1d3c --- /dev/null +++ b/opt_prime/examples/pd/http_matrix/experiments/2026-10-24-pd-producer-readiness246.md @@ -0,0 +1,36 @@ +# Terminal producer failure in HTTP readiness and admission + +- **Date**: 2026-10-24 +- **Task**: aicomp #246 +- **Host / device**: CPU development host, real Torch subprocess and real HTTP server; no accelerator reset +- **Commit**: ea09aca baseline; fix/246-pd-producer-readiness candidate + +## Question + +Does the service reject new work and stop advertising readiness when a required subprocess prefill producer has terminally failed? + +## Setup + +Actual tiny Llama checkpoint, SubprocessPrefillWorker and PDBatcher under BatchedOwner/HTTP. Kill only the test child and await its receiver termination. Check readiness and request admission, then explicitly replace the worker at an idle boundary. Also test a closed producer, a pool with one failed required member, and a per-request embedding error that must not make a live producer unavailable. + +## Command + +```bash +PYTHONPATH=opt_prime python -m pytest -q opt_prime/tests/test_pd_producer_readiness.py +PYTHONPATH=opt_prime python -m pytest -q opt_prime/tests/test_pd_producer_readiness.py opt_prime/tests/test_subprocess_prefill_failure.py opt_prime/tests/test_serve_batched.py opt_prime/tests/test_pd_llmd.py opt_prime/tests/test_pd_http_cancel.py +# From opt_prime/: +PYTHONPATH=. python -m pytest -q tests +``` + +Independent exact local probe:/tmp/probe_pd_readiness.py, run with PYTHONPATH pointing to the baseline/candidate package. JSON under analysis/serving-matrix-201/readiness-baseline.json and readiness246-fixed.json; selected copies accompany the change. + +## Results + +- Baseline: /ready200 before and after confirmed terminal worker failure; new completion500/non-retryable, service state stillidle. Regression set3failed/1passed. +- Candidate: /ready503 after failure, new completion503/retryable before scheduling; explicit replacement restores200 and correct output. Closed worker and failed pool member likewise block admission; request-only error leaves readiness200. +- Focused serving/PD suite30passed,1existing flex_attention warning,27.02s. +- Final full suite468passed,15skipped,135subtests passed,1existing flex_attention warning,151.40s. + +## Conclusion + +Known subprocess/pool terminal availability is now reflected consistently by readiness and admission. Already-admitted work keeps its existing error/drain handling; the service is not globally poisoned just because a producer can be replaced. Pools conservatively require all configured members because their routing still includes each member. No silent rerouting/retry or automatic restart. Legacy/custom workers without the optional readiness property keep prior behavior. The hardware fault record#244 predates this change and retains its later-request500; the corrected service now rejects new requests with503 instead. diff --git a/opt_prime/opt_prime/accel/pd/pd_batching.py b/opt_prime/opt_prime/accel/pd/pd_batching.py index ed56608c..da2a593c 100644 --- a/opt_prime/opt_prime/accel/pd/pd_batching.py +++ b/opt_prime/opt_prime/accel/pd/pd_batching.py @@ -281,6 +281,13 @@ def __init__(self, python, engine_args, *, slots, layers, kv_heads, head_dim, ma self.close() raise + @property + def ready(self): + """Terminal producer failure/closure is distinct from a single rejected request.""" + with self._lock: + return (not self._closed and self._failed is None and self._proc.poll() is None + and self._recv.is_alive()) + def submit(self, request, send_from=0, slot=None) -> Future: if slot is None or not 0 <= slot < self.slab.shape[0]: raise ValueError("SubprocessPrefillWorker needs a valid decode lane (slot)") @@ -408,6 +415,11 @@ def __init__(self, workers): self._load = [0] * len(self.workers) self._lock = threading.Lock() + @property + def ready(self): + # The pool still routes to every member; no implicit removal/retry of a failed producer. + return all(getattr(worker, "ready", True) for worker in self.workers) + def submit(self, request, send_from=0, slot=None) -> Future: with self._lock: i = min(range(len(self.workers)), key=self._load.__getitem__) diff --git a/opt_prime/opt_prime/accel/serve/batched.py b/opt_prime/opt_prime/accel/serve/batched.py index c928220b..1a5bb682 100644 --- a/opt_prime/opt_prime/accel/serve/batched.py +++ b/opt_prime/opt_prime/accel/serve/batched.py @@ -47,6 +47,11 @@ def extra(lab): return out + self._pd_metrics(lab) self.metrics.extra = extra + @property + def ready(self): + worker = getattr(self.batcher, "worker", None) + return self.state != "poisoned" and getattr(worker, "ready", True) + def _engines(self): return (self.batcher,) @@ -138,6 +143,8 @@ def submit_async(self, operation, *args, chat=False, on_token=None): svc, b = self.service, self.service.batcher if svc.state == "poisoned": raise StageError(503, "owner quarantined; restart required", retryable=True) + if not svc.ready: + raise StageError(503, "prefill producer unavailable", retryable=True) _stage, rid, body = args request, stops = svc._request(rid, body, chat) if len(b.by_id) >= self.capacity: diff --git a/opt_prime/opt_prime/accel/serve/llmd.py b/opt_prime/opt_prime/accel/serve/llmd.py index fad81b23..35fd4208 100644 --- a/opt_prime/opt_prime/accel/serve/llmd.py +++ b/opt_prime/opt_prime/accel/serve/llmd.py @@ -525,7 +525,8 @@ def do_GET(self): if self.path not in ("/ready", "/health"): self.reply(404, dict(error="not found")) return - ready = owner.started and owner.service.state != "poisoned" + ready = (owner.started and getattr(owner, "accepting", True) + and owner.service.state != "poisoned" and getattr(owner.service, "ready", True)) self.reply(200 if ready else 503, dict(ready=ready, stage=stage)) def cancel_if_gone(self, rid, state): diff --git a/opt_prime/tests/test_pd_producer_readiness.py b/opt_prime/tests/test_pd_producer_readiness.py new file mode 100644 index 00000000..04d55add --- /dev/null +++ b/opt_prime/tests/test_pd_producer_readiness.py @@ -0,0 +1,125 @@ +"""Real subprocess producer failure must affect HTTP readiness and admission.""" +import json +import sys +import threading +import urllib.error +import urllib.request + +import pytest +import torch +from opt_prime.accel.pd import PDRequest +from opt_prime.accel.pd.pd_batching import ( + PDBatcher, + PrefillWorkerPool, + SubprocessPrefillWorker, +) +from opt_prime.accel.serve.batched import BatchedOwner, BatchedService +from opt_prime.accel.serve.llmd import serve_stage +from test_pd_batching import _greedy, _llama + + +@pytest.fixture +def serving(tmp_path): + torch.set_num_threads(2) + model = _llama() + model.save_pretrained(tmp_path) + workers = [] + + def make_worker(): + worker = SubprocessPrefillWorker( + sys.executable, ["--engine", "torch", "--model", str(tmp_path), "--dtype", "float32"], + slots=1, layers=2, kv_heads=2, head_dim=8, max_context=128, startup_timeout=60, + ) + workers.append(worker) + return worker + + worker = make_worker() + batcher = PDBatcher(model, worker, slots=1, max_context=128, page_size=16) + service = BatchedService(batcher, model="tiny", vocab_size=97, max_output=8, render=str) + owner = BatchedOwner(service) + thread = threading.Thread(target=owner.run, daemon=True) + thread.start() + server, http = serve_stage(owner, "decode", 0) + + def request(path, body=None): + req = urllib.request.Request( + f"http://127.0.0.1:{server.server_port}{path}", + data=None if body is None else json.dumps(body).encode(), + headers={"x-request-id": "readiness-test"}, + ) + try: + with urllib.request.urlopen(req, timeout=15) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read()) + + yield model, batcher, request, make_worker + owner.submit("stop") + thread.join(10) + server.shutdown() + server.server_close() + http.join(5) + for worker in workers: + worker.close() + + +BODY = {"model": "tiny", "prompt": [4, 5, 6, 7], "max_tokens": 1} + + +def kill(worker): + worker._proc.kill() + worker._proc.wait(timeout=5) + worker._recv.join(timeout=5) + assert not worker._recv.is_alive() + + +def test_terminal_worker_is_unready_until_explicit_replacement(serving): + model, batcher, request, make_worker = serving + assert request("/ready")[0] == 200 + assert request("/v1/completions", BODY)[1]["choices"][0]["token_ids"] == _greedy(model, BODY["prompt"], 1) + kill(batcher.worker) + status, body = request("/ready") + assert status == 503 and body["ready"] is False + assert request("/health")[0] == 503 + previous = batcher.stats["prefills"] + status, body = request("/v1/completions", BODY) + assert status == 503 and body["retryable"] is True + assert batcher.idle and not batcher.by_id and batcher.stats["prefills"] == previous + old = batcher.worker + batcher.worker = make_worker() + old.close() + assert request("/ready")[0] == 200 + status, body = request("/v1/completions", BODY) + assert status == 200 and body["choices"][0]["token_ids"] == _greedy(model, BODY["prompt"], 1) + + +def test_closed_worker_rejects_new_admission(serving): + _, batcher, request, _ = serving + batcher.worker.close() + assert request("/ready")[0] == 503 + assert request("/v1/completions", BODY)[0] == 503 + assert batcher.idle and not batcher.by_id + + +def test_pool_failure_does_not_claim_ready_using_another_member(serving): + _, batcher, request, make_worker = serving + failed, healthy = batcher.worker, make_worker() + batcher.worker = PrefillWorkerPool([failed, healthy]) + assert request("/ready")[0] == 200 + kill(failed) + assert request("/ready")[0] == 503 + assert request("/v1/completions", BODY)[0] == 503 + # Removing the failed dependency is explicit; no silent retries or pool reconfiguration. + batcher.worker = PrefillWorkerPool([healthy]) + assert request("/ready")[0] == 200 + assert request("/v1/completions", BODY)[0] == 200 + + +def test_request_error_does_not_mark_live_producer_unready(serving): + _, batcher, request, _ = serving + # The child rejects an out-of-vocabulary embedding lookup, but stays able to serve. + pending = batcher.worker.submit(PDRequest("bad", (98,), 1), slot=0) + with pytest.raises(RuntimeError): + pending.result(timeout=10) + assert request("/ready")[0] == 200 + assert request("/v1/completions", BODY)[0] == 200