From 7bba1f5abe85540d7bee0aaec7281841514d6f1b Mon Sep 17 00:00:00 2001 From: gspark Date: Fri, 30 Oct 2026 09:51:37 +0900 Subject: [PATCH] pd: producer failure handling, readiness gate and stage metrics (#238, #246, #249) From feat/249-pd-stage-metrics (#239, #247, #250): SubprocessPrefillWorker failure settling, producer readiness gating /ready and admission, producer stage-duration metrics; only the /ready hunk of llmd.py. Fixes on top: - SubprocessPrefillWorker.close() closes stdin and waits for a clean exit before SIGTERM, then SIGKILL (prefill_server exits on stdin EOF; SIGTERM first skipped SDK cleanup that releases the device). - ProcessPrefillWorker.ready: a dead child now fails readiness and admission is rejected with 503 instead of quarantining the batcher. Co-Authored-By: Claude Opus 5.5 --- opt_prime/opt_prime/accel/pd/pd_batching.py | 185 ++++++++++++----- opt_prime/opt_prime/accel/serve/batched.py | 14 ++ opt_prime/opt_prime/accel/serve/llmd.py | 3 +- opt_prime/tests/test_pd_producer_readiness.py | 167 +++++++++++++++ opt_prime/tests/test_pd_stage_metrics.py | 46 +++++ .../tests/test_subprocess_prefill_failure.py | 191 ++++++++++++++++++ 6 files changed, 560 insertions(+), 46 deletions(-) create mode 100644 opt_prime/tests/test_pd_producer_readiness.py create mode 100644 opt_prime/tests/test_pd_stage_metrics.py create mode 100644 opt_prime/tests/test_subprocess_prefill_failure.py diff --git a/opt_prime/opt_prime/accel/pd/pd_batching.py b/opt_prime/opt_prime/accel/pd/pd_batching.py index 022c7b35..57edc7c6 100644 --- a/opt_prime/opt_prime/accel/pd/pd_batching.py +++ b/opt_prime/opt_prime/accel/pd/pd_batching.py @@ -194,6 +194,11 @@ def __init__(self, factory, out_device): self._recv = threading.Thread(target=self._receive, daemon=True, name="prefill-receiver") self._recv.start() + @property + def ready(self): + """False once the child has died or been closed (the serving readiness gate reads this).""" + return self._futures is not None and self._proc.is_alive() + def submit(self, request, send_from=0, slot=None) -> Future: f = Future() with self._lock: @@ -263,64 +268,149 @@ def __init__(self, python, engine_args, *, slots, layers, kv_heads, head_dim, ma cmd = [python, "-m", "opt_prime.accel.pd.prefill_server", "--shm", self._shm.name, "--slots", str(slots), "--layers", str(layers), "--kv-heads", str(kv_heads), "--head-dim", str(head_dim), "--max-context", str(max_context), *engine_args] - self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, bufsize=1, - env=env) self._json = json self._futures, self._next, self._lock = {}, 0, threading.Lock() + self._write_lock = threading.Lock() + self._failed = None + self._proc = self._recv = None + self._closed = False ready = Future() - self._recv = threading.Thread(target=self._receive, args=(ready,), daemon=True, name="prefill-subprocess") - self._recv.start() - ready.result(timeout=startup_timeout) + try: + self._proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, text=True, + bufsize=1, env=env) + self._recv = threading.Thread(target=self._receive, args=(ready,), daemon=True, + name="prefill-subprocess") + self._recv.start() + ready.result(timeout=startup_timeout) + except BaseException: + self.close() + raise + + @property + def ready(self): + """Terminal producer failure/closure is distinct from a single rejected request.""" + with self._lock: + return (not self._closed and self._failed is None and self._proc.poll() is None + and self._recv.is_alive()) def submit(self, request, send_from=0, slot=None) -> Future: - if slot is None: - raise ValueError("SubprocessPrefillWorker needs the decode lane (slot)") + if slot is None or not 0 <= slot < self.slab.shape[0]: + raise ValueError("SubprocessPrefillWorker needs a valid decode lane (slot)") f = Future() - with self._lock: - key, self._next = self._next, self._next + 1 - self._futures[key] = (f, slot, send_from) s = request.sampling - line = self._json.dumps(dict(id=key, tokens=list(request.input_ids), send_from=send_from, slot=slot, - sampling=dict(temperature=s.temperature, top_p=s.top_p, top_k=s.top_k, seed=s.seed, - stop_token_ids=list(s.stop_token_ids)))) - self._proc.stdin.write(line + "\n") - self._proc.stdin.flush() + write_error = None + # Keep pipe writes ordered without holding the response-map lock across blocking I/O. + with self._write_lock: + with self._lock: + error = self._failed + key, self._next = self._next, self._next + 1 + if error is None: + self._futures[key] = (f, slot, send_from) + if error is not None: + f.set_exception(error) + return f + try: + line = self._json.dumps(dict(id=key, tokens=list(request.input_ids), send_from=send_from, slot=slot, + sampling=dict(temperature=s.temperature, top_p=s.top_p, top_k=s.top_k, + seed=s.seed, stop_token_ids=list(s.stop_token_ids)))) + self._proc.stdin.write(line + "\n") + self._proc.stdin.flush() + except (OSError, ValueError, TypeError) as error: + write_error = RuntimeError(f"prefill subprocess write failed: {error}") + if write_error is not None: + self._fail(write_error) return f - def _receive(self, ready): - for line in self._proc.stdout: - try: - msg = self._json.loads(line) - except ValueError: - continue # SDK log lines on stdout - if msg.get("ready"): - ready.set_result(True) - continue - with self._lock: - f, slot, start = self._futures.pop(msg["id"]) - if "error" in msg: - f.set_exception(RuntimeError(f"prefill subprocess: {msg['error']}")) - continue - n = msg["n"] - kv = torch.from_numpy(self.slab[slot, :, :, :, :n].view("int16")).view(torch.bfloat16) - f.set_result(PrefillOut(msg["token"], start, kv[0], kv[1], msg.get("cached", 0), msg.get("prefill_s", 0.0))) - if not ready.done(): - ready.set_exception(RuntimeError(f"prefill subprocess exited ({self._proc.wait()})")) + def _fail(self, error): with self._lock: - for f, _, _ in self._futures.values(): - f.set_exception(RuntimeError("prefill subprocess exited")) - self._futures.clear() + if self._failed is None: + self._failed = error + pending, self._futures = self._futures, {} + error = self._failed + # Future callbacks may call back into the worker; never invoke them under its lock. + for f, _, _ in pending.values(): + if f.set_running_or_notify_cancel(): + f.set_exception(error) - def close(self): + def _receive(self, ready): try: - self._proc.stdin.close() - self._proc.wait(timeout=60) - finally: - self._shm.close() + for line in self._proc.stdout: + try: + msg = self._json.loads(line) + except ValueError: + continue # SDK log lines on stdout + if not isinstance(msg, dict): + continue + if msg.get("ready"): + if not ready.done(): + ready.set_result(True) + continue + if "id" not in msg: + continue # structured SDK logs + with self._lock: + entry = self._futures.get(msg["id"]) + failed = self._failed + if failed is not None: + return + if entry is None: + raise ValueError("unknown response ID") + f, slot, start = entry + error = None + if "error" in msg: + error = RuntimeError(f"prefill subprocess: {msg['error']}") + else: + n = msg["n"] + if type(n) is not int or not 0 <= n <= self.slab.shape[-2]: + raise ValueError("invalid KV span length") + kv = torch.from_numpy(self.slab[slot, :, :, :, :n].view("int16")).view(torch.bfloat16) + out = PrefillOut(msg["token"], start, kv[0], kv[1], msg.get("cached", 0), + msg.get("prefill_s", 0.0)) + # Keep the request registered until validation succeeds so failure settles it too. + with self._lock: + entry = self._futures.pop(msg["id"], None) + if entry is not None and f.set_running_or_notify_cancel(): + if error is not None: + f.set_exception(error) + else: + f.set_result(out) + raise RuntimeError("prefill subprocess exited") + except Exception as error: + failure = RuntimeError(f"prefill subprocess response failed: {error}") + self._fail(failure) + if not ready.done(): + ready.set_exception(failure) + + def close(self, timeout=5): + import subprocess + if self._closed: + return + self._closed = True + self._fail(RuntimeError("prefill subprocess closed")) + if self._proc is not None: + # Stop the writer before unmapping its slab, including when startup never completed. Closing stdin first + # lets prefill_server leave its request loop and exit normally (the SDK releases the device on exit); + # SIGTERM, then SIGKILL, only for a child that does not exit within `timeout`. try: - self._shm.unlink() - except FileNotFoundError: - pass + self._proc.stdin.close() + except OSError: + pass # a broken pipe must not leak the slab + for stop in (None, self._proc.terminate, self._proc.kill): + if self._proc.poll() is not None: + break + if stop is not None: + stop() + try: + self._proc.wait(timeout=None if stop == self._proc.kill else timeout) + except subprocess.TimeoutExpired: + pass + if self._recv is not None and self._recv is not threading.current_thread(): + self._recv.join(timeout=timeout) + self._proc.stdout.close() + self._shm.close() + try: + self._shm.unlink() + except FileNotFoundError: + pass class PrefillWorkerPool: @@ -334,6 +424,11 @@ def __init__(self, workers): self._load = [0] * len(self.workers) self._lock = threading.Lock() + @property + def ready(self): + # The pool still routes to every member; no implicit removal/retry of a failed producer. + return all(getattr(worker, "ready", True) for worker in self.workers) + def submit(self, request, send_from=0, slot=None) -> Future: with self._lock: i = min(range(len(self.workers)), key=self._load.__getitem__) diff --git a/opt_prime/opt_prime/accel/serve/batched.py b/opt_prime/opt_prime/accel/serve/batched.py index c928220b..67950cee 100644 --- a/opt_prime/opt_prime/accel/serve/batched.py +++ b/opt_prime/opt_prime/accel/serve/batched.py @@ -47,6 +47,11 @@ def extra(lab): return out + self._pd_metrics(lab) self.metrics.extra = extra + @property + def ready(self): + worker = getattr(self.batcher, "worker", None) + return self.state != "poisoned" and getattr(worker, "ready", True) + def _engines(self): return (self.batcher,) @@ -88,6 +93,13 @@ def _pd_metrics(self, lab): if "transferred_tokens" in b.stats: out.append(("airm_pd_transferred_tokens_total", "counter", "Prompt KV tokens received from the prefill side.", [f"airm_pd_transferred_tokens_total{{{lab}}} {b.stats['transferred_tokens']}"])) + # Worker-reported durations of admitted handoffs, not HTTP request latency or overlap. + durations = [f'airm_pd_producer_seconds_total{{{lab},stage="{stage}"}} {b.stats[key]}' + for stage, key in (("prefill", "prefill_s"), ("transfer", "transfer_s")) if key in b.stats] + if durations: + out.append(("airm_pd_producer_seconds_total", "counter", + "Accumulated worker-reported seconds for admitted PD handoffs; excludes failed/cancelled handoffs. " + "Prefill includes KV gathering; transfer excludes destination KV installation.", durations)) return out @staticmethod @@ -138,6 +150,8 @@ def submit_async(self, operation, *args, chat=False, on_token=None): svc, b = self.service, self.service.batcher if svc.state == "poisoned": raise StageError(503, "owner quarantined; restart required", retryable=True) + if not svc.ready: + raise StageError(503, "prefill producer unavailable", retryable=True) _stage, rid, body = args request, stops = svc._request(rid, body, chat) if len(b.by_id) >= self.capacity: diff --git a/opt_prime/opt_prime/accel/serve/llmd.py b/opt_prime/opt_prime/accel/serve/llmd.py index fad81b23..35fd4208 100644 --- a/opt_prime/opt_prime/accel/serve/llmd.py +++ b/opt_prime/opt_prime/accel/serve/llmd.py @@ -525,7 +525,8 @@ def do_GET(self): if self.path not in ("/ready", "/health"): self.reply(404, dict(error="not found")) return - ready = owner.started and owner.service.state != "poisoned" + ready = (owner.started and getattr(owner, "accepting", True) + and owner.service.state != "poisoned" and getattr(owner.service, "ready", True)) self.reply(200 if ready else 503, dict(ready=ready, stage=stage)) def cancel_if_gone(self, rid, state): diff --git a/opt_prime/tests/test_pd_producer_readiness.py b/opt_prime/tests/test_pd_producer_readiness.py new file mode 100644 index 00000000..3fcbf299 --- /dev/null +++ b/opt_prime/tests/test_pd_producer_readiness.py @@ -0,0 +1,167 @@ +"""Real subprocess producer failure must affect HTTP readiness and admission.""" +import json +import sys +import threading +import urllib.error +import urllib.request + +import pytest +import torch +from opt_prime.accel.pd import PDRequest +from opt_prime.accel.pd.pd_batching import ( + PDBatcher, + PrefillWorkerPool, + SubprocessPrefillWorker, +) +from opt_prime.accel.serve.batched import BatchedOwner, BatchedService +from opt_prime.accel.serve.llmd import serve_stage +from test_pd_batching import _greedy, _llama + + +@pytest.fixture +def serving(tmp_path): + torch.set_num_threads(2) + model = _llama() + model.save_pretrained(tmp_path) + workers = [] + + def make_worker(): + worker = SubprocessPrefillWorker( + sys.executable, ["--engine", "torch", "--model", str(tmp_path), "--dtype", "float32"], + slots=1, layers=2, kv_heads=2, head_dim=8, max_context=128, startup_timeout=60, + ) + workers.append(worker) + return worker + + worker = make_worker() + batcher = PDBatcher(model, worker, slots=1, max_context=128, page_size=16) + service = BatchedService(batcher, model="tiny", vocab_size=97, max_output=8, render=str) + owner = BatchedOwner(service) + thread = threading.Thread(target=owner.run, daemon=True) + thread.start() + server, http = serve_stage(owner, "decode", 0) + + def request(path, body=None): + req = urllib.request.Request( + f"http://127.0.0.1:{server.server_port}{path}", + data=None if body is None else json.dumps(body).encode(), + headers={"x-request-id": "readiness-test"}, + ) + try: + with urllib.request.urlopen(req, timeout=15) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read()) + + yield model, batcher, request, make_worker + owner.submit("stop") + thread.join(10) + server.shutdown() + server.server_close() + http.join(5) + for worker in workers: + worker.close() + + +BODY = {"model": "tiny", "prompt": [4, 5, 6, 7], "max_tokens": 1} + + +def kill(worker): + worker._proc.kill() + worker._proc.wait(timeout=5) + worker._recv.join(timeout=5) + assert not worker._recv.is_alive() + + +def test_terminal_worker_is_unready_until_explicit_replacement(serving): + model, batcher, request, make_worker = serving + assert request("/ready")[0] == 200 + assert request("/v1/completions", BODY)[1]["choices"][0]["token_ids"] == _greedy(model, BODY["prompt"], 1) + kill(batcher.worker) + status, body = request("/ready") + assert status == 503 and body["ready"] is False + assert request("/health")[0] == 503 + previous = batcher.stats["prefills"] + status, body = request("/v1/completions", BODY) + assert status == 503 and body["retryable"] is True + assert batcher.idle and not batcher.by_id and batcher.stats["prefills"] == previous + old = batcher.worker + batcher.worker = make_worker() + old.close() + assert request("/ready")[0] == 200 + status, body = request("/v1/completions", BODY) + assert status == 200 and body["choices"][0]["token_ids"] == _greedy(model, BODY["prompt"], 1) + + +def test_closed_worker_rejects_new_admission(serving): + _, batcher, request, _ = serving + batcher.worker.close() + assert request("/ready")[0] == 503 + assert request("/v1/completions", BODY)[0] == 503 + assert batcher.idle and not batcher.by_id + + +def test_pool_failure_does_not_claim_ready_using_another_member(serving): + _, batcher, request, make_worker = serving + failed, healthy = batcher.worker, make_worker() + batcher.worker = PrefillWorkerPool([failed, healthy]) + assert request("/ready")[0] == 200 + kill(failed) + assert request("/ready")[0] == 503 + assert request("/v1/completions", BODY)[0] == 503 + # Removing the failed dependency is explicit; no silent retries or pool reconfiguration. + batcher.worker = PrefillWorkerPool([healthy]) + assert request("/ready")[0] == 200 + assert request("/v1/completions", BODY)[0] == 200 + + +def test_request_error_does_not_mark_live_producer_unready(serving): + _, batcher, request, _ = serving + # The child rejects an out-of-vocabulary embedding lookup, but stays able to serve. + pending = batcher.worker.submit(PDRequest("bad", (98,), 1), slot=0) + with pytest.raises(RuntimeError): + pending.result(timeout=10) + assert request("/ready")[0] == 200 + assert request("/v1/completions", BODY)[0] == 200 + + +def test_dead_process_worker_is_unready_and_rejects_admission(): + """ProcessPrefillWorker: a killed child must fail readiness, not quarantine the batcher on the next admit.""" + import functools + import time + from opt_prime.accel.pd.pd_batching import ProcessPrefillWorker + model = _llama() + worker = ProcessPrefillWorker(functools.partial(time.sleep, 600), out_device="cpu") # never serves + batcher = PDBatcher(model, worker, slots=1, max_context=128, page_size=16) + service = BatchedService(batcher, model="tiny", vocab_size=97, max_output=8, render=str) + owner = BatchedOwner(service) + thread = threading.Thread(target=owner.run, daemon=True) + thread.start() + server, http = serve_stage(owner, "decode", 0) + + def call(path, body=None): + req = urllib.request.Request(f"http://127.0.0.1:{server.server_port}{path}", + data=None if body is None else json.dumps(body).encode(), + headers={"x-request-id": "process-readiness"}) + try: + with urllib.request.urlopen(req, timeout=15) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + return error.code, json.loads(error.read()) + + try: + assert worker.ready and call("/ready")[0] == 200 + worker._proc.kill() + worker._proc.join(timeout=5) + assert not worker.ready + assert call("/ready")[0] == 503 + status, body = call("/v1/completions", BODY) + assert status == 503 and body["retryable"] is True + assert service.state != "poisoned" and batcher.idle and not batcher.by_id + finally: + owner.submit("stop") + thread.join(10) + server.shutdown() + server.server_close() + http.join(5) + worker.close() diff --git a/opt_prime/tests/test_pd_stage_metrics.py b/opt_prime/tests/test_pd_stage_metrics.py new file mode 100644 index 00000000..1abb2f16 --- /dev/null +++ b/opt_prime/tests/test_pd_stage_metrics.py @@ -0,0 +1,46 @@ +"""Completed handoff durations reach public metrics without scrape-side accumulation.""" +import copy + +import pytest + +pytest.importorskip('torch') +from opt_prime.accel.pd import PDRequest +from opt_prime.accel.pd.pd_batching import PDBatcher, PrefillWorker, TorchPrefillEngine +from opt_prime.accel.serve.batched import BatchedService +from test_pd_batching import _llama, _prompts + + +def test_completed_handoff_timings_are_exported_once(): + model = _llama() + engine = TorchPrefillEngine(copy.deepcopy(model), max_context=128, chunk=16, page_size=16, + prefix_cache=False) + original = engine.prefill + + def measured(request, send_from=0): + result = original(request, send_from) + assert result.prefill_s > 0 + # Deterministic worker-reported durations; real prefill/KV and admission remain exercised. + result.prefill_s, result.transfer_s = .25, .125 + return result + + engine.prefill = measured + worker = PrefillWorker(engine) + batcher = PDBatcher(model, worker, slots=1, max_context=128, page_size=16, prefix_cache=False) + service = BatchedService(batcher, model='m', vocab_size=97, max_output=8, render=str) + try: + for i, prompt in enumerate(_prompts(2, 17, 18)): + future = batcher.submit(PDRequest(str(i), prompt, 2)) + batcher.run_until_idle() + assert len(future.result()) == 2 + for _ in range(2): + text = service.metrics.render(*service.gauges()) + assert 'airm_pd_producer_seconds_total{model_name="m",stage="prefill"} 0.5' in text + assert 'airm_pd_producer_seconds_total{model_name="m",stage="transfer"} 0.25' in text + del batcher.stats['transfer_s'] + text = service.metrics.render(*service.gauges()) + assert 'airm_pd_producer_seconds_total{model_name="m",stage="prefill"} 0.5' in text + assert 'stage="transfer"' not in text + del batcher.stats['prefill_s'] + assert 'airm_pd_producer_seconds_total' not in service.metrics.render(*service.gauges()) + finally: + worker.close() diff --git a/opt_prime/tests/test_subprocess_prefill_failure.py b/opt_prime/tests/test_subprocess_prefill_failure.py new file mode 100644 index 00000000..ea6d9bbe --- /dev/null +++ b/opt_prime/tests/test_subprocess_prefill_failure.py @@ -0,0 +1,191 @@ +"""Real pipe/process failures; the tiny child replaces only the expensive engine.""" +import os +import subprocess +import sys +import time +from concurrent.futures import TimeoutError +from multiprocessing import shared_memory + +import pytest +from opt_prime.accel.pd.contracts import PDRequest +from opt_prime.accel.pd.pd_batching import SubprocessPrefillWorker + +CHILD = ''' +import json, os, sys, time +from pathlib import Path +mode = os.environ.get("PREFILL_TEST_MODE", "ok") +if mode == "stubborn": + import signal + signal.signal(signal.SIGTERM, signal.SIG_IGN) +gate = Path(os.environ["PREFILL_TEST_GATE"]) +if mode == "startup_stall": + time.sleep(120) +if mode == "startup_exit": + sys.exit(7) +if mode == "logs": + print('{"compile_start": true}', flush=True) + print('[]', flush=True) +print('{"ready": true}', flush=True) +for line in sys.stdin: + msg = json.loads(line) + if msg["id"] == 0 and mode in ("cancel", "exit", "malformed", "stubborn"): + gate.with_suffix(".seen").touch() + while not gate.exists(): + time.sleep(.005) + if mode == "exit": + sys.exit(8) + if mode == "malformed": + print(json.dumps({"id": msg["id"], "n": "bad"}), flush=True) + continue + print(json.dumps({"id": msg["id"], "n": 1, "token": 17}), flush=True) +# stdin closed: the real prefill_server returns here and the SDK releases the device on interpreter exit +gate.with_suffix(".clean").touch() +''' + + +@pytest.fixture +def children(tmp_path, monkeypatch): + """Observe real allocated resources, and clean up even against the buggy baseline.""" + procs, names = [], [] + popen, shm = subprocess.Popen, shared_memory.SharedMemory + + def spawn(*args, **kwargs): + p = popen(*args, **kwargs) + procs.append(p) + return p + + def allocate(*args, **kwargs): + s = shm(*args, **kwargs) + if kwargs.get("create"): + names.append(s.name) + return s + + monkeypatch.setattr(subprocess, "Popen", spawn) + monkeypatch.setattr(shared_memory, "SharedMemory", allocate) + script = tmp_path / "child" + script.write_text(f"#!{sys.executable}\n" + CHILD) + script.chmod(0o755) + workers = [] + + def make(mode="ok", **kwargs): + env = dict(os.environ, PREFILL_TEST_MODE=mode, PREFILL_TEST_GATE=str(tmp_path / "gate")) + w = SubprocessPrefillWorker(str(script), [], slots=2, layers=1, kv_heads=1, + head_dim=2, max_context=8, env=env, **kwargs) + workers.append(w) + return w + + yield make, procs, names, tmp_path / "gate" + for p in procs: + if p.poll() is None: + p.kill() + p.wait(timeout=5) + for w in workers: + w.close() + for name in names: + try: + s = shm(name=name) + s.close() + s.unlink() + except FileNotFoundError: + pass + + +def request(): + return PDRequest("test", (1, 2), 1) + + +def release(gate): + deadline = time.monotonic() + 3 + while not gate.with_suffix(".seen").exists(): + assert time.monotonic() < deadline, "child did not receive request" + time.sleep(.005) + gate.touch() + + +def test_json_sdk_logs_do_not_break_startup(children): + make, _, _, _ = children + w = make("logs", startup_timeout=1) + assert w.submit(request(), slot=0).result(timeout=2).token == 17 + + +def test_cancelled_response_does_not_strand_next_request(children): + make, _, _, gate = children + w = make("cancel") + first = w.submit(request(), slot=0) + assert first.cancel() + second = w.submit(request(), slot=1) + release(gate) + assert second.result(timeout=2).token == 17 + assert first.cancelled() + + +@pytest.mark.parametrize("mode", ["exit", "malformed"]) +def test_terminal_failure_settles_all_requests_and_rejects_new_ones(children, mode): + make, _, _, gate = children + w = make(mode) + futures = [w.submit(request(), slot=i) for i in range(2)] + release(gate) + for f in futures: + with pytest.raises(RuntimeError): + f.result(timeout=2) + with pytest.raises(RuntimeError): + w.submit(request(), slot=0).result(timeout=2) + replacement = make() + assert replacement.submit(request(), slot=0).result(timeout=2).token == 17 + + +@pytest.mark.parametrize("mode", ["startup_stall", "startup_exit"]) +def test_failed_startup_releases_child_and_shared_memory(children, mode): + make, procs, names, _ = children + with pytest.raises((TimeoutError, RuntimeError)): + make(mode, startup_timeout=.2) + assert procs[-1].poll() is not None + with pytest.raises(FileNotFoundError): + shared_memory.SharedMemory(name=names[-1]) + + +def test_close_is_idempotent_and_rejects_new_work(children): + make, _, names, _ = children + w = make() + w.close() + w.close() + with pytest.raises(FileNotFoundError): + shared_memory.SharedMemory(name=names[-1]) + with pytest.raises(RuntimeError): + w.submit(request(), slot=0).result(timeout=2) + + +def test_close_lets_child_exit_cleanly_on_stdin_eof(children): + """SIGTERM-first would skip the child's own cleanup (SDK device release); closing stdin must come first.""" + make, procs, _, gate = children + w = make() + assert w.submit(request(), slot=0).result(timeout=2).token == 17 + w.close(timeout=5) + assert procs[-1].returncode == 0 + assert gate.with_suffix(".clean").exists() + + +def test_close_kills_unresponsive_worker_and_settles_pending(children): + make, procs, names, _ = children + w = make("stubborn") + pending = w.submit(request(), slot=0) + w.close(timeout=.1) + with pytest.raises(RuntimeError): + pending.result(timeout=2) + assert procs[-1].poll() is not None + assert not w._recv.is_alive() + with pytest.raises(FileNotFoundError): + shared_memory.SharedMemory(name=names[-1]) + + +def test_spawn_failure_does_not_leak_shared_memory(children, monkeypatch): + make, _, names, _ = children + + def fail(*args, **kwargs): + raise OSError("cannot start child") + + monkeypatch.setattr(subprocess, "Popen", fail) + with pytest.raises(OSError, match="cannot start child"): + make() + with pytest.raises(FileNotFoundError): + shared_memory.SharedMemory(name=names[-1])