From 93f0374bf663f65a71ebcccc99bf9bf11ec94469 Mon Sep 17 00:00:00 2001 From: gspark Date: Wed, 4 Nov 2026 02:02:33 +0900 Subject: [PATCH] serve: queue HTTP connection bursts before accept (backlog 128) Python 3.10's ThreadingHTTPServer listens with backlog 5, so 16 clients arriving before the accept loop runs get connection resets. Co-Authored-By: Claude Opus 5.5 --- opt_prime/opt_prime/accel/serve/llmd.py | 6 +++++- opt_prime/tests/test_http_listener_backlog.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 opt_prime/tests/test_http_listener_backlog.py diff --git a/opt_prime/opt_prime/accel/serve/llmd.py b/opt_prime/opt_prime/accel/serve/llmd.py index 52910e21..59e1d2c0 100644 --- a/opt_prime/opt_prime/accel/serve/llmd.py +++ b/opt_prime/opt_prime/accel/serve/llmd.py @@ -701,7 +701,11 @@ def text_delta(final=None): send(chunk(usage=response["usage"])) send("[DONE]") - server = ThreadingHTTPServer(("127.0.0.1", port), Handler) + class Server(ThreadingHTTPServer): + # Keep connection bursts queued while the accept thread is briefly delayed. + request_queue_size = 128 + + server = Server(("127.0.0.1", port), Handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() return server, thread diff --git a/opt_prime/tests/test_http_listener_backlog.py b/opt_prime/tests/test_http_listener_backlog.py new file mode 100644 index 00000000..1aba3726 --- /dev/null +++ b/opt_prime/tests/test_http_listener_backlog.py @@ -0,0 +1,18 @@ +"""A serving burst must survive a brief pause before the accept loop runs.""" +import socket +import threading + +from opt_prime.accel.serve.llmd import serve_stage + + +def test_sixteen_clients_connect_before_accept_loop_runs(monkeypatch): + monkeypatch.setattr(threading.Thread, 'start', lambda self: None) + server, _ = serve_stage(None, 'decode', 0) + clients = [] + try: + for _ in range(16): + clients.append(socket.create_connection(server.server_address, timeout=0.1)) + finally: + for client in clients: + client.close() + server.server_close()