diff --git a/opt_prime/opt_prime/accel/serve/llmd.py b/opt_prime/opt_prime/accel/serve/llmd.py index 52910e2..59e1d2c 100644 --- a/opt_prime/opt_prime/accel/serve/llmd.py +++ b/opt_prime/opt_prime/accel/serve/llmd.py @@ -701,7 +701,11 @@ def text_delta(final=None): send(chunk(usage=response["usage"])) send("[DONE]") - server = ThreadingHTTPServer(("127.0.0.1", port), Handler) + class Server(ThreadingHTTPServer): + # Keep connection bursts queued while the accept thread is briefly delayed. + request_queue_size = 128 + + server = Server(("127.0.0.1", port), Handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() return server, thread diff --git a/opt_prime/tests/test_http_listener_backlog.py b/opt_prime/tests/test_http_listener_backlog.py new file mode 100644 index 0000000..1aba372 --- /dev/null +++ b/opt_prime/tests/test_http_listener_backlog.py @@ -0,0 +1,18 @@ +"""A serving burst must survive a brief pause before the accept loop runs.""" +import socket +import threading + +from opt_prime.accel.serve.llmd import serve_stage + + +def test_sixteen_clients_connect_before_accept_loop_runs(monkeypatch): + monkeypatch.setattr(threading.Thread, 'start', lambda self: None) + server, _ = serve_stage(None, 'decode', 0) + clients = [] + try: + for _ in range(16): + clients.append(socket.create_connection(server.server_address, timeout=0.1)) + finally: + for client in clients: + client.close() + server.server_close()