diff --git a/docs/ARAX_PORT_BASELINE.md b/docs/ARAX_PORT_BASELINE.md index f9916e7..522c4c2 100644 --- a/docs/ARAX_PORT_BASELINE.md +++ b/docs/ARAX_PORT_BASELINE.md @@ -237,7 +237,7 @@ The port matches the status, error code, final envelope and INFO-and-above log, Connect is checked by port-only tests (`test_ARAX_connect.py`), because DEC-7 makes pathfinder parity a non-goal; xCRG (DEC-21) also has `test_xcrg.py` and its own suite in `xcrg_suite/`. The stand-ins are checked by `test_ARAX_query_shepherd.py`. -**Worker:** `workers/arax/worker.py` sends a non-pathfinder query to the remote ARAX service at `settings.arax_url` unless it runs in-process (`parameters.arax_internal: true`, or no `arax_internal` with `ARAX_INTERNAL_DEFAULT=true`). The legacy path: ARAX's response, with Shepherd's provenance, is saved as the answer, and ARAX's log goes to the query's log store; a failed call saves an error response and fails the task with ARAX's status, or 502/504 when ARAX never answered; `stream_progress` is not forwarded, and the stream relays only the final response. In-process, it runs the query in a process-pool child, the way ARAX's non-streaming `/query` does: `ARAXQuery(response_id=...).query_return_message(query)`, then `to_dict()` plus `http_status`. The response is saved under the query's response id, whose URL becomes `envelope.id`. A successful response gets Shepherd's provenance (`infores:shepherd-arax`). ARAX's own error responses are saved as ARAX returns them, with its status, description and log, and the task fails with ARAX's HTTP status. A response ARAX could not serialize (NaN) is a 500, as it is in ARAX. On startup the worker fetches the pathfinder DBs plus curie_to_pmids, ExplainableDTD, FDA drugs and COHD. +**Worker:** `workers/arax/worker.py` sends a non-pathfinder query to the remote ARAX service at `settings.arax_url` unless it runs in-process (`parameters.arax_internal: true`, or no `arax_internal` with `ARAX_INTERNAL_DEFAULT=true`). The legacy path: ARAX's response, with Shepherd's provenance, is saved as the answer, and ARAX's log goes to the query's log store; a 502, 503, 504 or dropped connection is retried with backoff, up to five tries and never past the query's deadline; a failed call saves an error response and fails the task with ARAX's status, or 502/504 when ARAX never answered; `stream_progress` is not forwarded, and the stream relays only the final response. In-process, it runs the query in a process-pool child, the way ARAX's non-streaming `/query` does: `ARAXQuery(response_id=...).query_return_message(query)`, then `to_dict()` plus `http_status`. The response is saved under the query's response id, whose URL becomes `envelope.id`. A successful response gets Shepherd's provenance (`infores:shepherd-arax`). ARAX's own error responses are saved as ARAX returns them, with its status, description and log, and the task fails with ARAX's HTTP status. A response ARAX could not serialize (NaN) is a 500, as it is in ARAX. On startup the worker fetches the pathfinder DBs plus curie_to_pmids, ExplainableDTD, FDA drugs and COHD. **API (`shepherd_server/aras/arax.py`, mounted at `/arax`):** - `POST /query` returns ARAX's own envelope, with ARAX's log and ARAX's HTTP status (API-01). Shepherd's generic `/query` would replace the log with Shepherd's and answer any failure with 500. A response ARAX did not produce (a pathfinder query, or an error response the worker wrote) is finished the generic way. diff --git a/tests/unit/test_arax.py b/tests/unit/test_arax.py index c3d939d..8afc050 100644 --- a/tests/unit/test_arax.py +++ b/tests/unit/test_arax.py @@ -449,6 +449,8 @@ def _handle(request): side_effect=lambda **kwargs: real_client(transport=transport, **kwargs), ) mocker.patch.object(worker.settings, "arax_url", "http://arax.test/query") + # failed calls are retried; don't wait between the tries + mocker.patch.object(worker, "ARAX_RETRY_DELAY", 0) return sent return _setup diff --git a/workers/arax/worker.py b/workers/arax/worker.py index c80ff36..eb9c578 100644 --- a/workers/arax/worker.py +++ b/workers/arax/worker.py @@ -34,6 +34,7 @@ import logging import multiprocessing import os +import random import time import uuid @@ -69,6 +70,7 @@ from shepherd_utils.otel import setup_pool_child_tracer, setup_tracer from shepherd_utils.process_pool import ProcessPoolManager from shepherd_utils.shared import run_task_lifecycle, run_poll_loop +from shepherd_utils.task_deadline import get_deadline from shepherd_utils.trapi import normalize_query_graph, query_parameters # Queue name @@ -104,6 +106,12 @@ # code of its own to pass on: we are a gateway in front of it. BAD_GATEWAY = 502 GATEWAY_TIMEOUT = 504 +# The legacy path: a gateway error or a dropped connection is often a moment of +# overload in front of ARAX, so the call gets a few more tries (see retry_delay) +ARAX_RETRY_STATUS_CODES = frozenset({BAD_GATEWAY, 503, GATEWAY_TIMEOUT}) +ARAX_RETRY_ERRORS = (httpx.NetworkError, httpx.RemoteProtocolError) +ARAX_MAX_ATTEMPTS = 5 +ARAX_RETRY_DELAY = 5 class ARAXServiceError(Exception): @@ -111,12 +119,14 @@ class ARAXServiceError(Exception): Carries ARAX's HTTP status code (the one its ``/query`` would have answered with), so it reaches the task span as ``arax.status_code``, the query's logs - via ``run_task_lifecycle``, and the caller. + via ``run_task_lifecycle``, and the caller. ``retryable`` says whether + another try could go differently. """ - def __init__(self, message: str, status_code: int): + def __init__(self, message: str, status_code: int, retryable: bool = False): super().__init__(message) self.status_code = status_code + self.retryable = retryable def is_pathfinder_query(message): @@ -233,12 +243,35 @@ def body_head(response: httpx.Response) -> str: return f": {body.decode('utf-8', 'replace')}" -async def call_arax(message: dict, logger: logging.Logger) -> dict: +def retry_delay(attempt: int, deadline: float | None) -> float | None: + """Seconds to wait after failed try ``attempt``, or None to give up. + + The wait doubles each time, plus up to half of it again at random, so + queries that failed together don't all come back together. There is no + retry once the tries are used up, or when the wait alone would reach the + query's deadline. + """ + if attempt >= ARAX_MAX_ATTEMPTS: + return None + delay = ARAX_RETRY_DELAY * 2 ** (attempt - 1) + delay += random.uniform(0, delay / 2) + if deadline is not None and time.time() + delay >= deadline: + return None + return delay + + +async def call_arax( + message: dict, logger: logging.Logger, deadline: float | None = None +) -> dict: """POST the query to the remote ARAX service and return its TRAPI response. Raises ``ARAXServiceError`` -- carrying ARAX's own status code -- for anything that isn't a parseable 2xx, so the code reaches the span, the query's logs and the response instead of being logged and dropped. + + A gateway error or a dropped connection is retried with backoff (see + ``retry_delay``), and a retry is cut off at the query's ``deadline``: + nobody is waiting for its answer after that. """ # The remote call is always non-streaming: Shepherd relays nothing from it, # and a streamed body is NDJSON rather than a TRAPI response. @@ -246,6 +279,43 @@ async def call_arax(message: dict, logger: logging.Logger) -> dict: if "submitter" not in message: message["submitter"] = default_submitter() url = settings.arax_url + span = get_current_span() + attempt = 1 + while True: + span.set_attribute("arax.attempts", attempt) + time_left = None + if attempt > 1 and deadline is not None: + time_left = deadline - time.time() + try: + async with asyncio.timeout(time_left): + return await post_to_arax(message, url, logger) + except TimeoutError as e: + # post_to_arax turns its own errors into ARAXServiceError, so this + # is the deadline above + span.set_attribute("arax.status_code", GATEWAY_TIMEOUT) + raise ARAXServiceError( + f"ARAX service at {url} did not respond before the query's " + f"deadline (gave up after {attempt} attempts)", + GATEWAY_TIMEOUT, + ) from e + except ARAXServiceError as e: + delay = retry_delay(attempt, deadline) if e.retryable else None + if delay is None: + if attempt == 1: + raise + raise ARAXServiceError( + f"{e} (gave up after {attempt} attempts)", e.status_code + ) from e + logger.warning( + f"ARAX attempt {attempt} of {ARAX_MAX_ATTEMPTS} failed, " + f"retrying in {delay:.0f}s: {e}" + ) + await asyncio.sleep(delay) + attempt += 1 + + +async def post_to_arax(message: dict, url: str, logger: logging.Logger) -> dict: + """One try at the remote ARAX service; see ``call_arax``.""" logger.info(f"Sending the query to the ARAX service at {url}") span = get_current_span() try: @@ -267,6 +337,7 @@ async def call_arax(message: dict, logger: logging.Logger) -> dict: f"Error occurred calling ARAX service at {url}: " f"{type(e).__name__}: {e}", BAD_GATEWAY, + retryable=isinstance(e, ARAX_RETRY_ERRORS), ) from e status_code = response.status_code @@ -276,6 +347,7 @@ async def call_arax(message: dict, logger: logging.Logger) -> dict: raise ARAXServiceError( f"ARAX service at {url} returned HTTP {status_code}{body_head(response)}", status_code, + retryable=status_code in ARAX_RETRY_STATUS_CODES, ) try: result = response.json() @@ -297,12 +369,15 @@ async def call_arax(message: dict, logger: logging.Logger) -> dict: async def run_legacy_arax( - message: dict, response_id: str, logger: logging.Logger + message: dict, + response_id: str, + logger: logging.Logger, + deadline: float | None = None, ) -> None: """The legacy path: answer the query with the remote ARAX service.""" try: try: - result = await call_arax(message, logger) + result = await call_arax(message, logger, deadline) except ARAXServiceError as e: # Leave the caller a TRAPI response that says what happened before # letting the failure reach run_task_lifecycle, which routes the @@ -653,7 +728,7 @@ async def arax(task, logger: logging.Logger, loop=None, pool=None): response_id = task[1]["response_id"] if not uses_internal_workers(message): span.set_attribute("arax.path", "legacy") - await run_legacy_arax(message, response_id, logger) + await run_legacy_arax(message, response_id, logger, get_deadline(task[1])) task[1]["workflow"] = json.dumps([{"id": "arax"}]) return span.set_attribute("arax.path", "internal")