Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/ARAX_PORT_BASELINE.md
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,7 @@ The port matches the status, error code, final envelope and INFO-and-above log,

Connect is checked by port-only tests (`test_ARAX_connect.py`), because DEC-7 makes pathfinder parity a non-goal; xCRG (DEC-21) also has `test_xcrg.py` and its own suite in `xcrg_suite/`. The stand-ins are checked by `test_ARAX_query_shepherd.py`.

**Worker:** `workers/arax/worker.py` sends a non-pathfinder query to the remote ARAX service at `settings.arax_url` unless it runs in-process (`parameters.arax_internal: true`, or no `arax_internal` with `ARAX_INTERNAL_DEFAULT=true`). The legacy path: ARAX's response, with Shepherd's provenance, is saved as the answer, and ARAX's log goes to the query's log store; a failed call saves an error response and fails the task with ARAX's status, or 502/504 when ARAX never answered; `stream_progress` is not forwarded, and the stream relays only the final response. In-process, it runs the query in a process-pool child, the way ARAX's non-streaming `/query` does: `ARAXQuery(response_id=...).query_return_message(query)`, then `to_dict()` plus `http_status`. The response is saved under the query's response id, whose URL becomes `envelope.id`. A successful response gets Shepherd's provenance (`infores:shepherd-arax`). ARAX's own error responses are saved as ARAX returns them, with its status, description and log, and the task fails with ARAX's HTTP status. A response ARAX could not serialize (NaN) is a 500, as it is in ARAX. On startup the worker fetches the pathfinder DBs plus curie_to_pmids, ExplainableDTD, FDA drugs and COHD.
**Worker:** `workers/arax/worker.py` sends a non-pathfinder query to the remote ARAX service at `settings.arax_url` unless it runs in-process (`parameters.arax_internal: true`, or no `arax_internal` with `ARAX_INTERNAL_DEFAULT=true`). The legacy path: ARAX's response, with Shepherd's provenance, is saved as the answer, and ARAX's log goes to the query's log store; a 502, 503, 504 or dropped connection is retried with backoff, up to five tries and never past the query's deadline; a failed call saves an error response and fails the task with ARAX's status, or 502/504 when ARAX never answered; `stream_progress` is not forwarded, and the stream relays only the final response. In-process, it runs the query in a process-pool child, the way ARAX's non-streaming `/query` does: `ARAXQuery(response_id=...).query_return_message(query)`, then `to_dict()` plus `http_status`. The response is saved under the query's response id, whose URL becomes `envelope.id`. A successful response gets Shepherd's provenance (`infores:shepherd-arax`). ARAX's own error responses are saved as ARAX returns them, with its status, description and log, and the task fails with ARAX's HTTP status. A response ARAX could not serialize (NaN) is a 500, as it is in ARAX. On startup the worker fetches the pathfinder DBs plus curie_to_pmids, ExplainableDTD, FDA drugs and COHD.

**API (`shepherd_server/aras/arax.py`, mounted at `/arax`):**
- `POST /query` returns ARAX's own envelope, with ARAX's log and ARAX's HTTP status (API-01). Shepherd's generic `/query` would replace the log with Shepherd's and answer any failure with 500. A response ARAX did not produce (a pathfinder query, or an error response the worker wrote) is finished the generic way.
Expand Down
2 changes: 2 additions & 0 deletions tests/unit/test_arax.py
Original file line number Diff line number Diff line change
Expand Up @@ -449,6 +449,8 @@ def _handle(request):
side_effect=lambda **kwargs: real_client(transport=transport, **kwargs),
)
mocker.patch.object(worker.settings, "arax_url", "http://arax.test/query")
# failed calls are retried; don't wait between the tries
mocker.patch.object(worker, "ARAX_RETRY_DELAY", 0)
return sent

return _setup
Expand Down
87 changes: 81 additions & 6 deletions workers/arax/worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import logging
import multiprocessing
import os
import random
import time
import uuid

Expand Down Expand Up @@ -69,6 +70,7 @@
from shepherd_utils.otel import setup_pool_child_tracer, setup_tracer
from shepherd_utils.process_pool import ProcessPoolManager
from shepherd_utils.shared import run_task_lifecycle, run_poll_loop
from shepherd_utils.task_deadline import get_deadline
from shepherd_utils.trapi import normalize_query_graph, query_parameters

# Queue name
Expand Down Expand Up @@ -104,19 +106,27 @@
# code of its own to pass on: we are a gateway in front of it.
BAD_GATEWAY = 502
GATEWAY_TIMEOUT = 504
# The legacy path: a gateway error or a dropped connection is often a moment of
# overload in front of ARAX, so the call gets a few more tries (see retry_delay)
ARAX_RETRY_STATUS_CODES = frozenset({BAD_GATEWAY, 503, GATEWAY_TIMEOUT})
ARAX_RETRY_ERRORS = (httpx.NetworkError, httpx.RemoteProtocolError)
ARAX_MAX_ATTEMPTS = 5
ARAX_RETRY_DELAY = 5


class ARAXServiceError(Exception):
"""ARAX did not produce a successful TRAPI response.

Carries ARAX's HTTP status code (the one its ``/query`` would have answered
with), so it reaches the task span as ``arax.status_code``, the query's logs
via ``run_task_lifecycle``, and the caller.
via ``run_task_lifecycle``, and the caller. ``retryable`` says whether
another try could go differently.
"""

def __init__(self, message: str, status_code: int):
def __init__(self, message: str, status_code: int, retryable: bool = False):
super().__init__(message)
self.status_code = status_code
self.retryable = retryable


def is_pathfinder_query(message):
Expand Down Expand Up @@ -233,19 +243,79 @@ def body_head(response: httpx.Response) -> str:
return f": {body.decode('utf-8', 'replace')}"


async def call_arax(message: dict, logger: logging.Logger) -> dict:
def retry_delay(attempt: int, deadline: float | None) -> float | None:
"""Seconds to wait after failed try ``attempt``, or None to give up.

The wait doubles each time, plus up to half of it again at random, so
queries that failed together don't all come back together. There is no
retry once the tries are used up, or when the wait alone would reach the
query's deadline.
"""
if attempt >= ARAX_MAX_ATTEMPTS:
return None
delay = ARAX_RETRY_DELAY * 2 ** (attempt - 1)
delay += random.uniform(0, delay / 2)
if deadline is not None and time.time() + delay >= deadline:
return None
return delay


async def call_arax(
message: dict, logger: logging.Logger, deadline: float | None = None
) -> dict:
"""POST the query to the remote ARAX service and return its TRAPI response.

Raises ``ARAXServiceError`` -- carrying ARAX's own status code -- for
anything that isn't a parseable 2xx, so the code reaches the span, the
query's logs and the response instead of being logged and dropped.

A gateway error or a dropped connection is retried with backoff (see
``retry_delay``), and a retry is cut off at the query's ``deadline``:
nobody is waiting for its answer after that.
"""
# The remote call is always non-streaming: Shepherd relays nothing from it,
# and a streamed body is NDJSON rather than a TRAPI response.
message = {k: v for k, v in message.items() if k != "stream_progress"}
if "submitter" not in message:
message["submitter"] = default_submitter()
url = settings.arax_url
span = get_current_span()
attempt = 1
while True:
span.set_attribute("arax.attempts", attempt)
time_left = None
if attempt > 1 and deadline is not None:
time_left = deadline - time.time()
try:
async with asyncio.timeout(time_left):
return await post_to_arax(message, url, logger)
except TimeoutError as e:
# post_to_arax turns its own errors into ARAXServiceError, so this
# is the deadline above
span.set_attribute("arax.status_code", GATEWAY_TIMEOUT)
raise ARAXServiceError(
f"ARAX service at {url} did not respond before the query's "
f"deadline (gave up after {attempt} attempts)",
GATEWAY_TIMEOUT,
) from e
except ARAXServiceError as e:
delay = retry_delay(attempt, deadline) if e.retryable else None
if delay is None:
if attempt == 1:
raise
raise ARAXServiceError(
f"{e} (gave up after {attempt} attempts)", e.status_code
) from e
logger.warning(
f"ARAX attempt {attempt} of {ARAX_MAX_ATTEMPTS} failed, "
f"retrying in {delay:.0f}s: {e}"
)
await asyncio.sleep(delay)
attempt += 1


async def post_to_arax(message: dict, url: str, logger: logging.Logger) -> dict:
"""One try at the remote ARAX service; see ``call_arax``."""
logger.info(f"Sending the query to the ARAX service at {url}")
span = get_current_span()
try:
Expand All @@ -267,6 +337,7 @@ async def call_arax(message: dict, logger: logging.Logger) -> dict:
f"Error occurred calling ARAX service at {url}: "
f"{type(e).__name__}: {e}",
BAD_GATEWAY,
retryable=isinstance(e, ARAX_RETRY_ERRORS),
) from e

status_code = response.status_code
Expand All @@ -276,6 +347,7 @@ async def call_arax(message: dict, logger: logging.Logger) -> dict:
raise ARAXServiceError(
f"ARAX service at {url} returned HTTP {status_code}{body_head(response)}",
status_code,
retryable=status_code in ARAX_RETRY_STATUS_CODES,
)
try:
result = response.json()
Expand All @@ -297,12 +369,15 @@ async def call_arax(message: dict, logger: logging.Logger) -> dict:


async def run_legacy_arax(
message: dict, response_id: str, logger: logging.Logger
message: dict,
response_id: str,
logger: logging.Logger,
deadline: float | None = None,
) -> None:
"""The legacy path: answer the query with the remote ARAX service."""
try:
try:
result = await call_arax(message, logger)
result = await call_arax(message, logger, deadline)
except ARAXServiceError as e:
# Leave the caller a TRAPI response that says what happened before
# letting the failure reach run_task_lifecycle, which routes the
Expand Down Expand Up @@ -653,7 +728,7 @@ async def arax(task, logger: logging.Logger, loop=None, pool=None):
response_id = task[1]["response_id"]
if not uses_internal_workers(message):
span.set_attribute("arax.path", "legacy")
await run_legacy_arax(message, response_id, logger)
await run_legacy_arax(message, response_id, logger, get_deadline(task[1]))
task[1]["workflow"] = json.dumps([{"id": "arax"}])
return
span.set_attribute("arax.path", "internal")
Expand Down
Loading