From 7b8dc1ae4b080d525ce4e13dff1e29a4711106b9 Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Wed, 30 Sep 2026 10:43:57 -0700 Subject: [PATCH 1/8] Add experimental explainable query workflows and evidence replay --- README.md | 17 + TCT/__init__.py | 5 + TCT/interfaces/WORKFLOWS.md | 170 ++++++ TCT/interfaces/tools.py | 77 +++ TCT/workflow.py | 685 ++++++++++++++++++++++++ examples/workflows/offline_pathway.json | 57 ++ tests/test_mcp_contract.py | 51 +- tests/test_tools.py | 5 +- tests/test_workflow.py | 522 ++++++++++++++++++ 9 files changed, 1586 insertions(+), 3 deletions(-) create mode 100644 TCT/interfaces/WORKFLOWS.md create mode 100644 TCT/workflow.py create mode 100644 examples/workflows/offline_pathway.json create mode 100644 tests/test_workflow.py diff --git a/README.md b/README.md index 5114cc2..6ee6b44 100644 --- a/README.md +++ b/README.md @@ -120,6 +120,22 @@ uv sync --extra mcp --extra langfuse ## Python, CLI, and MCP interfaces +### Experimental explainable query workflows + +TCT can execute explicit query workflows and return a portable evidence bundle: +the recipe, complete provider responses, filter decisions, and row lineage through +joins. `validate_query_workflow`, `run_query_workflow`, and +`replay_query_workflow` expose this through MCP and the CLI; Python callers use +`TCT.validate_workflow`, `TCT.run_workflow`, and `TCT.replay_workflow`. +Offline replay recomputes transformations using saved responses. Live reruns are +explicit and may change when remote data changes. Existing tools keep their +return formats and are not automatically instrumented as workflows. + +See [workflow recipes and limitations](TCT/interfaces/WORKFLOWS.md) and the +[synthetic offline pathway example](examples/workflows/offline_pathway.json). + +### Shared operations + TCT exposes one curated set of well-documented operations through three interfaces: @@ -248,6 +264,7 @@ Python names in snake_case. | MetaKG extension | `add-custom-api-to-metakg`, `add-plover-apis-to-metakg` | | TRAPI query preparation and execution | `optimize-query-for-api`, `query-knowledge-provider`, `parallel-query-apis` | | Graph finding | `neighborhood-finder`, `path-finder` | +| Explainable workflows | `validate-query-workflow`, `run-query-workflow`, `replay-query-workflow` | | Legacy compatibility | `trapi-query-endpoint` | `trapi-query-endpoint` preserves the existing public tool contract but is a diff --git a/TCT/__init__.py b/TCT/__init__.py index c52ca44..e47aee8 100644 --- a/TCT/__init__.py +++ b/TCT/__init__.py @@ -2,6 +2,11 @@ from .TCT import * from .translator_node import TranslatorNode as TranslatorNode +from .workflow import ( + replay_workflow as replay_workflow, + run_workflow as run_workflow, + validate_workflow as validate_workflow, +) from .config import ( RuntimeConfig as RuntimeConfig, diff --git a/TCT/interfaces/WORKFLOWS.md b/TCT/interfaces/WORKFLOWS.md new file mode 100644 index 0000000..4ec2f0c --- /dev/null +++ b/TCT/interfaces/WORKFLOWS.md @@ -0,0 +1,170 @@ +# Explainable query workflows (experimental, version 1) + +The workflow API records how explicit operations produce an answer: requests, +responses, data dependencies, filter decisions, and row lineage through joins. +It returns both an executable recipe and a self-contained evidence bundle. +Descriptions are caller-supplied intent, not inferred explanations or private +model reasoning. A reproducible graph path is not proof of a biological mechanism. + +This is an **opt-in first slice**. Existing finder, CLI, and MCP tools retain +their existing contracts. Their internal calls do not automatically become +workflow steps. There is no hosted dashboard or automatic question-to-recipe +planner in this change. Workflows do not need Langfuse or an MCP installation. + +## Python, CLI, and MCP + +```python +import json +from pathlib import Path +from TCT import run_workflow, replay_workflow, validate_workflow + +# This committed fixture contains synthetic data and makes no network calls. +recipe = json.loads(Path("examples/workflows/offline_pathway.json").read_text()) +plan = validate_workflow(recipe) +bundle = run_workflow(plan["workflow"]) +Path("evidence.json").write_text(json.dumps(bundle, indent=2)) + +replayed = replay_workflow(json.loads(Path("evidence.json").read_text())) +assert replayed["outputs"] == bundle["outputs"] +``` + +The same operations are MCP tools `validate_query_workflow`, +`run_query_workflow`, and `replay_query_workflow`, with structured JSON outputs. +The CLI publishes `validate-query-workflow`, `run-query-workflow`, and +`replay-query-workflow`; `--workflow` and `--bundle` accept JSON objects as in +the other TCT commands. Save the entire returned bundle for replay. A successful +tool/CLI invocation means the bundle was produced: inspect `bundle.status`, +per-step status, and `output_errors` to determine whether the workflow completed. + +## Recipe and operations + +A recipe has `schema_version: 1`, `steps`, and an `outputs` object mapping +names to references. A step has a unique `id`, an `op`, operation fields, +and optionally a `description`. References have the form +`{"$ref": "earlier_or_later_step", "path": "/message/results"}`. The path +is an RFC 6901 JSON pointer; omit it to use the entire output. Escape `/` in a +key as `~1`, and `~` as `~0`. Arrays use zero-based indices. + +References determine the DAG. Forward references are allowed, cycles are not. +Steps execute sequentially in topological order. Independent branches continue +after another branch fails, and dependents of failed steps are explicitly skipped. +There is no dynamic code evaluation, implicit coercion, or automatic join inference. + +| Operation | Fields | Behavior | +| --- | --- | --- | +| `literal` | `value` | Retain caller-supplied JSON, explicitly labeled as such; embedded `$ref` keys are data. | +| `query` | `url`, `query`, optional `timeout_seconds` | POST the resolved TRAPI request exactly as specified. | +| `rows` | `input` reference | Convert an object into sorted `{key, value}` rows, preserving links to original keys. | +| `filter` | `input` reference, `conditions` | AND explicit `{path, operator, value}` conditions; record the disposition of every input row. | +| `project` | `input` reference, `fields` | Select named fields using JSON pointers within each row. | +| `pluck` | `input` reference, `path`, optional `unique` | Extract an array of values; optional deduplication retains every contributing row link. | +| `join` | `left`, `right` references, `left_on`, `right_on` | Exact scalar-key inner join; retain both rows and their lineage, and identify unmatched inputs. | + +Filters support `eq`, `ne`, `in`, `not_in`, `gt`, `gte`, `lt`, `lte`, and +`exists`. Equality uses canonical JSON (no string/number or boolean/number +coercion). Ordered comparisons require numbers. Missing paths fail conditions +with a recorded reason, except `exists: false`. Missing projection or join +keys fail the step. Joins reject null keys. No hierarchy, conflation, direction +reversal, or biological equivalence is inferred by local operators. + +Each transformation records counts and `row_lineage`. For a row at `/2`, its +lineage names input step IDs and JSON pointers; follow these recursively to +the original query response or caller-supplied data. A query's `inputs` records +the resolved request, while `dependencies` and the recipe record where its +parameters came from. Filters additionally record each condition's outcome. + +## Feeding a filtered selection into another query + +After turning a first response's `/message/knowledge_graph/edges` into `rows` +and filtering them, this pair of steps sends only selected intermediate IDs +to a second provider. Supply a real, explicitly chosen endpoint in place of +the placeholder: + +```json +[ + { + "id": "selected_ids", "op": "pluck", + "input": {"$ref": "filtered_edges"}, + "path": "/value/object", "unique": true + }, + { + "id": "second_query", "op": "query", + "url": "https://provider.example/query", + "timeout_seconds": 60, + "query": { + "message": { + "query_graph": { + "nodes": { + "n0": {"ids": {"$ref": "selected_ids"}}, + "n1": {"categories": ["biolink:Disease"]} + }, + "edges": { + "e0": {"subject": "n0", "object": "n1", "predicates": ["biolink:associated_with"]} + } + } + } + } + } +] +``` + +Empty `ids` fail before dispatch so an empty upstream selection cannot silently +become an unbounded query. This currently makes that branch incomplete; an +explicit empty-branch operator is future work. No predicates are pruned, and +no retries, redirects, or provider discovery occur. The minimal request and +response checks are **not full TRAPI schema or semantic validation**. + +Only synchronous HTTP 200 JSON responses with `message.results` and a +`knowledge_graph` containing `nodes` and `edges` objects are accepted. An +optional provider status must be `Success` or `OK`. Other provider statuses, +HTTP errors, asynchronous job responses, malformed data, and timeouts are +recorded as failures, never empty results. Valid zero-result responses succeed +and carry a `result_count` of zero. The response body, parsed response, HTTP +status, endpoint, and selected response headers are retained when available. + +TRAPI response `results`, `analyses`, `sources`, `qualifiers`, attributes, and +auxiliary graphs remain in the snapshot. Joining raw knowledge-graph edges +creates candidate graph connections; it does not enforce result bindings or +establish that a provider answered a multi-hop query. For supported answers, +inspect or operate on the returned result bindings and their evidence. + +## Replaying and interpreting evidence + +`replay_workflow(bundle)` defaults to **snapshot** mode: no remote calls, saved +query evidence, and recomputed local operations. It checks content hashes and +compares recomputed outputs and lineage with the saved run. Provider timestamps +are retained separately from replay timestamps. Hashes detect modification but +are not signatures or proof of source authenticity. + +`replay_workflow(bundle, mode="live")` explicitly contacts the same endpoints +again using the same recipe. It recalculates downstream query inputs from the +new upstream results; it is a workflow rerun, not replay of identical HTTP +bodies regardless of inputs. The new bundle links back with `rerun_of`. +Remote services may change data, normalization, or software. The bundle records +TCT/Python/requests versions and an engine format version, but does not pin remote +builds or promise identical biological results. Build identifiers included by +providers remain in the full responses; absent identifiers are not invented. + +Limits in version 1 are 100 steps, 10,000 rows per local transformation, +8 MiB of decoded bytes per HTTP response, and a maximum 120-second requests +socket timeout (60 seconds by default; not a whole-workflow deadline). Exceeding +a limit fails explicitly rather than silently truncating evidence. Large bundles +are returned inline, so callers should use bounded queries and persist results. +There is no server-side session store, filesystem access, or arbitrary tool +invocation exposed through the workflow interpreter. + +## Next engineering milestones + +1. Record name resolution and normalization as explicit versioned operations, + retaining rejected candidates and selection criteria. Version 1 expects + resolved identifiers or caller-supplied literal data. +2. Instrument the existing neighborhood/path finder internals using the same + evidence model, including per-provider failures and actual query rewrites. +3. Add operations over TRAPI result bindings, preserving analysis/support graphs + while assembling paths; distinguish inference from retrieved assertions. +4. Store large immutable artifacts by reference and expose inspection tools that + return only the requested steps/rows, keeping MCP context bounded. +5. Attach RIG conformance evidence using explicit contract/build identities. + Do not turn source-level conformance into an answer-correctness score. +6. Build the user view: choose a returned path, walk backward through filters + and joins, inspect supporting records, and export the replayable recipe. diff --git a/TCT/interfaces/tools.py b/TCT/interfaces/tools.py index eb92800..094dd29 100644 --- a/TCT/interfaces/tools.py +++ b/TCT/interfaces/tools.py @@ -28,6 +28,7 @@ query_KP, ) from ..trapi import query as trapi_query +from ..workflow import replay_workflow, run_workflow, validate_workflow def get_translator_resources() -> Any: @@ -336,6 +337,79 @@ def path_finder( ) +def validate_query_workflow(workflow: dict[str, Any]) -> dict[str, Any]: + """Validate a composable query recipe without contacting any providers. + + Recipes have schema_version=1, steps, and named outputs. Each step has + id and op. Data references are {"$ref": "step_id", "path": "/json/pointer"}. + Supported operations: literal (value); query (url, query, timeout_seconds); + rows (input object); filter (input, conditions); project (input, fields); + pluck (input, path, unique); join (left, right, left_on, right_on). + Transformation inputs and named outputs must be references. Filters AND + conditions with path/operator/value; operators are eq, ne, in, not_in, + gt, gte, lt, lte, exists. Joins are exact inner joins. Paths are JSON + pointers, not Python expressions. Descriptions are caller-supplied intent. + + Args: + workflow: Versioned JSON recipe with steps and named output references. + + Returns: + Normalized recipe, execution order, and explicit step dependencies. + Dynamic paths and types are checked only when upstream outputs exist. + """ + return validate_workflow(workflow) + + +def run_query_workflow(workflow: dict[str, Any]) -> dict[str, Any]: + """Execute a query DAG and return replayable evidence with row lineage. + + Use validate_query_workflow to inspect the recipe first. Query operations + POST explicit TRAPI bodies to explicit URLs without pruning predicates, + retries, redirects, or automatic provider expansion. Full synchronous + responses, resolved requests, filter decisions, and join lineage are saved. + Errors remain distinct from valid empty results; dependent steps are skipped. + Check the returned status and output_errors before interpreting outputs. + + Agent effort (advisory): + Routine: Execute an agreed, bounded recipe using resolved identifiers. + Escalate: Review identifiers, direction, qualifiers, source selection, + and evidence before interpreting a joined path as a mechanism. + Execution: Up to 100 steps, 10,000 transformation rows, 8 MiB per + response, and 120 seconds per query socket timeout. Evidence is + returned inline; callers are responsible for saving the bundle. + + Args: + workflow: Version-1 recipe described by validate_query_workflow. + + Returns: + Structured bundle containing recipe, per-step evidence and failures, + row lineage, named outputs, timestamps, versions, and content hashes. + A traceable graph connection does not establish biological causality. + """ + return run_workflow(workflow) + + +def replay_query_workflow( + bundle: dict[str, Any], mode: str = "snapshot" +) -> dict[str, Any]: + """Inspect saved query evidence offline, or explicitly rerun the recipe. + + Snapshot mode makes no network calls: it verifies bundle checksums, reuses + saved query responses, and recomputes filters, projections, and joins. + Live mode contacts the recorded endpoints again; changing remote data can + change results. Checksums detect modification, not evidence authenticity. + + Args: + bundle: Complete evidence bundle returned by run_query_workflow. + mode: snapshot (offline, default) or live (new remote requests). + + Returns: + New bundle linked to the original, with replayed or freshly queried + evidence. Snapshot replay requires the same workflow engine version. + """ + return replay_workflow(bundle, mode) + + TOOLS: tuple[Callable[..., Any], ...] = ( get_translator_resources, name_lookup, @@ -353,6 +427,9 @@ def path_finder( trapi_query_endpoint, neighborhood_finder, path_finder, + validate_query_workflow, + run_query_workflow, + replay_query_workflow, ) __all__ = [tool.__name__ for tool in TOOLS] + ["TOOLS"] diff --git a/TCT/workflow.py b/TCT/workflow.py new file mode 100644 index 0000000..b60f996 --- /dev/null +++ b/TCT/workflow.py @@ -0,0 +1,685 @@ +"""Explicit query composition, row lineage, and portable evidence snapshots. + +This opt-in API is independent of tracing and never executes arbitrary Python. +Recipes are JSON DAGs; references carry data between named operations. Remote +TRAPI responses are retained intact instead of being flattened into merged edges. +""" + +from __future__ import annotations + +from copy import deepcopy +from datetime import datetime, timezone +import hashlib +from importlib.metadata import PackageNotFoundError, version +import json +import math +import platform +import re +from time import monotonic +from typing import Any +from urllib.parse import urlsplit + +import requests + + +SCHEMA_VERSION = 1 +ENGINE_VERSION = "1" +MAX_STEPS = 100 +MAX_ROWS = 10000 +MAX_RESPONSE_BYTES = 8 * 1024 * 1024 +_OPERATIONS = { + "literal": ({"value"}, set()), + "query": ({"url", "query"}, {"timeout_seconds"}), + "rows": ({"input"}, set()), + "filter": ({"input", "conditions"}, set()), + "project": ({"input", "fields"}, set()), + "pluck": ({"input", "path"}, {"unique"}), + "join": ({"left", "right", "left_on", "right_on"}, set()), +} + + +def _json(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False) + + +def _hash(value: Any) -> str: + return hashlib.sha256(_json(value).encode("utf-8")).hexdigest() + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _pointer_tokens(path: str) -> list[str]: + if not isinstance(path, str) or (path and not path.startswith("/")): + raise ValueError("paths must be JSON pointers: empty or beginning with '/'") + if re.search(r"~(?![01])", path): + raise ValueError("invalid JSON pointer escape") + return [part.replace("~1", "/").replace("~0", "~") for part in path.split("/")[1:]] + + +class _MissingPath(ValueError): + pass + + +def _at(value: Any, path: str) -> Any: + for token in _pointer_tokens(path): + if isinstance(value, dict) and token in value: + value = value[token] + elif isinstance(value, list) and re.fullmatch(r"0|[1-9][0-9]*", token): + index = int(token) + if index >= len(value): + raise _MissingPath(f"missing path {path!r}") + value = value[index] + else: + raise _MissingPath(f"missing path {path!r}") + return value + + +def _references(value: Any) -> list[dict[str, str]]: + if isinstance(value, dict): + if "$ref" in value: + if set(value) - {"$ref", "path"} or not isinstance(value["$ref"], str): + raise ValueError( + "references contain only a string $ref and optional path" + ) + _pointer_tokens(value.get("path", "")) + return [value] + return [ref for item in value.values() for ref in _references(item)] + if isinstance(value, list): + return [ref for item in value for ref in _references(item)] + return [] + + +def _require_reference(value: Any) -> None: + if not isinstance(value, dict) or "$ref" not in value: + raise ValueError("transformation inputs must be step references") + _references(value) + + +def validate_workflow(workflow: dict[str, Any]) -> dict[str, Any]: + """Validate and normalize a version-1 recipe without making network calls. + + References may point forward in the recipe; execution is topologically sorted. + Unknown fields, cycles, unknown steps, and unsupported operations are rejected. + Dynamic input types and paths are checked when their upstream results exist. + """ + # Strict JSON round-trip also isolates execution from caller-owned objects. + recipe = json.loads(_json(workflow)) + if not isinstance(recipe, dict) or set(recipe) - { + "schema_version", + "steps", + "outputs", + "description", + }: + raise ValueError( + "workflow must contain schema_version, steps, outputs, and optional description" + ) + if ( + type(recipe.get("schema_version")) is not int + or recipe["schema_version"] != SCHEMA_VERSION + ): + raise ValueError("unsupported workflow schema_version") + if "description" in recipe and not isinstance(recipe["description"], str): + raise ValueError("workflow description must be a string") + steps = recipe.get("steps") + if not isinstance(steps, list) or not 1 <= len(steps) <= MAX_STEPS: + raise ValueError(f"workflow must have 1..{MAX_STEPS} steps") + outputs = recipe.get("outputs") + if not isinstance(outputs, dict) or not outputs: + raise ValueError( + "workflow outputs must be a nonempty mapping of names to references" + ) + for output in outputs.values(): + _require_reference(output) + dependencies: dict[str, list[str]] = {} + for step in steps: + if not isinstance(step, dict): + raise ValueError("each step must be an object") + step_id = step.get("id") + if not isinstance(step_id, str) or not re.fullmatch( + r"[A-Za-z][A-Za-z0-9_-]*", step_id + ): + raise ValueError( + "step IDs must start with a letter and contain letters, digits, '_' or '-'" + ) + if step_id in dependencies: + raise ValueError(f"duplicate step ID: {step_id}") + op = step.get("op") + if not isinstance(op, str) or op not in _OPERATIONS: + raise ValueError(f"unsupported operation: {op}") + required, optional = _OPERATIONS[op] + if required - step.keys() or step.keys() - required - optional - { + "id", + "op", + "description", + }: + raise ValueError(f"invalid fields for {op} step {step_id}") + if "description" in step and not isinstance(step["description"], str): + raise ValueError( + "step descriptions must be strings (caller-supplied intent)" + ) + for field in ("input", "left", "right"): + if field in step: + _require_reference(step[field]) + if op == "query": + url = step["url"] + if not isinstance(url, str): + raise ValueError("query url must be an explicit HTTP(S) endpoint") + parsed = urlsplit(url) + if ( + parsed.scheme not in {"http", "https"} + or not parsed.hostname + or parsed.username + or parsed.password + or parsed.fragment + ): + raise ValueError( + "query url must be HTTP(S), without credentials or fragment" + ) + timeout = step.setdefault("timeout_seconds", 60) + if ( + type(timeout) not in (int, float) + or not math.isfinite(timeout) + or not 0 < timeout <= 120 + ): + raise ValueError("timeout_seconds must be a number in (0, 120]") + if not isinstance(step["query"], dict): + raise ValueError("query must be a JSON object") + if op == "filter": + conditions = step["conditions"] + if not isinstance(conditions, list) or not conditions: + raise ValueError( + "filter conditions must be a nonempty list (combined with AND)" + ) + for condition in conditions: + if not isinstance(condition, dict) or set(condition) != { + "path", + "operator", + "value", + }: + raise ValueError( + "each condition requires path, operator, and value" + ) + _pointer_tokens(condition["path"]) + operator = condition["operator"] + if operator not in ( + "eq", + "ne", + "in", + "not_in", + "gt", + "gte", + "lt", + "lte", + "exists", + ): + raise ValueError(f"unsupported filter operator: {operator}") + if operator in ("in", "not_in") and not isinstance( + condition["value"], list + ): + raise ValueError("in/not_in conditions require a list value") + if operator == "exists" and type(condition["value"]) is not bool: + raise ValueError("exists conditions require a boolean value") + if operator in ("gt", "gte", "lt", "lte") and type( + condition["value"] + ) not in (int, float): + raise ValueError("ordered conditions require a numeric value") + if op == "project": + if not isinstance(step["fields"], dict) or not step["fields"]: + raise ValueError( + "project fields must map output names to JSON pointers" + ) + for path in step["fields"].values(): + _pointer_tokens(path) + for field in ("path", "left_on", "right_on"): + if field in step: + _pointer_tokens(step[field]) + if op == "pluck": + if type(step.setdefault("unique", False)) is not bool: + raise ValueError("pluck unique must be boolean") + refs = [] if op == "literal" else _references(step) + dependencies[step_id] = sorted({ref["$ref"] for ref in refs}) + all_refs = [ref["$ref"] for ref in _references(outputs)] + all_refs += [ref for refs in dependencies.values() for ref in refs] + unknown = set(all_refs) - dependencies.keys() + if unknown: + raise ValueError(f"unknown step references: {sorted(unknown)}") + order: list[str] = [] + pending = list(dependencies) + while pending: + ready = [key for key in pending if set(dependencies[key]) <= set(order)] + if not ready: + raise ValueError("workflow contains a dependency cycle") + order.extend(ready) + pending = [key for key in pending if key not in ready] + return {"workflow": recipe, "execution_order": order, "dependencies": dependencies} + + +def _resolve(value: Any, records: dict[str, Any]) -> Any: + if isinstance(value, dict): + if "$ref" in value: + return deepcopy( + _at(records[value["$ref"]]["output"], value.get("path", "")) + ) + return {key: _resolve(item, records) for key, item in value.items()} + if isinstance(value, list): + return [_resolve(item, records) for item in value] + return value + + +def _link(ref: dict[str, str], key: Any = None) -> dict[str, str]: + path = ref.get("path", "") + if key is not None: + path += "/" + str(key).replace("~", "~0").replace("/", "~1") + return {"step": ref["$ref"], "path": path} + + +def _rows(value: Any) -> list[Any]: + if not isinstance(value, list): + raise ValueError("operation requires an array; use rows to expand an object") + if len(value) > MAX_ROWS: + raise ValueError(f"row limit exceeded ({MAX_ROWS}); narrow the query") + return value + + +def _condition(row: Any, condition: dict[str, Any]) -> tuple[bool, str]: + operator, expected = condition["operator"], condition["value"] + try: + actual = _at(row, condition["path"]) + except _MissingPath: + return (operator == "exists" and expected is False, "missing_path") + if operator == "exists": + return expected, "path_present" + if operator in ("eq", "ne"): + equal = _json(actual) == _json(expected) + return (equal if operator == "eq" else not equal), "value_comparison" + if operator in ("in", "not_in"): + member = any(_json(actual) == _json(item) for item in expected) + return (member if operator == "in" else not member), "membership" + if type(actual) not in (int, float): + return False, "not_numeric" + comparison = { + "gt": actual > expected, + "gte": actual >= expected, + "lt": actual < expected, + "lte": actual <= expected, + } + return comparison[operator], "numeric_comparison" + + +def _transform( + step: dict[str, Any], args: dict[str, Any] +) -> tuple[Any, dict[str, Any]]: + op = step["op"] + if op == "literal": + return args["value"], {"origin": "caller_supplied"} + lineage: list[dict[str, Any]] = [] + details: dict[str, Any] = {"row_lineage": lineage} + output: list[Any] = [] + + def emit(value: Any, links: list[dict[str, str]]) -> None: + if len(output) >= MAX_ROWS: + raise ValueError(f"row limit exceeded ({MAX_ROWS}); narrow the query") + lineage.append({"output_path": f"/{len(output)}", "inputs": links}) + output.append(value) + + if op == "rows": + source = args["input"] + if not isinstance(source, dict): + raise ValueError("rows requires an object keyed by record ID") + for key in sorted(source): + emit({"key": key, "value": source[key]}, [_link(step["input"], key)]) + details["input_count"] = len(source) + elif op == "join": + left, right = _rows(args["left"]), _rows(args["right"]) + index: dict[str, list[int]] = {} + + def join_key(row: Any, path: str) -> str: + value = _at(row, path) + if value is None or isinstance(value, (dict, list)): + raise ValueError("join keys must be non-null JSON scalars") + return _json(value) + + for j, row in enumerate(right): + index.setdefault(join_key(row, args["right_on"]), []).append(j) + matched_left, matched_right = set(), set() + for i, row in enumerate(left): + for j in index.get(join_key(row, args["left_on"]), []): + emit( + {"left": row, "right": right[j]}, + [_link(step["left"], i), _link(step["right"], j)], + ) + matched_left.add(i) + matched_right.add(j) + details.update( + left_count=len(left), + right_count=len(right), + unmatched_left=[i for i in range(len(left)) if i not in matched_left], + unmatched_right=[i for i in range(len(right)) if i not in matched_right], + join_type="inner", + equality="exact_json", + ) + else: + source = _rows(args["input"]) + details["input_count"] = len(source) + decisions = [] + unique: dict[str, int] = {} + for i, row in enumerate(source): + links = [_link(step["input"], i)] + if op == "filter": + checks = [ + _condition(row, condition) for condition in args["conditions"] + ] + keep = all(passed for passed, _ in checks) + decisions.append( + { + "input": links[0], + "kept": keep, + "conditions": [ + {"index": j, "passed": passed, "basis": basis} + for j, (passed, basis) in enumerate(checks) + ], + } + ) + if keep: + emit(row, links) + elif op == "project": + emit( + {name: _at(row, path) for name, path in args["fields"].items()}, + links, + ) + elif op == "pluck": + value = _at(row, args["path"]) + key = _json(value) + if args["unique"] and key in unique: + lineage[unique[key]]["inputs"].extend(links) + else: + unique[key] = len(output) + emit(value, links) + if op == "filter": + details.update(decisions=decisions, removed_count=len(source) - len(output)) + if op == "pluck": + details["duplicates_removed"] = len(source) - len(output) + details["output_count"] = len(output) + return output, details + + +def _check_query(query: Any) -> None: + if not isinstance(query, dict): + raise ValueError("resolved query must be an object") + message = query.get("message") + if not isinstance(message, dict) or not isinstance( + message.get("query_graph"), dict + ): + raise ValueError("query requires a message.query_graph object") + graph = message["query_graph"] + nodes, edges = graph.get("nodes"), graph.get("edges") + if ( + not isinstance(nodes, dict) + or not nodes + or not isinstance(edges, dict) + or not edges + ): + raise ValueError( + "query requires a nonempty message.query_graph with nodes and edges" + ) + for node in nodes.values(): + if not isinstance(node, dict): + raise ValueError("query nodes must be objects") + if "ids" in node and ( + not isinstance(node["ids"], list) + or not node["ids"] + or any( + not isinstance(value, str) or not value.strip() for value in node["ids"] + ) + ): + raise ValueError( + "empty or invalid query IDs; refusing to broaden an empty upstream selection" + ) + for edge in edges.values(): + if ( + not isinstance(edge, dict) + or edge.get("subject") not in nodes + or edge.get("object") not in nodes + ): + raise ValueError("query edge endpoints must reference query nodes") + + +def _query(args: dict[str, Any], record: dict[str, Any]) -> Any: + _check_query(args["query"]) + # No predicate pruning, retry, redirect, or implicit provider discovery. + with requests.post( + args["url"], + json=args["query"], + timeout=args["timeout_seconds"], + allow_redirects=False, + stream=True, + ) as response: + record["evidence_origin"] = "provider_response" + record["transport"] = { + "status_code": response.status_code, + "url": response.url, + "headers": { + key: response.headers[key] + for key in ("Content-Type", "Date") + if key in response.headers + }, + } + chunks = bytearray() + for chunk in response.iter_content(chunk_size=65536): + chunks.extend(chunk) + if len(chunks) > MAX_RESPONSE_BYTES: + raise ValueError( + f"response exceeds {MAX_RESPONSE_BYTES} bytes; no complete snapshot captured" + ) + body = bytes(chunks).decode("utf-8") + record["response_text"] = body + payload = json.loads(body) + _json(payload) # Reject non-finite numbers before treating this as evidence. + record["response"] = payload + if response.status_code != 200: + raise ValueError( + f"provider HTTP {response.status_code}; not an empty result" + ) + if not isinstance(payload, dict): + raise ValueError("provider response must be a TRAPI object") + if payload.get("status") not in (None, "Success", "OK"): + raise ValueError( + f"provider status {payload['status']!r}; inspect retained response" + ) + message = payload.get("message") + if not isinstance(message, dict) or not isinstance( + message.get("results"), list + ): + raise ValueError("synchronous response requires message.results array") + kg = message.get("knowledge_graph") + if ( + not isinstance(kg, dict) + or not isinstance(kg.get("nodes"), dict) + or not isinstance(kg.get("edges"), dict) + ): + raise ValueError( + "synchronous response requires knowledge_graph nodes and edges objects" + ) + record["summary"] = { + "result_count": len(message["results"]), + "node_count": len(kg["nodes"]), + "edge_count": len(kg["edges"]), + } + return payload + + +def _software() -> dict[str, str]: + try: + tct_version = version("TCT") + except PackageNotFoundError: + tct_version = "uninstalled" + return { + "tct": tct_version, + "python": platform.python_version(), + "requests": requests.__version__, + } + + +def _execute( + plan: dict[str, Any], saved: dict[str, Any] | None = None +) -> dict[str, Any]: + recipe = plan["workflow"] + by_id = {step["id"]: step for step in recipe["steps"]} + records: dict[str, Any] = {} + bundle: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "engine_version": ENGINE_VERSION, + "mode": "snapshot" if saved is not None else "live", + "started_at": _now(), + "software": _software(), + "workflow": recipe, + "workflow_sha256": _hash(recipe), + "execution_order": plan["execution_order"], + "steps": records, + } + for step_id in plan["execution_order"]: + step = by_id[step_id] + dependencies = plan["dependencies"][step_id] + record: dict[str, Any] = { + "op": step["op"], + "dependencies": dependencies, + "started_at": _now(), + } + records[step_id] = record + started = monotonic() + blocked = [key for key in dependencies if records[key]["status"] != "succeeded"] + if blocked: + record.update(status="skipped", blocked_by=blocked) + else: + try: + parameters = { + key: value + for key, value in step.items() + if key not in {"id", "op", "description"} + } + args = ( + deepcopy(parameters) + if step["op"] == "literal" + else _resolve(parameters, records) + ) + record["inputs"] = args + if step["op"] == "query": + if saved is None: + output = _query(args, record) + else: + previous = saved["steps"][step_id] + if args != previous.get("inputs"): + raise ValueError( + "snapshot query inputs differ from recorded request" + ) + for key in ( + "transport", + "response_text", + "response", + "summary", + "error", + ): + if key in previous: + record[key] = deepcopy(previous[key]) + record["evidence_origin"] = ( + "saved_response" + if "response_text" in previous + else "saved_failure" + ) + record["original_started_at"] = previous.get( + "original_started_at", previous["started_at"] + ) + if previous["status"] == "failed": + record["status"] = "failed" + output = deepcopy(previous.get("output")) + else: + output, details = _transform(step, args) + record.update(details) + if record.get("status") != "failed": + record.update( + status="succeeded", output=output, output_sha256=_hash(output) + ) + except ( + ValueError, + TypeError, + KeyError, + requests.RequestException, + ) as error: + record.update( + status="failed", + error={"type": type(error).__name__, "message": str(error)}, + ) + record["duration_seconds"] = monotonic() - started + record["finished_at"] = _now() + outputs, output_errors = {}, {} + for name, ref in recipe["outputs"].items(): + if records[ref["$ref"]]["status"] != "succeeded": + output_errors[name] = f"step {ref['$ref']} did not succeed" + else: + try: + outputs[name] = _resolve(ref, records) + except _MissingPath as error: + output_errors[name] = str(error) + bundle.update(outputs=outputs, output_errors=output_errors, finished_at=_now()) + bundle["status"] = ( + "succeeded" + if not output_errors + and all(r["status"] == "succeeded" for r in records.values()) + else "incomplete" + ) + if saved is not None: + bundle["replayed_from"] = saved["bundle_sha256"] + # Replay recomputes transformations; it must agree with saved evidence. + for step_id, record in records.items(): + for field in ("status", "inputs", "output", "row_lineage", "decisions"): + if record.get(field) != saved["steps"][step_id].get(field): + raise ValueError(f"snapshot replay differs at {step_id}.{field}") + bundle["bundle_sha256"] = _hash(bundle) + return bundle + + +def run_workflow(workflow: dict[str, Any]) -> dict[str, Any]: + """Execute a validated recipe and return a self-contained evidence bundle. + + Step failures are retained and their dependents skipped; independent branches + continue. ``status=incomplete`` must never be interpreted as no evidence found. + Existing TCT helpers and return shapes are unaffected by this opt-in API. + """ + return _execute(validate_workflow(workflow)) + + +def replay_workflow(bundle: dict[str, Any], mode: str = "snapshot") -> dict[str, Any]: + """Replay offline by default, or explicitly rerun remote queries with live. + + Hashes detect accidental modification, not authenticity. Offline replay uses + saved query responses and recomputes local transformations. Live replay fixes + the recipe and endpoints, but cannot pin remote data or guarantee equal output. + """ + if mode not in {"snapshot", "live"}: + raise ValueError("mode must be snapshot or live") + snapshot = json.loads(_json(bundle)) + if not isinstance(snapshot, dict): + raise ValueError("evidence bundle must be a JSON object") + digest = snapshot.pop("bundle_sha256", None) + if digest != _hash(snapshot): + raise ValueError("evidence bundle checksum mismatch") + snapshot["bundle_sha256"] = digest + if ( + snapshot.get("schema_version") != SCHEMA_VERSION + or snapshot.get("engine_version") != ENGINE_VERSION + ): + raise ValueError("unsupported evidence bundle or engine version") + plan = validate_workflow(snapshot["workflow"]) + if snapshot.get("workflow_sha256") != _hash(plan["workflow"]): + raise ValueError("workflow checksum mismatch") + if set(snapshot.get("steps", {})) != set(plan["execution_order"]): + raise ValueError("snapshot step set differs from recipe") + if mode == "snapshot": + return _execute(plan, saved=snapshot) + result = _execute(plan) + result.pop("bundle_sha256") + result["rerun_of"] = digest + result["bundle_sha256"] = _hash(result) + return result diff --git a/examples/workflows/offline_pathway.json b/examples/workflows/offline_pathway.json new file mode 100644 index 0000000..ae03e5d --- /dev/null +++ b/examples/workflows/offline_pathway.json @@ -0,0 +1,57 @@ +{ + "schema_version": 1, + "description": "Synthetic offline demonstration of evidence flow; these are not biological findings.", + "steps": [ + { + "id": "first_hop", + "op": "literal", + "description": "Caller-supplied synthetic edge records, not a provider response.", + "value": [ + {"subject": "EXAMPLE:compound", "predicate": "biolink:affects", "object": "EXAMPLE:gene1", "evidence": "fixture:first"}, + {"subject": "EXAMPLE:compound", "predicate": "biolink:related_to", "object": "EXAMPLE:gene2", "evidence": "fixture:excluded"} + ] + }, + { + "id": "filtered", + "op": "filter", + "description": "Retain the exact requested predicate; this is not a causality assessment.", + "input": {"$ref": "first_hop"}, + "conditions": [{"path": "/predicate", "operator": "eq", "value": "biolink:affects"}] + }, + { + "id": "intermediates", + "op": "pluck", + "input": {"$ref": "filtered"}, + "path": "/object", + "unique": true + }, + { + "id": "second_hop", + "op": "literal", + "description": "Independent synthetic branch with both matched and unmatched rows.", + "value": [ + {"subject": "EXAMPLE:gene1", "predicate": "biolink:associated_with", "object": "EXAMPLE:condition", "evidence": "fixture:second"}, + {"subject": "EXAMPLE:gene2", "predicate": "biolink:associated_with", "object": "EXAMPLE:condition", "evidence": "fixture:unmatched"} + ] + }, + { + "id": "joined", + "op": "join", + "left": {"$ref": "filtered"}, + "right": {"$ref": "second_hop"}, + "left_on": "/object", + "right_on": "/subject" + }, + { + "id": "pathways", + "op": "project", + "input": {"$ref": "joined"}, + "fields": {"start": "/left/subject", "intermediate": "/left/object", "end": "/right/object"} + } + ], + "outputs": { + "pathways": {"$ref": "pathways"}, + "supporting_rows": {"$ref": "joined"}, + "intermediate_ids": {"$ref": "intermediates"} + } +} diff --git a/tests/test_mcp_contract.py b/tests/test_mcp_contract.py index 4665b22..1a6bb75 100644 --- a/tests/test_mcp_contract.py +++ b/tests/test_mcp_contract.py @@ -60,6 +60,9 @@ "(start: 'str', end: 'str', " "intermediate_categories: 'list[str] | None' = None) -> 'Any'" ), + "validate_query_workflow": "(workflow: 'dict[str, Any]') -> 'dict[str, Any]'", + "run_query_workflow": "(workflow: 'dict[str, Any]') -> 'dict[str, Any]'", + "replay_query_workflow": "(bundle: 'dict[str, Any]', mode: 'str' = 'snapshot') -> 'dict[str, Any]'", } @@ -260,7 +263,51 @@ def test_generated_mcp_input_schemas_are_stable(): def test_tools_keep_unstructured_output_contract(): - """Adding annotations must not silently enable structured MCP outputs.""" + """Existing tools keep their outputs; new workflows use structured results.""" tools = asyncio.run(mcp.get_tools()) - assert all(tool.output_schema is None for tool in tools.values()) + for name, tool in tools.items(): + if name in { + "validate_query_workflow", + "run_query_workflow", + "replay_query_workflow", + }: + assert tool.output_schema is not None + else: + assert tool.output_schema is None + + +def test_workflow_tool_inputs_are_explicit_json_objects(): + tools = asyncio.run(mcp.get_tools()) + for name in ("validate_query_workflow", "run_query_workflow"): + schema = _without_titles(tools[name].parameters) + assert schema["properties"] == { + "workflow": {"type": "object", "additionalProperties": True} + } + assert schema["required"] == ["workflow"] + schema = _without_titles(tools["replay_query_workflow"].parameters) + assert schema["properties"] == { + "bundle": {"type": "object", "additionalProperties": True}, + "mode": {"type": "string", "default": "snapshot"}, + } + assert schema["required"] == ["bundle"] + + +def test_mcp_workflow_returns_structured_replayable_evidence(): + """Exercise the actual registered MCP adapters, including JSON output.""" + + async def roundtrip(): + tools = await mcp.get_tools() + workflow = { + "schema_version": 1, + "steps": [{"id": "data", "op": "literal", "value": ["EXAMPLE:1"]}], + "outputs": {"ids": {"$ref": "data"}}, + } + result = await tools["run_query_workflow"].run({"workflow": workflow}) + bundle = result.structured_content + replayed = await tools["replay_query_workflow"].run({"bundle": bundle}) + assert bundle["status"] == "succeeded" + assert replayed.structured_content["outputs"] == {"ids": ["EXAMPLE:1"]} + assert replayed.structured_content["replayed_from"] == bundle["bundle_sha256"] + + asyncio.run(roundtrip()) diff --git a/tests/test_tools.py b/tests/test_tools.py index 62ce71f..252a476 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -25,6 +25,9 @@ "trapi_query_endpoint", "neighborhood_finder", "path_finder", + "validate_query_workflow", + "run_query_workflow", + "replay_query_workflow", ] @@ -54,7 +57,7 @@ def find_spec(self, fullname, path=None, target=None): sys.meta_path.insert(0, RejectMcpImports()) from TCT.interfaces.cli import build_parser from TCT.interfaces.tools import TOOLS -assert len(TOOLS) == 16 +assert len(TOOLS) == 19 assert build_parser().prog == "tct" """ diff --git a/tests/test_workflow.py b/tests/test_workflow.py new file mode 100644 index 0000000..0968a8b --- /dev/null +++ b/tests/test_workflow.py @@ -0,0 +1,522 @@ +"""End-to-end query composition against a controlled local HTTP provider.""" + +from copy import deepcopy +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from pathlib import Path +import threading + +import pytest +import requests + +from TCT import replay_workflow, run_workflow, validate_workflow +from TCT import workflow as engine +from TCT.interfaces import cli + + +def ref(step, path=""): + return {"$ref": step, "path": path} + + +def trapi(edges): + nodes = { + edge[key]: {"categories": ["biolink:NamedThing"]} + for edge in edges.values() + for key in ("subject", "object") + } + return { + "message": { + "knowledge_graph": {"nodes": nodes, "edges": edges}, + "auxiliary_graphs": {}, + "results": [ + { + "node_bindings": { + "n0": [{"id": edge["subject"]}], + "n1": [{"id": edge["object"]}], + }, + "analyses": [ + { + "resource_id": "infores:fixture", + "edge_bindings": {"e0": [{"id": key}]}, + } + ], + } + for key, edge in edges.items() + ], + } + } + + +FIRST = trapi( + { + "e/a~1": { + "subject": "EXAMPLE:drug", + "predicate": "biolink:affects", + "object": "EXAMPLE:gene1", + "qualifiers": [ + { + "qualifier_type_id": "biolink:object_direction_qualifier", + "qualifier_value": "decreased", + } + ], + "sources": [ + { + "resource_id": "infores:fixture", + "resource_role": "primary_knowledge_source", + } + ], + }, + "e2": { + "subject": "EXAMPLE:drug", + "predicate": "biolink:related_to", + "object": "EXAMPLE:gene2", + }, + "e3": { + "subject": "EXAMPLE:drug", + "predicate": "biolink:affects", + "object": "EXAMPLE:gene1", + }, + } +) +SECOND = trapi( + { + "f1": { + "subject": "EXAMPLE:gene1", + "predicate": "biolink:associated_with", + "object": "EXAMPLE:disease", + }, + "f2": { + "subject": "EXAMPLE:gene2", + "predicate": "biolink:associated_with", + "object": "EXAMPLE:disease", + }, + } +) + + +@pytest.fixture +def provider(): + calls = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + calls.append( + ( + self.path, + json.loads(self.rfile.read(int(self.headers["Content-Length"]))), + ) + ) + status, payload = { + "/first": (200, FIRST), + "/second": (200, SECOND), + "/empty": (200, trapi({})), + "/fail": (503, {"error": "unavailable"}), + "/pending": (202, {"job": "pending"}), + "/application-error": (200, {"status": "Error", "message": {}}), + "/malformed": (200, {"message": {}}), + "/redirect": (302, {"location": "/first"}), + }.get(self.path, (200, "not JSON")) + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.end_headers() + body = json.dumps(payload) if isinstance(payload, dict) else payload + self.wfile.write(body.encode()) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", calls + finally: + server.shutdown() + server.server_close() + thread.join() + + +def query(ids): + return { + "message": { + "query_graph": { + "nodes": { + "n0": {"ids": ids}, + "n1": {"categories": ["biolink:NamedThing"]}, + }, + "edges": { + "e0": { + "subject": "n0", + "object": "n1", + "predicates": ["biolink:related_to"], + } + }, + } + } + } + + +def recipe(base): + return { + "schema_version": 1, + "steps": [ + { + "id": "first", + "op": "query", + "url": base + "/first", + "query": query(["EXAMPLE:drug"]), + }, + { + "id": "edges", + "op": "rows", + "input": ref("first", "/message/knowledge_graph/edges"), + }, + { + "id": "filtered", + "op": "filter", + "input": ref("edges"), + "conditions": [ + { + "path": "/value/predicate", + "operator": "eq", + "value": "biolink:affects", + } + ], + }, + { + "id": "ids", + "op": "pluck", + "input": ref("filtered"), + "path": "/value/object", + "unique": True, + }, + { + "id": "second", + "op": "query", + "url": base + "/second", + "query": query(ref("ids")), + }, + { + "id": "right", + "op": "rows", + "input": ref("second", "/message/knowledge_graph/edges"), + }, + { + "id": "paths", + "op": "join", + "left": ref("filtered"), + "right": ref("right"), + "left_on": "/value/object", + "right_on": "/value/subject", + }, + { + "id": "summary", + "op": "project", + "input": ref("paths"), + "fields": { + "start": "/left/value/subject", + "intermediate": "/left/value/object", + "end": "/right/value/object", + }, + }, + ], + "outputs": {"paths": ref("paths"), "summary": ref("summary")}, + } + + +def test_two_hop_query_keeps_evidence_and_explains_every_transformation( + provider, monkeypatch +): + base, calls = provider + workflow = recipe(base) + original = deepcopy(workflow) + bundle = run_workflow(workflow) + assert workflow == original + assert bundle["status"] == "succeeded" + assert [path for path, _ in calls] == ["/first", "/second"] + assert calls[0][1] == original["steps"][0]["query"] + assert calls[1][1]["message"]["query_graph"]["nodes"]["n0"]["ids"] == [ + "EXAMPLE:gene1" + ] + assert len(bundle["outputs"]["paths"]) == 2 + assert bundle["steps"]["first"]["output"] == FIRST + assert bundle["steps"]["first"]["transport"]["status_code"] == 200 + assert bundle["steps"]["filtered"]["removed_count"] == 1 + assert [ + decision["kept"] for decision in bundle["steps"]["filtered"]["decisions"] + ] == [True, False, True] + assert bundle["steps"]["edges"]["row_lineage"][0]["inputs"] == [ + {"step": "first", "path": "/message/knowledge_graph/edges/e~1a~01"} + ] + assert bundle["steps"]["ids"]["row_lineage"][0]["inputs"] == [ + {"step": "filtered", "path": "/0"}, + {"step": "filtered", "path": "/1"}, + ] + assert bundle["steps"]["paths"]["unmatched_right"] == [1] + assert bundle["steps"]["paths"]["row_lineage"][0]["inputs"] == [ + {"step": "filtered", "path": "/0"}, + {"step": "right", "path": "/0"}, + ] + + def no_network(*args, **kwargs): + raise AssertionError("offline replay contacted a provider") + + monkeypatch.setattr(requests, "post", no_network) + restored = json.loads(json.dumps(bundle)) + replay = replay_workflow(restored) + assert replay["outputs"] == bundle["outputs"] + assert replay["steps"]["first"]["evidence_origin"] == "saved_response" + assert replay["replayed_from"] == bundle["bundle_sha256"] + second_replay = replay_workflow(replay) + assert second_replay["outputs"] == bundle["outputs"] + assert ( + second_replay["steps"]["first"]["original_started_at"] + == bundle["steps"]["first"]["started_at"] + ) + + +def test_live_replay_explicitly_reissues_queries(provider): + base, calls = provider + bundle = run_workflow(recipe(base)) + replay = replay_workflow(bundle, mode="live") + assert len(calls) == 4 + assert replay["rerun_of"] == bundle["bundle_sha256"] + assert replay["outputs"] == bundle["outputs"] + + +@pytest.mark.parametrize( + "path", + ["fail", "pending", "application-error", "malformed", "bad-json", "redirect"], +) +def test_failures_are_retained_and_dependents_skipped(provider, path): + base, calls = provider + workflow = recipe(base) + workflow["steps"][0]["url"] = f"{base}/{path}" + workflow["steps"].append({"id": "independent", "op": "literal", "value": [1]}) + bundle = run_workflow(workflow) + assert len(calls) == 1 + assert bundle["status"] == "incomplete" + assert bundle["steps"]["first"]["status"] == "failed" + assert "response_text" in bundle["steps"]["first"] + assert bundle["steps"]["second"]["status"] == "skipped" + assert bundle["steps"]["independent"]["status"] == "succeeded" + assert bundle["output_errors"] + assert replay_workflow(bundle)["status"] == "incomplete" + assert len(calls) == 1 + + +def test_empty_result_is_successful_but_empty_ids_never_broaden_query(provider): + base, calls = provider + workflow = recipe(base) + workflow["steps"][0]["url"] = f"{base}/empty" + bundle = run_workflow(workflow) + assert len(calls) == 1 + assert bundle["steps"]["first"]["status"] == "succeeded" + assert bundle["steps"]["first"]["summary"]["result_count"] == 0 + assert bundle["steps"]["ids"]["output"] == [] + assert bundle["steps"]["second"]["status"] == "failed" + assert "empty upstream selection" in bundle["steps"]["second"]["error"]["message"] + assert replay_workflow(bundle)["status"] == "incomplete" + + +def test_timeout_is_not_an_empty_result(monkeypatch): + def timeout(*args, **kwargs): + raise requests.Timeout("provider timed out") + + monkeypatch.setattr(requests, "post", timeout) + bundle = run_workflow(recipe("https://example.invalid")) + assert bundle["steps"]["first"]["error"]["type"] == "Timeout" + assert bundle["status"] == "incomplete" + assert ( + replay_workflow(bundle)["steps"]["first"]["error"] + == bundle["steps"]["first"]["error"] + ) + assert ( + replay_workflow(bundle)["steps"]["first"]["evidence_origin"] == "saved_failure" + ) + + +@pytest.mark.parametrize( + "body", + [ + {"message": None}, + {"message": {"query_graph": None}}, + query([None]), + query([""]), + ], +) +def test_invalid_dynamic_query_is_recorded_without_network(body, monkeypatch): + def forbidden(*args, **kwargs): + raise AssertionError("invalid query contacted a provider") + + monkeypatch.setattr(requests, "post", forbidden) + workflow = recipe("https://example.invalid") + workflow["steps"][0]["query"] = body + bundle = run_workflow(workflow) + assert bundle["steps"]["first"]["status"] == "failed" + assert bundle["status"] == "incomplete" + assert replay_workflow(bundle)["status"] == "incomplete" + + +def test_response_size_limit_is_explicit(provider, monkeypatch): + base, _ = provider + monkeypatch.setattr(engine, "MAX_RESPONSE_BYTES", 10) + bundle = run_workflow(recipe(base)) + assert "no complete snapshot" in bundle["steps"]["first"]["error"]["message"] + assert bundle["status"] == "incomplete" + + +def test_forward_references_are_sorted_before_execution(): + workflow = { + "schema_version": 1, + "steps": [ + {"id": "ids", "op": "pluck", "input": ref("data"), "path": "/id"}, + {"id": "data", "op": "literal", "value": [{"id": "EXAMPLE:1"}]}, + ], + "outputs": {"ids": ref("ids")}, + } + assert validate_workflow(workflow)["execution_order"] == ["data", "ids"] + assert run_workflow(workflow)["outputs"] == {"ids": ["EXAMPLE:1"]} + + +@pytest.mark.parametrize( + "change", + [ + lambda w: w.update(schema_version=2), + lambda w: w["steps"].append(deepcopy(w["steps"][0])), + lambda w: w["steps"][2].update(input=ref("missing")), + lambda w: w["steps"][2].update(input=ref("ids")), + lambda w: w["steps"][2].update(op="eval"), + lambda w: w["steps"][2].update(typo=True), + lambda w: w["steps"][2].update(input=[]), + lambda w: w["steps"][2]["conditions"][0].update(operator="python"), + lambda w: w["steps"][2]["conditions"][0].update(path="not-a-pointer"), + lambda w: w["steps"][0].update(timeout_seconds=0), + lambda w: w["steps"][0].update(url="file:///etc/passwd"), + lambda w: w["outputs"].update(paths=ref("paths", "/bad~escape")), + ], +) +def test_invalid_plans_are_rejected_before_any_query(change, monkeypatch): + def forbidden(*args, **kwargs): + raise AssertionError("invalid recipe contacted a provider") + + monkeypatch.setattr(requests, "post", forbidden) + workflow = recipe("https://example.invalid") + change(workflow) + with pytest.raises(ValueError): + run_workflow(workflow) + + +def test_missing_filter_values_are_explained_and_missing_projection_is_failure(): + workflow = { + "schema_version": 1, + "steps": [ + { + "id": "data", + "op": "literal", + "value": [{"score": 2}, {}, {"score": "unknown"}], + }, + { + "id": "filtered", + "op": "filter", + "input": ref("data"), + "conditions": [{"path": "/score", "operator": "gte", "value": 1}], + }, + { + "id": "bad", + "op": "project", + "input": ref("data"), + "fields": {"value": "/absent"}, + }, + ], + "outputs": {"result": ref("filtered")}, + } + bundle = run_workflow(workflow) + assert bundle["outputs"]["result"] == [{"score": 2}] + assert ( + bundle["steps"]["filtered"]["decisions"][1]["conditions"][0]["basis"] + == "missing_path" + ) + assert ( + bundle["steps"]["filtered"]["decisions"][2]["conditions"][0]["basis"] + == "not_numeric" + ) + assert bundle["steps"]["bad"]["status"] == "failed" + assert bundle["status"] == "incomplete" + + +def test_join_limits_and_null_keys_are_not_silently_truncated(monkeypatch): + workflow = { + "schema_version": 1, + "steps": [ + {"id": "data", "op": "literal", "value": [{"id": "x"}, {"id": "x"}]}, + { + "id": "joined", + "op": "join", + "left": ref("data"), + "right": ref("data"), + "left_on": "/id", + "right_on": "/id", + }, + ], + "outputs": {"result": ref("joined")}, + } + monkeypatch.setattr(engine, "MAX_ROWS", 2) + result = run_workflow(workflow) + assert "row limit exceeded" in result["steps"]["joined"]["error"]["message"] + assert "output" not in result["steps"]["joined"] + workflow["steps"][0]["value"] = [{"id": None}] + result = run_workflow(workflow) + assert "non-null" in result["steps"]["joined"]["error"]["message"] + + +def test_modified_snapshot_is_rejected(): + workflow = { + "schema_version": 1, + "steps": [{"id": "data", "op": "literal", "value": [1]}], + "outputs": {"result": ref("data")}, + } + bundle = run_workflow(workflow) + bundle["steps"]["data"]["output"] = [2] + with pytest.raises(ValueError, match="checksum"): + replay_workflow(bundle) + + +def test_missing_named_output_path_is_incomplete(): + workflow = { + "schema_version": 1, + "steps": [{"id": "data", "op": "literal", "value": {}}], + "outputs": {"result": ref("data", "/missing")}, + } + bundle = run_workflow(workflow) + assert bundle["status"] == "incomplete" + assert bundle["outputs"] == {} + assert "missing path" in bundle["output_errors"]["result"] + + +def test_cli_workflow_roundtrip(capsys): + workflow = { + "schema_version": 1, + "steps": [{"id": "data", "op": "literal", "value": [1]}], + "outputs": {"result": ref("data")}, + } + assert cli.main(["run-query-workflow", "--workflow", json.dumps(workflow)]) == 0 + bundle = json.loads(capsys.readouterr().out) + assert cli.main(["replay-query-workflow", "--bundle", json.dumps(bundle)]) == 0 + replay = json.loads(capsys.readouterr().out) + assert replay["outputs"] == {"result": [1]} + + +def test_documented_offline_example_is_executable(): + path = Path(__file__).parents[1] / "examples/workflows/offline_pathway.json" + workflow = json.loads(path.read_text()) + bundle = run_workflow(workflow) + assert bundle["status"] == "succeeded" + assert bundle["outputs"]["pathways"] == [ + { + "start": "EXAMPLE:compound", + "intermediate": "EXAMPLE:gene1", + "end": "EXAMPLE:condition", + } + ] + assert replay_workflow(bundle)["outputs"] == bundle["outputs"] From 96b3aa1176d7398810bab109417ebed794da2c20 Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Wed, 30 Sep 2026 11:11:12 -0700 Subject: [PATCH 2/8] Add optional TRAPI representations to shared query tools --- TCT/interfaces/WORKFLOWS.md | 37 ++++++++ TCT/interfaces/tools.py | 47 +++++++-- TCT/trapi_representation.py | 65 +++++++++++++ tests/test_mcp_contract.py | 28 +++++- tests/test_trapi_representation.py | 148 +++++++++++++++++++++++++++++ 5 files changed, 315 insertions(+), 10 deletions(-) create mode 100644 TCT/trapi_representation.py create mode 100644 tests/test_trapi_representation.py diff --git a/TCT/interfaces/WORKFLOWS.md b/TCT/interfaces/WORKFLOWS.md index 4ec2f0c..50d9256 100644 --- a/TCT/interfaces/WORKFLOWS.md +++ b/TCT/interfaces/WORKFLOWS.md @@ -168,3 +168,40 @@ invocation exposed through the workflow interpreter. Do not turn source-level conformance into an answer-correctness score. 6. Build the user view: choose a returned path, walk backward through filters and joins, inspect supporting records, and export the replayable recipe. + +### Optional TRAPI representations + +The shared CLI/MCP functions `query_knowledge_provider`, `parallel_query_apis`, +`neighborhood_finder`, `path_finder`, `run_query_workflow`, and +`replay_query_workflow` accept `include_trapi=True` (CLI: `--include-trapi`). +By default they retain their existing return values. Opting in returns: + +```json +{ + "result": "the unchanged native result", + "trapi": { + "representation_version": 1, + "schema_validation": "not_performed", + "artifacts": [], + "limitations": [] + } +} +``` + +The representation is produced locally after the ordinary operation, without +additional provider requests. Artifacts are detached copies. Provider messages +and finder raw output are wrapped as response-shaped objects. Finder output may +use path extensions or incomplete bindings; this is not a schema-validation or +RIG-compliance guarantee. Parallel queries expose only a +`knowledge_graph_fragment` containing merged edges, not an invented response. +A provider's `None` result yields no artifact because emptiness and failure +cannot be distinguished by that existing interface. + +Workflow artifacts retain each executed query's resolved request and any saved +response, tagged with `step_id` and `execution_status`. Failed requests are not +represented as empty successful responses. Skipped queries have no artifact. +Filters, joins, row lineage, and reproducibility remain in `result`; no single +TRAPI query is synthesized for arbitrary composition. Pass `envelope["result"]` +to replay, preserving the original bundle's checksums. Literal-only workflows +have no TRAPI artifacts. Name resolution and normalization retain their native +interfaces. diff --git a/TCT/interfaces/tools.py b/TCT/interfaces/tools.py index 094dd29..7cc0a3b 100644 --- a/TCT/interfaces/tools.py +++ b/TCT/interfaces/tools.py @@ -28,6 +28,7 @@ query_KP, ) from ..trapi import query as trapi_query +from ..trapi_representation import with_trapi from ..workflow import replay_workflow, run_workflow, validate_workflow @@ -227,10 +228,13 @@ def query_knowledge_provider( query_json: dict[str, Any], api_names: dict[str, str], api_predicates: dict[str, list[str]], + include_trapi: bool = False, ) -> Any: """Send a TRAPI query to one Translator Knowledge Provider. Args: + include_trapi: Return {result, trapi} with an unvalidated TRAPI sidecar. + Default false preserves the native result. No extra network calls. api_name: Name of the API to query. query_json: TRAPI query sent to the provider. api_names: Mapping from API names to query URLs. @@ -240,7 +244,9 @@ def query_knowledge_provider( The provider's knowledge graph, or ``None`` when the response contains no knowledge-graph edges. """ - return query_KP(api_name, query_json, api_names, api_predicates) + result = query_KP(api_name, query_json, api_names, api_predicates) + + return with_trapi(result, "message") if include_trapi else result def parallel_query_apis( @@ -249,10 +255,13 @@ def parallel_query_apis( api_names: dict[str, str], api_predicates: dict[str, list[str]], max_workers: int = 1, + include_trapi: bool = False, ) -> Any: """Query multiple Translator APIs and merge their knowledge graphs. Args: + include_trapi: Return {result, trapi} with an unvalidated TRAPI sidecar. + Default false preserves the native result. No extra network calls. query_json: TRAPI query sent to each selected API. selected_apis: Names of APIs to query. api_names: Mapping from API names to query URLs. @@ -262,7 +271,7 @@ def parallel_query_apis( Returns: A merged knowledge graph from successful provider responses. """ - return parallel_api_query( + result = parallel_api_query( query_json, selected_apis, api_names, @@ -270,6 +279,8 @@ def parallel_query_apis( max_workers, ) + return with_trapi(result, "edges") if include_trapi else result + def trapi_query_endpoint(url: str) -> Any: """Invoke the legacy TRAPI endpoint placeholder. @@ -291,10 +302,13 @@ def trapi_query_endpoint(url: str) -> Any: def neighborhood_finder( node: list[str], neighbor_categories: list[str], + include_trapi: bool = False, ) -> Any: """Find category-filtered neighbors for one or more CURIEs using TCT. Args: + include_trapi: Return {result, trapi} with an unvalidated TRAPI sidecar. + Default false preserves the native result. No extra network calls. node: CURIEs whose neighboring nodes should be found. neighbor_categories: Biolink categories used to filter returned neighbors. @@ -304,21 +318,26 @@ def neighborhood_finder( graph, results, auxiliary graphs, and raw parsed response. """ resources = _get_translator_resources() - return tct_neighborhood_finder( + result = tct_neighborhood_finder( node=node, neighbor_categories=neighbor_categories, resources=resources, ) + return with_trapi(result, "finder") if include_trapi else result + def path_finder( start: str, end: str, intermediate_categories: list[str] | None = None, + include_trapi: bool = False, ) -> Any: """Find paths between two CURIEs using TCT. Args: + include_trapi: Return {result, trapi} with an unvalidated TRAPI sidecar. + Default false preserves the native result. No extra network calls. start: CURIE of the starting node. end: CURIE of the ending node. intermediate_categories: Optional Biolink categories allowed for @@ -329,13 +348,15 @@ def path_finder( graph, results, auxiliary graphs, and raw parsed response. """ resources = _get_translator_resources() - return query_TCT_pathfinder( + result = query_TCT_pathfinder( start, end, intermediate_categories=intermediate_categories, resources=resources, ) + return with_trapi(result, "finder") if include_trapi else result + def validate_query_workflow(workflow: dict[str, Any]) -> dict[str, Any]: """Validate a composable query recipe without contacting any providers. @@ -360,7 +381,9 @@ def validate_query_workflow(workflow: dict[str, Any]) -> dict[str, Any]: return validate_workflow(workflow) -def run_query_workflow(workflow: dict[str, Any]) -> dict[str, Any]: +def run_query_workflow( + workflow: dict[str, Any], include_trapi: bool = False +) -> dict[str, Any]: """Execute a query DAG and return replayable evidence with row lineage. Use validate_query_workflow to inspect the recipe first. Query operations @@ -379,6 +402,8 @@ def run_query_workflow(workflow: dict[str, Any]) -> dict[str, Any]: returned inline; callers are responsible for saving the bundle. Args: + include_trapi: Return {result, trapi} with an unvalidated TRAPI sidecar. + Default false preserves the native result. No extra network calls. workflow: Version-1 recipe described by validate_query_workflow. Returns: @@ -386,11 +411,13 @@ def run_query_workflow(workflow: dict[str, Any]) -> dict[str, Any]: row lineage, named outputs, timestamps, versions, and content hashes. A traceable graph connection does not establish biological causality. """ - return run_workflow(workflow) + result = run_workflow(workflow) + + return with_trapi(result, "workflow") if include_trapi else result def replay_query_workflow( - bundle: dict[str, Any], mode: str = "snapshot" + bundle: dict[str, Any], mode: str = "snapshot", include_trapi: bool = False ) -> dict[str, Any]: """Inspect saved query evidence offline, or explicitly rerun the recipe. @@ -400,6 +427,8 @@ def replay_query_workflow( change results. Checksums detect modification, not evidence authenticity. Args: + include_trapi: Return {result, trapi} with an unvalidated TRAPI sidecar. + Default false preserves the native result. No extra network calls. bundle: Complete evidence bundle returned by run_query_workflow. mode: snapshot (offline, default) or live (new remote requests). @@ -407,7 +436,9 @@ def replay_query_workflow( New bundle linked to the original, with replayed or freshly queried evidence. Snapshot replay requires the same workflow engine version. """ - return replay_workflow(bundle, mode) + result = replay_workflow(bundle, mode) + + return with_trapi(result, "workflow") if include_trapi else result TOOLS: tuple[Callable[..., Any], ...] = ( diff --git a/TCT/trapi_representation.py b/TCT/trapi_representation.py new file mode 100644 index 0000000..47799cb --- /dev/null +++ b/TCT/trapi_representation.py @@ -0,0 +1,65 @@ +"""Loss-aware, optional TRAPI representations; never execute or validate queries.""" + +from copy import deepcopy +from typing import Any + + +def with_trapi(result: Any, kind: str) -> dict[str, Any]: + """Keep the native result and attach detached, explicitly unvalidated artifacts. + + These artifacts describe retained data, not schema conformance or biological + correctness. In particular, merged edges cannot recover provider bindings. + """ + artifacts = [] + limitations = [] + + def add(artifact_kind, content, **metadata): + artifacts.append( + {"kind": artifact_kind, "content": deepcopy(content), **metadata} + ) + + if kind == "message": + if result is None: + limitations.append( + "Native result cannot distinguish failure from empty results." + ) + else: + add("response", {"message": result}) + limitations.append("Original HTTP response envelope was not retained.") + elif kind == "finder": + add("response", {"message": result.raw}) + limitations.append( + "Finder output is TRAPI-style; path extensions, bindings and auxiliary " + "graphs may not conform to a TRAPI schema." + ) + elif kind == "edges": + add("knowledge_graph_fragment", {"edges": result}) + limitations.append( + "Only merged edges remain; nodes, result bindings, auxiliary graphs " + "and provider failures cannot be recovered. This is not a TRAPI response." + ) + elif kind == "workflow": + for step_id, step in result["steps"].items(): + if step["op"] != "query": + continue + metadata = {"step_id": step_id, "execution_status": step["status"]} + inputs = step.get("inputs", {}) + if "query" in inputs: + add("query", inputs["query"], **metadata) + if "response" in step: + add("response", step["response"], **metadata) + limitations.append( + "Artifacts describe individual query steps. Filters and joins remain " + "in the evidence bundle; they are not compiled into a single TRAPI query." + ) + else: + raise ValueError(f"Unknown TRAPI representation kind: {kind}") + return { + "result": result, + "trapi": { + "representation_version": 1, + "schema_validation": "not_performed", + "artifacts": artifacts, + "limitations": limitations, + }, + } diff --git a/tests/test_mcp_contract.py b/tests/test_mcp_contract.py index 1a6bb75..93f52af 100644 --- a/tests/test_mcp_contract.py +++ b/tests/test_mcp_contract.py @@ -66,6 +66,20 @@ } +TRAPI_TOOLS = { + "query_knowledge_provider", + "parallel_query_apis", + "neighborhood_finder", + "path_finder", + "run_query_workflow", + "replay_query_workflow", +} +for name in TRAPI_TOOLS: + EXPECTED_SIGNATURES[name] = EXPECTED_SIGNATURES[name].replace( + ") ->", ", include_trapi: 'bool' = False) ->" + ) + + EXPECTED_INPUTS = { "get_translator_resources": ({}, []), "name_lookup": ( @@ -258,7 +272,10 @@ def test_generated_mcp_input_schemas_are_stable(): for name, (properties, required) in EXPECTED_INPUTS.items(): schema = _without_titles(tools[name].parameters) assert schema["type"] == "object" - assert schema["properties"] == properties + expected = dict(properties) + if name in TRAPI_TOOLS: + expected["include_trapi"] = {"type": "boolean", "default": False} + assert schema["properties"] == expected assert schema.get("required", []) == required @@ -281,7 +298,13 @@ def test_workflow_tool_inputs_are_explicit_json_objects(): tools = asyncio.run(mcp.get_tools()) for name in ("validate_query_workflow", "run_query_workflow"): schema = _without_titles(tools[name].parameters) - assert schema["properties"] == { + properties = dict(schema["properties"]) + if name == "run_query_workflow": + assert properties.pop("include_trapi") == { + "type": "boolean", + "default": False, + } + assert properties == { "workflow": {"type": "object", "additionalProperties": True} } assert schema["required"] == ["workflow"] @@ -289,6 +312,7 @@ def test_workflow_tool_inputs_are_explicit_json_objects(): assert schema["properties"] == { "bundle": {"type": "object", "additionalProperties": True}, "mode": {"type": "string", "default": "snapshot"}, + "include_trapi": {"type": "boolean", "default": False}, } assert schema["required"] == ["bundle"] diff --git a/tests/test_trapi_representation.py b/tests/test_trapi_representation.py new file mode 100644 index 0000000..e67604a --- /dev/null +++ b/tests/test_trapi_representation.py @@ -0,0 +1,148 @@ +"""Optional representations preserve execution, native results and evidence.""" + +from copy import deepcopy +from types import SimpleNamespace + +import pytest + +from TCT.interfaces import tools +from TCT.trapi_representation import with_trapi + + +@pytest.mark.parametrize( + ("name", "backend", "args", "native"), + [ + ( + "query_knowledge_provider", + "query_KP", + ["kp", {}, {}, {}], + {"knowledge_graph": {"nodes": {}, "edges": {}}}, + ), + ( + "parallel_query_apis", + "parallel_api_query", + [{}, [], {}, {}], + {"e": {"subject": "X:1", "object": "X:2"}}, + ), + ( + "neighborhood_finder", + "tct_neighborhood_finder", + [[], []], + SimpleNamespace(raw={"results": []}), + ), + ( + "path_finder", + "query_TCT_pathfinder", + ["X:1", "X:2"], + SimpleNamespace(raw={"results": []}), + ), + ], +) +def test_optional_wrappers_preserve_result_and_call_once( + monkeypatch, name, backend, args, native +): + calls = [] + + def invoke(*args, **kwargs): + calls.append((args, kwargs)) + return native + + monkeypatch.setattr(tools, backend, invoke) + monkeypatch.setattr(tools, "_get_translator_resources", lambda: {}) + assert getattr(tools, name)(*args) is native + baseline = calls.pop() + wrapped = getattr(tools, name)(*args, include_trapi=True) + assert calls == [baseline] + assert wrapped["result"] is native + assert wrapped["trapi"]["schema_validation"] == "not_performed" + + +def test_detached_sidecar_and_unknown_empty(): + native = { + "knowledge_graph": { + "edges": {"e": {"sources": [{"resource_id": "infores:test"}]}} + } + } + original = deepcopy(native) + sidecar = with_trapi(native, "message") + sidecar["trapi"]["artifacts"][0]["content"]["message"].clear() + assert native == original + assert with_trapi(None, "message")["trapi"]["artifacts"] == [] + + +def test_workflow_representation_keeps_each_step_and_failures(): + bundle = { + "steps": { + "one": { + "op": "query", + "status": "succeeded", + "inputs": {"query": {"message": {}}}, + "response": {"message": {"results": []}}, + }, + "two": { + "op": "query", + "status": "failed", + "inputs": {"query": {"message": {}}}, + }, + "three": {"op": "query", "status": "skipped"}, + "filter": {"op": "filter", "status": "succeeded"}, + } + } + artifacts = with_trapi(bundle, "workflow")["trapi"]["artifacts"] + assert [(a["step_id"], a["kind"]) for a in artifacts] == [ + ("one", "query"), + ("one", "response"), + ("two", "query"), + ] + + +def test_wrapped_bundle_still_replays(): + recipe = { + "schema_version": 1, + "steps": [{"id": "data", "op": "literal", "value": ["X:1"]}], + "outputs": {"ids": {"$ref": "data"}}, + } + envelope = tools.run_query_workflow(recipe, include_trapi=True) + replayed = tools.replay_query_workflow(envelope["result"], include_trapi=True) + assert replayed["result"]["outputs"] == {"ids": ["X:1"]} + assert envelope["trapi"]["artifacts"] == [] + + +def test_cli_optional_representation(capsys): + import json + from TCT.interfaces.cli import main + + recipe = { + "schema_version": 1, + "steps": [{"id": "x", "op": "literal", "value": 1}], + "outputs": {"x": {"$ref": "x"}}, + } + assert ( + main( + ["run-query-workflow", "--workflow", json.dumps(recipe), "--include-trapi"] + ) + == 0 + ) + output = json.loads(capsys.readouterr().out) + assert output["result"]["outputs"] == {"x": 1} + assert output["trapi"]["schema_validation"] == "not_performed" + + +def test_mcp_optional_representation(): + import asyncio + from TCT.interfaces.mcp import mcp + + async def run(): + registry = await mcp.get_tools() + recipe = { + "schema_version": 1, + "steps": [{"id": "x", "op": "literal", "value": 1}], + "outputs": {"x": {"$ref": "x"}}, + } + output = await registry["run_query_workflow"].run( + {"workflow": recipe, "include_trapi": True} + ) + assert output.structured_content["result"]["outputs"] == {"x": 1} + assert output.structured_content["trapi"]["artifacts"] == [] + + asyncio.run(run()) From fffcfd978731b67dbe927fb4602ac714b8aa5040 Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Wed, 30 Sep 2026 12:50:17 -0700 Subject: [PATCH 3/8] Replace TRAPI return envelopes with opt-in decorated capture --- TCT/__init__.py | 2 + TCT/execution.py | 78 +++++++++++++++++ TCT/interfaces/WORKFLOWS.md | 57 ++++++------ TCT/interfaces/tools.py | 42 ++++----- TCT/trapi_representation.py | 15 ++-- tests/test_mcp_contract.py | 22 ----- tests/test_trapi_representation.py | 135 +++++++++++++++++++++++------ 7 files changed, 236 insertions(+), 115 deletions(-) create mode 100644 TCT/execution.py diff --git a/TCT/__init__.py b/TCT/__init__.py index e47aee8..324f811 100644 --- a/TCT/__init__.py +++ b/TCT/__init__.py @@ -16,3 +16,5 @@ ) from . import name_resolver as name_resolver, node_normalizer as node_normalizer, node_annotator as node_annotator, trapi as trapi, translator_kpinfo as translator_kpinfo + +from .execution import execution as execution diff --git a/TCT/execution.py b/TCT/execution.py new file mode 100644 index 0000000..70bfc10 --- /dev/null +++ b/TCT/execution.py @@ -0,0 +1,78 @@ +"""Opt-in, context-local evidence collection without changing operation returns. + +This initial collector is in-memory. It is not a durable replay journal or a +TRAPI validator. Langfuse remains independently controlled by its existing flag. +""" + +from contextlib import contextmanager +from contextvars import ContextVar +from copy import deepcopy +from functools import wraps +from inspect import signature + +from .trapi_representation import represent_trapi + + +_collector = ContextVar("tct_trapi_collector", default=None) + + +@contextmanager +def execution(config=None): + """Collect TRAPI observations with ``execution({"trapi": True})``. + + An omitted setting inherits the enclosing collector; explicit false disables + it locally. A true setting starts a fresh collection. No files are written. + """ + config = {} if config is None else config + if not isinstance(config, dict) or set(config) - {"trapi"}: + raise ValueError("execution currently accepts only the 'trapi' setting") + if "trapi" in config and type(config["trapi"]) is not bool: + raise ValueError("trapi must be a boolean") + records = _collector.get() + if "trapi" in config: + records = [] if config["trapi"] else None + token = _collector.set(records) + try: + yield records + finally: + _collector.reset(token) + + +def capture_trapi(kind): + """Decorate an operation, emitting evidence separately from its result.""" + + def decorate(function): + contract = signature(function) + + @wraps(function) + def captured(*args, **kwargs): + records = _collector.get() + if records is None: + return function(*args, **kwargs) + bound = contract.bind(*args, **kwargs) + bound.apply_defaults() + record = { + "operation": function.__name__, + "status": "running", + } + try: + record["inputs"] = deepcopy(dict(bound.arguments)) + except Exception as error: + record["input_capture_error"] = type(error).__name__ + records.append(record) + try: + result = function(*args, **kwargs) + except BaseException as error: + record.update(status="failed", error_type=type(error).__name__) + raise + record["status"] = "succeeded" + try: + record["trapi"] = represent_trapi(result, kind) + except Exception as error: + # The operation has already completed; never replace its result. + record["capture_error"] = type(error).__name__ + return result + + return captured + + return decorate diff --git a/TCT/interfaces/WORKFLOWS.md b/TCT/interfaces/WORKFLOWS.md index 50d9256..ddf7463 100644 --- a/TCT/interfaces/WORKFLOWS.md +++ b/TCT/interfaces/WORKFLOWS.md @@ -169,39 +169,32 @@ invocation exposed through the workflow interpreter. 6. Build the user view: choose a returned path, walk backward through filters and joins, inspect supporting records, and export the replayable recipe. -### Optional TRAPI representations +### Optional TRAPI observation capture -The shared CLI/MCP functions `query_knowledge_provider`, `parallel_query_apis`, -`neighborhood_finder`, `path_finder`, `run_query_workflow`, and -`replay_query_workflow` accept `include_trapi=True` (CLI: `--include-trapi`). -By default they retain their existing return values. Opting in returns: +The experimental `include_trapi` argument and return envelope have been removed. +The six query/finder/workflow wrappers always return native results. Their +introspected CLI/MCP signatures no longer advertise a representation flag. -```json -{ - "result": "the unchanged native result", - "trapi": { - "representation_version": 1, - "schema_validation": "not_performed", - "artifacts": [], - "limitations": [] - } -} +Python callers can opt into in-memory observations: + +```python +from TCT import execution +from TCT.interfaces.tools import run_query_workflow + +with execution({"trapi": True}) as observations: + bundle = run_query_workflow(recipe) +# bundle is unchanged and can be replayed directly. +# observations contains inputs, operation status, and detached representations. ``` -The representation is produced locally after the ordinary operation, without -additional provider requests. Artifacts are detached copies. Provider messages -and finder raw output are wrapped as response-shaped objects. Finder output may -use path extensions or incomplete bindings; this is not a schema-validation or -RIG-compliance guarantee. Parallel queries expose only a -`knowledge_graph_fragment` containing merged edges, not an invented response. -A provider's `None` result yields no artifact because emptiness and failure -cannot be distinguished by that existing interface. - -Workflow artifacts retain each executed query's resolved request and any saved -response, tagged with `step_id` and `execution_status`. Failed requests are not -represented as empty successful responses. Skipped queries have no artifact. -Filters, joins, row lineage, and reproducibility remain in `result`; no single -TRAPI query is synthesized for arbitrary composition. Pass `envelope["result"]` -to replay, preserving the original bundle's checksums. Literal-only workflows -have no TRAPI artifacts. Name resolution and normalization retain their native -interfaces. +Capture is disabled by default. An omitted setting inherits an enclosing context; +explicit false temporarily disables capture. Langfuse is independently enabled +through its existing environment setting and still observes interface invocations. +This context neither writes files nor enables caching. It does not yet provide +CLI/MCP execution configuration, durable replay state, or a validated TRAPI rollup. + +Representations retain their prior limitations: provider failures and empty +results may both return None; parallel results contain only merged edges; finder +output is TRAPI-style and unvalidated. Workflow artifacts preserve individual +query requests/responses, not a compiled query for arbitrary joins. Capture +failures after execution are recorded without replacing the native return value. diff --git a/TCT/interfaces/tools.py b/TCT/interfaces/tools.py index 7cc0a3b..aae51f7 100644 --- a/TCT/interfaces/tools.py +++ b/TCT/interfaces/tools.py @@ -28,7 +28,7 @@ query_KP, ) from ..trapi import query as trapi_query -from ..trapi_representation import with_trapi +from ..execution import capture_trapi from ..workflow import replay_workflow, run_workflow, validate_workflow @@ -223,18 +223,16 @@ def optimize_query_for_api( return optimize_query_json(query_json, api_name, api_predicates) +@capture_trapi("message") def query_knowledge_provider( api_name: str, query_json: dict[str, Any], api_names: dict[str, str], api_predicates: dict[str, list[str]], - include_trapi: bool = False, ) -> Any: """Send a TRAPI query to one Translator Knowledge Provider. Args: - include_trapi: Return {result, trapi} with an unvalidated TRAPI sidecar. - Default false preserves the native result. No extra network calls. api_name: Name of the API to query. query_json: TRAPI query sent to the provider. api_names: Mapping from API names to query URLs. @@ -246,22 +244,20 @@ def query_knowledge_provider( """ result = query_KP(api_name, query_json, api_names, api_predicates) - return with_trapi(result, "message") if include_trapi else result + return result +@capture_trapi("edges") def parallel_query_apis( query_json: dict[str, Any], selected_apis: list[str], api_names: dict[str, str], api_predicates: dict[str, list[str]], max_workers: int = 1, - include_trapi: bool = False, ) -> Any: """Query multiple Translator APIs and merge their knowledge graphs. Args: - include_trapi: Return {result, trapi} with an unvalidated TRAPI sidecar. - Default false preserves the native result. No extra network calls. query_json: TRAPI query sent to each selected API. selected_apis: Names of APIs to query. api_names: Mapping from API names to query URLs. @@ -279,7 +275,7 @@ def parallel_query_apis( max_workers, ) - return with_trapi(result, "edges") if include_trapi else result + return result def trapi_query_endpoint(url: str) -> Any: @@ -299,16 +295,14 @@ def trapi_query_endpoint(url: str) -> Any: return trapi_query(url) +@capture_trapi("finder") def neighborhood_finder( node: list[str], neighbor_categories: list[str], - include_trapi: bool = False, ) -> Any: """Find category-filtered neighbors for one or more CURIEs using TCT. Args: - include_trapi: Return {result, trapi} with an unvalidated TRAPI sidecar. - Default false preserves the native result. No extra network calls. node: CURIEs whose neighboring nodes should be found. neighbor_categories: Biolink categories used to filter returned neighbors. @@ -324,20 +318,18 @@ def neighborhood_finder( resources=resources, ) - return with_trapi(result, "finder") if include_trapi else result + return result +@capture_trapi("finder") def path_finder( start: str, end: str, intermediate_categories: list[str] | None = None, - include_trapi: bool = False, ) -> Any: """Find paths between two CURIEs using TCT. Args: - include_trapi: Return {result, trapi} with an unvalidated TRAPI sidecar. - Default false preserves the native result. No extra network calls. start: CURIE of the starting node. end: CURIE of the ending node. intermediate_categories: Optional Biolink categories allowed for @@ -355,7 +347,7 @@ def path_finder( resources=resources, ) - return with_trapi(result, "finder") if include_trapi else result + return result def validate_query_workflow(workflow: dict[str, Any]) -> dict[str, Any]: @@ -381,9 +373,8 @@ def validate_query_workflow(workflow: dict[str, Any]) -> dict[str, Any]: return validate_workflow(workflow) -def run_query_workflow( - workflow: dict[str, Any], include_trapi: bool = False -) -> dict[str, Any]: +@capture_trapi("workflow") +def run_query_workflow(workflow: dict[str, Any]) -> dict[str, Any]: """Execute a query DAG and return replayable evidence with row lineage. Use validate_query_workflow to inspect the recipe first. Query operations @@ -402,8 +393,6 @@ def run_query_workflow( returned inline; callers are responsible for saving the bundle. Args: - include_trapi: Return {result, trapi} with an unvalidated TRAPI sidecar. - Default false preserves the native result. No extra network calls. workflow: Version-1 recipe described by validate_query_workflow. Returns: @@ -413,11 +402,12 @@ def run_query_workflow( """ result = run_workflow(workflow) - return with_trapi(result, "workflow") if include_trapi else result + return result +@capture_trapi("workflow") def replay_query_workflow( - bundle: dict[str, Any], mode: str = "snapshot", include_trapi: bool = False + bundle: dict[str, Any], mode: str = "snapshot" ) -> dict[str, Any]: """Inspect saved query evidence offline, or explicitly rerun the recipe. @@ -427,8 +417,6 @@ def replay_query_workflow( change results. Checksums detect modification, not evidence authenticity. Args: - include_trapi: Return {result, trapi} with an unvalidated TRAPI sidecar. - Default false preserves the native result. No extra network calls. bundle: Complete evidence bundle returned by run_query_workflow. mode: snapshot (offline, default) or live (new remote requests). @@ -438,7 +426,7 @@ def replay_query_workflow( """ result = replay_workflow(bundle, mode) - return with_trapi(result, "workflow") if include_trapi else result + return result TOOLS: tuple[Callable[..., Any], ...] = ( diff --git a/TCT/trapi_representation.py b/TCT/trapi_representation.py index 47799cb..846b76e 100644 --- a/TCT/trapi_representation.py +++ b/TCT/trapi_representation.py @@ -4,8 +4,8 @@ from typing import Any -def with_trapi(result: Any, kind: str) -> dict[str, Any]: - """Keep the native result and attach detached, explicitly unvalidated artifacts. +def represent_trapi(result: Any, kind: str) -> dict[str, Any]: + """Build detached, explicitly unvalidated artifacts from a native result. These artifacts describe retained data, not schema conformance or biological correctness. In particular, merged edges cannot recover provider bindings. @@ -55,11 +55,8 @@ def add(artifact_kind, content, **metadata): else: raise ValueError(f"Unknown TRAPI representation kind: {kind}") return { - "result": result, - "trapi": { - "representation_version": 1, - "schema_validation": "not_performed", - "artifacts": artifacts, - "limitations": limitations, - }, + "representation_version": 1, + "schema_validation": "not_performed", + "artifacts": artifacts, + "limitations": limitations, } diff --git a/tests/test_mcp_contract.py b/tests/test_mcp_contract.py index 93f52af..df8d29e 100644 --- a/tests/test_mcp_contract.py +++ b/tests/test_mcp_contract.py @@ -66,20 +66,6 @@ } -TRAPI_TOOLS = { - "query_knowledge_provider", - "parallel_query_apis", - "neighborhood_finder", - "path_finder", - "run_query_workflow", - "replay_query_workflow", -} -for name in TRAPI_TOOLS: - EXPECTED_SIGNATURES[name] = EXPECTED_SIGNATURES[name].replace( - ") ->", ", include_trapi: 'bool' = False) ->" - ) - - EXPECTED_INPUTS = { "get_translator_resources": ({}, []), "name_lookup": ( @@ -273,8 +259,6 @@ def test_generated_mcp_input_schemas_are_stable(): schema = _without_titles(tools[name].parameters) assert schema["type"] == "object" expected = dict(properties) - if name in TRAPI_TOOLS: - expected["include_trapi"] = {"type": "boolean", "default": False} assert schema["properties"] == expected assert schema.get("required", []) == required @@ -299,11 +283,6 @@ def test_workflow_tool_inputs_are_explicit_json_objects(): for name in ("validate_query_workflow", "run_query_workflow"): schema = _without_titles(tools[name].parameters) properties = dict(schema["properties"]) - if name == "run_query_workflow": - assert properties.pop("include_trapi") == { - "type": "boolean", - "default": False, - } assert properties == { "workflow": {"type": "object", "additionalProperties": True} } @@ -312,7 +291,6 @@ def test_workflow_tool_inputs_are_explicit_json_objects(): assert schema["properties"] == { "bundle": {"type": "object", "additionalProperties": True}, "mode": {"type": "string", "default": "snapshot"}, - "include_trapi": {"type": "boolean", "default": False}, } assert schema["required"] == ["bundle"] diff --git a/tests/test_trapi_representation.py b/tests/test_trapi_representation.py index e67604a..89a763e 100644 --- a/tests/test_trapi_representation.py +++ b/tests/test_trapi_representation.py @@ -6,7 +6,8 @@ import pytest from TCT.interfaces import tools -from TCT.trapi_representation import with_trapi +from TCT.trapi_representation import represent_trapi +from TCT.execution import execution, capture_trapi @pytest.mark.parametrize( @@ -51,10 +52,11 @@ def invoke(*args, **kwargs): monkeypatch.setattr(tools, "_get_translator_resources", lambda: {}) assert getattr(tools, name)(*args) is native baseline = calls.pop() - wrapped = getattr(tools, name)(*args, include_trapi=True) + with execution({"trapi": True}) as records: + wrapped = getattr(tools, name)(*args) assert calls == [baseline] - assert wrapped["result"] is native - assert wrapped["trapi"]["schema_validation"] == "not_performed" + assert wrapped is native + assert records[0]["trapi"]["schema_validation"] == "not_performed" def test_detached_sidecar_and_unknown_empty(): @@ -64,10 +66,10 @@ def test_detached_sidecar_and_unknown_empty(): } } original = deepcopy(native) - sidecar = with_trapi(native, "message") - sidecar["trapi"]["artifacts"][0]["content"]["message"].clear() + sidecar = represent_trapi(native, "message") + sidecar["artifacts"][0]["content"]["message"].clear() assert native == original - assert with_trapi(None, "message")["trapi"]["artifacts"] == [] + assert represent_trapi(None, "message")["artifacts"] == [] def test_workflow_representation_keeps_each_step_and_failures(): @@ -88,7 +90,7 @@ def test_workflow_representation_keeps_each_step_and_failures(): "filter": {"op": "filter", "status": "succeeded"}, } } - artifacts = with_trapi(bundle, "workflow")["trapi"]["artifacts"] + artifacts = represent_trapi(bundle, "workflow")["artifacts"] assert [(a["step_id"], a["kind"]) for a in artifacts] == [ ("one", "query"), ("one", "response"), @@ -102,10 +104,11 @@ def test_wrapped_bundle_still_replays(): "steps": [{"id": "data", "op": "literal", "value": ["X:1"]}], "outputs": {"ids": {"$ref": "data"}}, } - envelope = tools.run_query_workflow(recipe, include_trapi=True) - replayed = tools.replay_query_workflow(envelope["result"], include_trapi=True) - assert replayed["result"]["outputs"] == {"ids": ["X:1"]} - assert envelope["trapi"]["artifacts"] == [] + with execution({"trapi": True}) as records: + bundle = tools.run_query_workflow(recipe) + replayed = tools.replay_query_workflow(bundle) + assert replayed["outputs"] == {"ids": ["X:1"]} + assert records[0]["trapi"]["artifacts"] == [] def test_cli_optional_representation(capsys): @@ -117,15 +120,10 @@ def test_cli_optional_representation(capsys): "steps": [{"id": "x", "op": "literal", "value": 1}], "outputs": {"x": {"$ref": "x"}}, } - assert ( - main( - ["run-query-workflow", "--workflow", json.dumps(recipe), "--include-trapi"] - ) - == 0 - ) + assert main(["run-query-workflow", "--workflow", json.dumps(recipe)]) == 0 output = json.loads(capsys.readouterr().out) - assert output["result"]["outputs"] == {"x": 1} - assert output["trapi"]["schema_validation"] == "not_performed" + assert output["outputs"] == {"x": 1} + assert "trapi" not in output def test_mcp_optional_representation(): @@ -139,10 +137,97 @@ async def run(): "steps": [{"id": "x", "op": "literal", "value": 1}], "outputs": {"x": {"$ref": "x"}}, } - output = await registry["run_query_workflow"].run( - {"workflow": recipe, "include_trapi": True} - ) - assert output.structured_content["result"]["outputs"] == {"x": 1} - assert output.structured_content["trapi"]["artifacts"] == [] + output = await registry["run_query_workflow"].run({"workflow": recipe}) + assert output.structured_content["outputs"] == {"x": 1} + assert "trapi" not in output.structured_content asyncio.run(run()) + + +def test_disabled_fast_path_and_nested_disable(monkeypatch): + @capture_trapi("message") + def sample(): + return {"results": []} + + with execution({"trapi": True}) as records: + sample() + with execution({"trapi": False}): + sample() + with execution(): + sample() + assert len(records) == 2 + + def forbidden(*args, **kwargs): + raise AssertionError("disabled capture converted a payload") + + import importlib + + monkeypatch.setattr( + importlib.import_module("TCT.execution"), "represent_trapi", forbidden + ) + assert sample() == {"results": []} + + +def test_failure_identity_and_no_retry(): + error = ValueError("original") + + @capture_trapi("message") + def fail(): + raise error + + with execution({"trapi": True}) as records: + with pytest.raises(ValueError) as caught: + fail() + assert caught.value is error + assert len(records) == 1 + assert records[0]["status"] == "failed" + + +def test_capture_error_preserves_native_result(): + native = object() + + @capture_trapi("finder") + def sample(): + return native + + with execution({"trapi": True}) as records: + assert sample() is native + assert records[0]["status"] == "succeeded" + assert records[0]["capture_error"] == "AttributeError" + + +def test_input_capture_failure_does_not_change_execution(): + class Uncopyable: + def __deepcopy__(self, memo): + raise RuntimeError("cannot copy") + + @capture_trapi("message") + def sample(value): + return None + + with execution({"trapi": True}) as records: + assert sample(Uncopyable()) is None + assert records[0]["input_capture_error"] == "RuntimeError" + assert records[0]["status"] == "succeeded" + + +def test_async_contexts_are_isolated(): + import asyncio + + @capture_trapi("message") + def sample(value): + return {"value": value} + + async def run(value): + with execution({"trapi": True}) as records: + await asyncio.sleep(0) + sample(value) + return records + + async def both(): + return await asyncio.gather(run(1), run(2)) + + first, second = asyncio.run(both()) + assert first[0]["inputs"] == {"value": 1} + assert second[0]["inputs"] == {"value": 2} + assert first is not second From 62622b59b0a22820d39dca91b9a0a3a97c9aef7b Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Wed, 30 Sep 2026 13:30:50 -0700 Subject: [PATCH 4/8] Add scoped execution controls and optional durable recording --- TCT/execution.py | 285 +++++++++++++++++++++----- TCT/interfaces/WORKFLOWS.md | 83 ++++++-- TCT/interfaces/cli.py | 22 +- TCT/interfaces/invocation.py | 9 +- TCT/interfaces/mcp.py | 39 +++- TCT/interfaces/observability.py | 6 + TCT/interfaces/telemetry.py | 8 +- TCT/journal.py | 119 +++++++++++ TCT/translator_query.py | 3 +- tests/test_execution.py | 343 ++++++++++++++++++++++++++++++++ tests/test_mcp_contract.py | 27 ++- 11 files changed, 861 insertions(+), 83 deletions(-) create mode 100644 TCT/journal.py create mode 100644 tests/test_execution.py diff --git a/TCT/execution.py b/TCT/execution.py index 70bfc10..22eb3d7 100644 --- a/TCT/execution.py +++ b/TCT/execution.py @@ -1,78 +1,273 @@ -"""Opt-in, context-local evidence collection without changing operation returns. - -This initial collector is in-memory. It is not a durable replay journal or a -TRAPI validator. Langfuse remains independently controlled by its existing flag. -""" +"""Shared opt-in execution context for interface and decorated Python calls.""" from contextlib import contextmanager -from contextvars import ContextVar +from contextvars import ContextVar, copy_context from copy import deepcopy from functools import wraps from inspect import signature +import logging +import os +from uuid import uuid4 from .trapi_representation import represent_trapi +_settings = ContextVar("tct_execution_settings", default=None) +_run = ContextVar("tct_execution_run", default=None) +_active = ContextVar("tct_execution_call", default=None) +logger = logging.getLogger(__name__) + + +class ExecutionRecordingError(RuntimeError): + """A requested journal write failed; expose whether the operation completed.""" + + def __init__(self, message, *, operation_completed, run_id=None, call_id=None): + self.operation_completed = operation_completed + self.run_id = run_id + self.call_id = call_id + super().__init__( + f"{message}; operation_completed={operation_completed}; run_id={run_id}; call_id={call_id}" + ) + + +class ExecutionRun(list): + """In-memory observations plus an optional durable run identity.""" + + def __init__(self, run_id, journal=None): + super().__init__() + self.run_id = run_id + self.journal = journal + + +def langfuse_override(): + return (_settings.get() or {}).get("langfuse") -_collector = ContextVar("tct_trapi_collector", default=None) + +def execution_metadata(): + active = _active.get() + return ( + { + f"tct.execution.{key}": active[key] + for key in ("run_id", "call_id", "parent_call_id") + } + if active + else {} + ) + + +def validate_execution(config): + allowed = { + "trapi", + "record", + "langfuse", + "store", + "run_id", + "step_id", + "depends_on", + "cache", + } + if not isinstance(config, dict) or set(config) - allowed: + raise ValueError("Unknown execution settings") + for key in ("trapi", "record", "langfuse", "cache"): + if key in config and type(config[key]) is not bool: + raise ValueError(f"{key} must be a boolean") + if config.get("cache"): + raise ValueError("Execution caching is not implemented yet") + for key in ("store", "run_id", "step_id"): + if key in config and ( + not isinstance(config[key], str) or not config[key].strip() + ): + raise ValueError(f"{key} must be a nonempty string") + if "depends_on" in config and ( + not isinstance(config["depends_on"], list) + or not all(isinstance(x, str) and x for x in config["depends_on"]) + or len(set(config["depends_on"])) != len(config["depends_on"]) + ): + raise ValueError("depends_on must contain distinct nonempty step IDs") + return dict(config) @contextmanager def execution(config=None): - """Collect TRAPI observations with ``execution({"trapi": True})``. + """Enable independent tracing, TRAPI capture, or durable recording. - An omitted setting inherits the enclosing collector; explicit false disables - it locally. A true setting starts a fresh collection. No files are written. + Unspecified settings inherit. False explicitly disables a feature. Durable + recording requires a store; Python can generate a run_id on first activation. """ - config = {} if config is None else config - if not isinstance(config, dict) or set(config) - {"trapi"}: - raise ValueError("execution currently accepts only the 'trapi' setting") - if "trapi" in config and type(config["trapi"]) is not bool: - raise ValueError("trapi must be a boolean") - records = _collector.get() - if "trapi" in config: - records = [] if config["trapi"] else None - token = _collector.set(records) + supplied = validate_execution({} if config is None else config) + previous = _settings.get() or {} + settings = previous | supplied + if settings.get("record") and not settings.get("store"): + raise ValueError("record=true requires an explicit store") + if settings.get("depends_on") and not settings.get("record"): + raise ValueError("depends_on requires durable recording") + active = bool(settings.get("trapi") or settings.get("record")) + run = _run.get() if active else None + changed_run = any( + key in supplied and supplied[key] != previous.get(key) + for key in ("run_id", "store", "record") + ) + if active and (run is None or changed_run): + run_id = settings.get("run_id") or uuid4().hex + journal = None + if settings.get("record"): + try: + from .journal import Journal + + journal = Journal(settings["store"]) + except Exception as error: + raise ExecutionRecordingError( + "Cannot initialize journal", + operation_completed=False, + run_id=run_id, + ) from error + run = ExecutionRun(run_id, journal) + settings["run_id"] = run_id + setting_token = _settings.set(settings) + run_token = _run.set(run) try: - yield records + yield run finally: - _collector.reset(token) + _run.reset(run_token) + _settings.reset(setting_token) -def capture_trapi(kind): - """Decorate an operation, emitting evidence separately from its result.""" +@contextmanager +def interface_execution(config=None): + supplied = validate_execution({} if config is None else config) + merged = (_settings.get() or {}) | supplied + if merged.get("record") and not merged.get("run_id"): + raise ValueError("CLI/MCP durable recording requires an explicit run_id") + with execution(supplied) as run: + yield run - def decorate(function): - contract = signature(function) - @wraps(function) - def captured(*args, **kwargs): - records = _collector.get() - if records is None: - return function(*args, **kwargs) - bound = contract.bind(*args, **kwargs) +def submit_with_context(executor, function, /, *args, **kwargs): + """Submit with a separate copy of the caller's context for each worker.""" + return executor.submit(copy_context().run, function, *args, **kwargs) + + +def _finish(run, record, completed): + if run.journal: + try: + run.journal.finish(record) + except Exception as error: + raise ExecutionRecordingError( + "Cannot finalize journal; inspect before retrying", + operation_completed=completed, + run_id=run.run_id, + call_id=record["call_id"], + ) from error + + +def execute(function, args, kwargs, *, interface=None, kind=None): + """One execution boundary shared by interface wrappers and decorators.""" + run = _run.get() + settings = _settings.get() or {} + if interface is None and run is None: + enabled = settings.get("langfuse") + if enabled is False or ( + enabled is None + and os.getenv("TCT_LANGFUSE_ENABLED", "").strip().lower() + in {"", "0", "false", "no", "off"} + ): + return function(*args, **kwargs) + record = None + token = None + if run is not None: + parent = _active.get() + same_run = parent and parent["run_id"] == run.run_id + call_id = uuid4().hex + record = { + "format_version": 1, + "run_id": run.run_id, + "call_id": call_id, + "step_id": (settings.get("step_id") if not same_run else None) or call_id, + "parent_call_id": parent["call_id"] if same_run else None, + "depends_on": list(settings.get("depends_on", [])) if not same_run else [], + "operation": function.__name__, + "module": function.__module__, + "interface": interface or "python", + "status": "running", + } + try: + bound = signature(function).bind(*args, **kwargs) bound.apply_defaults() - record = { - "operation": function.__name__, - "status": "running", - } + record["inputs"] = deepcopy(dict(bound.arguments)) + except Exception as error: + record["input_capture_error"] = type(error).__name__ + if run.journal: + raise ExecutionRecordingError( + "Cannot capture inputs before execution", + operation_completed=False, + run_id=run.run_id, + call_id=call_id, + ) from error + if run.journal: try: - record["inputs"] = deepcopy(dict(bound.arguments)) + run.journal.begin(record) except Exception as error: - record["input_capture_error"] = type(error).__name__ - records.append(record) + raise ExecutionRecordingError( + str(error), + operation_completed=False, + run_id=run.run_id, + call_id=call_id, + ) from error + run.append(record) + token = _active.set(record) + elif _active.get() is not None: + token = _active.set(None) + try: + # Interfaces retain their existing tracing path. Decorated direct calls + # can also trace, but import no SDK when the feature is disabled. + from .interfaces.telemetry import observe_invocation + + with observe_invocation(function, args, kwargs, interface) as observe_result: + result = function(*args, **kwargs) + observe_result(result) + except BaseException as error: + if record is not None: + record.update(status="failed", error_type=type(error).__name__) try: - result = function(*args, **kwargs) - except BaseException as error: - record.update(status="failed", error_type=type(error).__name__) - raise + _finish(run, record, False) + except ExecutionRecordingError as recording_error: + if hasattr(error, "add_note"): + error.add_note(str(recording_error)) + logger.error("%s", recording_error) + raise + else: + if record is not None: record["status"] = "succeeded" try: - record["trapi"] = represent_trapi(result, kind) + record["output"] = deepcopy(result) + if kind and settings.get("trapi"): + record["trapi"] = represent_trapi(result, kind) except Exception as error: - # The operation has already completed; never replace its result. record["capture_error"] = type(error).__name__ - return result + if run.journal: + record["status"] = "capture_failed" + _finish(run, record, True) + raise ExecutionRecordingError( + "Operation succeeded but output capture failed", + operation_completed=True, + run_id=run.run_id, + call_id=record["call_id"], + ) from error + _finish(run, record, True) + return result + finally: + if token is not None: + _active.reset(token) + + +def capture_trapi(kind): + """Record separately from the native return; adapters avoid double wrapping.""" + + def decorate(function): + @wraps(function) + def captured(*args, **kwargs): + return execute(function, args, kwargs, kind=kind) + captured.__tct_capture_kind__ = kind return captured return decorate diff --git a/TCT/interfaces/WORKFLOWS.md b/TCT/interfaces/WORKFLOWS.md index ddf7463..573c7a3 100644 --- a/TCT/interfaces/WORKFLOWS.md +++ b/TCT/interfaces/WORKFLOWS.md @@ -169,32 +169,77 @@ invocation exposed through the workflow interpreter. 6. Build the user view: choose a returned path, walk backward through filters and joins, inspect supporting records, and export the replayable recipe. -### Optional TRAPI observation capture +### Optional execution controls The experimental `include_trapi` argument and return envelope have been removed. -The six query/finder/workflow wrappers always return native results. Their -introspected CLI/MCP signatures no longer advertise a representation flag. - -Python callers can opt into in-memory observations: +Functions retain their native results. CLI and MCP adapters each expose one +optional execution setting; ordinary library signatures are unchanged. ```python from TCT import execution from TCT.interfaces.tools import run_query_workflow -with execution({"trapi": True}) as observations: +with execution({"record": True, "store": "./evidence", "trapi": True}) as run: bundle = run_query_workflow(recipe) -# bundle is unchanged and can be replayed directly. -# observations contains inputs, operation status, and detached representations. +print(run.run_id) # Available even when the caller did not choose an ID. +# bundle remains a normal workflow bundle; run is also a list of observations. +``` + +Use `tct COMMAND ... --execution execution.json` in the CLI, or an `execution` +object alongside normal MCP arguments. An example file/object is: + +```json +{ + "record": true, + "store": "./evidence", + "run_id": "experiment-001", + "step_id": "query-001", + "depends_on": [], + "trapi": true, + "langfuse": false +} ``` -Capture is disabled by default. An omitted setting inherits an enclosing context; -explicit false temporarily disables capture. Langfuse is independently enabled -through its existing environment setting and still observes interface invocations. -This context neither writes files nor enables caching. It does not yet provide -CLI/MCP execution configuration, durable replay state, or a validated TRAPI rollup. - -Representations retain their prior limitations: provider failures and empty -results may both return None; parallel results contain only merged edges; finder -output is TRAPI-style and unvalidated. Workflow artifacts preserve individual -query requests/responses, not a compiled query for arbitrary joins. Capture -failures after execution are recorded without replacing the native return value. +CLI/MCP durable recording requires a caller-supplied `run_id`, so continuation +never depends on extracting a handle from a changed result payload. Store paths +are local to the machine running TCT. Use a new step_id for each operation; an +omitted step_id is generated. Explicit dependencies must already have succeeded +within the same run. Parent-call links describe nesting, not data dependencies. + +`record`, `trapi`, and `langfuse` are independent booleans. Unspecified settings +inherit from the enclosing Python context, and Langfuse otherwise uses the existing +`TCT_LANGFUSE_ENABLED` setting. Explicit false disables the corresponding feature. +A store path alone does not activate recording. Recording requires an explicit +store. Unknown settings and invalid types fail before execution. `cache: false` +is accepted; true is rejected until the caching phase is implemented. + +With all features disabled, no new persistence, evidence hashing, copying, or SDK +loading occurs. Existing toolkit resource caching is unchanged. Langfuse failures +remain nonfatal. The shared invocation boundary unwraps decorated tools once so +interface calls do not produce duplicate observations. Direct decorated Python +calls use that same instrumentation; undecorated direct library calls do not yet +receive automatic instrumentation. + +With only `trapi: true`, observations remain in memory. Representations are +unvalidated and retain their earlier limitations: merged provider results may +contain only edges, and finder output is TRAPI-style. No full rollup is constructed. + +Durable recording writes content-addressed JSON observations under `artifacts/` +and indexes calls in `journal.sqlite3`. Artifacts are written atomically and the +SQLite index registers a running call before invocation, then its outcome after. +Duplicate step IDs are rejected before re-execution, including interrupted steps. +This is not automatic retry, caching, or an exactly-once remote execution guarantee. +Inspect an interrupted run before choosing a new step identity. Payload conversion +uses the existing interface serializer; it is not a general Python-object restore +format. Replay reconstruction and schema-validated rollup remain later phases. + +A recording failure before invocation reports `operation_completed=False`. A +failure after a successful invocation reports `operation_completed=True`, with +run/call IDs: do not blindly retry. If the tool itself raises and journal finalization +also fails, the original exception is preserved, with a diagnostic note where +supported and a warning/error log. The index may retain a running entry for review. + +Thread-pool dispatch uses an explicit copied context for every worker. Context +reset on scope exit prevents settings leaking into unrelated calls. No files or +full responses are sent to Langfuse merely because local recording was enabled; +existing Langfuse input/output capture runs only when tracing is independently on. diff --git a/TCT/interfaces/cli.py b/TCT/interfaces/cli.py index 8e60888..5fdb2d2 100644 --- a/TCT/interfaces/cli.py +++ b/TCT/interfaces/cli.py @@ -7,6 +7,9 @@ import json import sys import types +from pathlib import Path + +from ..execution import interface_execution, ExecutionRecordingError from collections.abc import Callable from typing import Any, Union, get_args, get_origin, get_type_hints @@ -141,6 +144,12 @@ def _add_tool_parser( parser.add_argument(option, **options) + parser.add_argument( + "--execution", + type=Path, + default=None, + help="Optional execution settings JSON file (tracing and recording).", + ) parser.set_defaults(_tool=tool) @@ -171,14 +180,17 @@ def main(argv: list[str] | None = None) -> int: values = vars(namespace) tool = values.pop("_tool") command = values.pop("command") + execution_path = values.pop("execution") try: - result = invoke(tool, _interface="cli", **values) - except ToolInvocationError as error: + settings = json.loads(execution_path.read_text()) if execution_path else None + with interface_execution(settings): + try: + result = invoke(tool, _interface="cli", **values) + finally: + flush_observability() + except (ToolInvocationError, ExecutionRecordingError, ValueError, OSError) as error: print(f"{parser.prog}: {command}: {error}", file=sys.stderr) return 1 - finally: - # The CLI is short-lived, so ensure queued observations are delivered. - flush_observability() try: output = dumps_result(result) except ResultSerializationError as error: diff --git a/TCT/interfaces/invocation.py b/TCT/interfaces/invocation.py index eb537b0..d1db732 100644 --- a/TCT/interfaces/invocation.py +++ b/TCT/interfaces/invocation.py @@ -6,7 +6,7 @@ from typing import Any from .serialization import ResultSerializationError, dumps_result, to_jsonable -from .telemetry import observe_invocation +from ..execution import execute class ToolInvocationError(RuntimeError): @@ -33,10 +33,9 @@ def invoke( ) -> Any: """Run a tool once and translate its exception for the calling interface.""" try: - with observe_invocation(tool, args, kwargs, _interface) as record_result: - result = tool(*args, **kwargs) - record_result(result) - return result + kind = getattr(tool, "__tct_capture_kind__", None) + function = tool.__wrapped__ if kind is not None else tool + return execute(function, args, kwargs, interface=_interface or "shared", kind=kind) except ToolInvocationError: raise except Exception as error: diff --git a/TCT/interfaces/mcp.py b/TCT/interfaces/mcp.py index 4a918f1..fca7317 100644 --- a/TCT/interfaces/mcp.py +++ b/TCT/interfaces/mcp.py @@ -10,6 +10,9 @@ from collections.abc import Callable, Mapping from functools import wraps +from inspect import signature, Parameter + +from ..execution import interface_execution, ExecutionRecordingError from typing import Any from fastmcp import FastMCP @@ -58,8 +61,9 @@ async def on_call_tool( message = message.model_copy(update={"arguments": arguments}) context = context.copy(message=message) - with use_incoming_trace_context(metadata): - return await call_next(context) + with interface_execution(arguments.get("execution")): + with use_incoming_trace_context(metadata): + return await call_next(context) mcp.add_middleware(_TraceContextMiddleware()) @@ -73,15 +77,40 @@ def _register_tool( @wraps(tool) def invoke(*args: Any, **kwargs: Any) -> Any: try: - return invoke_tool(tool, *args, _interface="mcp", **kwargs) - except ToolInvocationError as error: + settings = kwargs.pop("execution", None) + with interface_execution(settings): + return invoke_tool(tool, *args, _interface="mcp", **kwargs) + except ( + ToolInvocationError, + ExecutionRecordingError, + ValueError, + OSError, + ) as error: raise McpError( ErrorData( code=INTERNAL_ERROR, - message=error.contextual_message, + message=error.contextual_message + if isinstance(error, ToolInvocationError) + else str(error), ) ) from error + invoke.__annotations__ = { + **tool.__annotations__, + "execution": dict[str, Any] | None, + } + original = signature(tool) + invoke.__signature__ = original.replace( + parameters=[ + *original.parameters.values(), + Parameter( + "execution", + Parameter.KEYWORD_ONLY, + default=None, + annotation=dict[str, Any] | None, + ), + ] + ) return mcp.tool()(invoke) diff --git a/TCT/interfaces/observability.py b/TCT/interfaces/observability.py index 44165ba..760fba1 100644 --- a/TCT/interfaces/observability.py +++ b/TCT/interfaces/observability.py @@ -50,6 +50,12 @@ def langfuse_enabled(environ: Mapping[str, str] | None = None) -> bool: set to an accepted true value. Langfuse credentials alone never activate instrumentation. """ + if environ is None: + from ..execution import langfuse_override + + override = langfuse_override() + if override is not None: + return override variables = os.environ if environ is None else environ configured = variables.get(_ENABLED_VARIABLE) if configured is not None: diff --git a/TCT/interfaces/telemetry.py b/TCT/interfaces/telemetry.py index 0a9d2dd..3983005 100644 --- a/TCT/interfaces/telemetry.py +++ b/TCT/interfaces/telemetry.py @@ -132,7 +132,10 @@ def observe_invocation( interface: str | None, ) -> Generator[Callable[[Any], None], None, None]: """Measure an invocation; recording failures never replace tool outcomes.""" + from ..execution import execution_metadata + metadata = { + **execution_metadata(), "tct.interface": interface or "shared", "tct.module": tool.__module__, "tct.tool": tool.__name__, @@ -149,6 +152,7 @@ def trace_input() -> Any: input_factory=trace_input, metadata=metadata, ) as observation: + def record_result(result: Any) -> None: if observation is None: return @@ -159,6 +163,8 @@ def record_result(result: Any) -> None: metadata=_payload_metadata("tct.output", output), ) except Exception as error: - logger.warning("Unable to record tool telemetry (%s)", type(error).__name__) + logger.warning( + "Unable to record tool telemetry (%s)", type(error).__name__ + ) yield record_result diff --git a/TCT/journal.py b/TCT/journal.py new file mode 100644 index 0000000..6fdc07d --- /dev/null +++ b/TCT/journal.py @@ -0,0 +1,119 @@ +"""Local immutable observations and transactional execution index (opt-in only).""" + +from contextlib import contextmanager +import hashlib +import json +import os +from pathlib import Path +import sqlite3 +import tempfile + + +class Journal: + def __init__(self, store): + self.root = Path(store).expanduser().resolve() + self.artifacts = self.root / "artifacts" + self.artifacts.mkdir(parents=True, exist_ok=True) + with self.connect() as db: + db.executescript(""" + CREATE TABLE IF NOT EXISTS runs ( + run_id TEXT PRIMARY KEY, format_version INTEGER NOT NULL); + CREATE TABLE IF NOT EXISTS calls ( + run_id TEXT NOT NULL, step_id TEXT NOT NULL, + call_id TEXT NOT NULL, parent_call_id TEXT, + operation TEXT NOT NULL, status TEXT NOT NULL, + input_hash TEXT NOT NULL, event_hash TEXT, + PRIMARY KEY(run_id, step_id)); + """) + + @contextmanager + def connect(self): + connection = sqlite3.connect(self.root / "journal.sqlite3", timeout=30) + try: + with connection: + yield connection + finally: + connection.close() + + def artifact(self, value): + from .interfaces.serialization import to_jsonable + + payload = json.dumps( + to_jsonable(value), + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode() + digest = hashlib.sha256(payload).hexdigest() + destination = self.artifacts / digest + if destination.exists(): + if destination.read_bytes() != payload: + raise ValueError("Stored artifact checksum conflict") + return digest + fd, temporary = tempfile.mkstemp(dir=self.artifacts, prefix=".pending-") + try: + with os.fdopen(fd, "wb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, destination) + directory_fd = os.open(self.artifacts, os.O_RDONLY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + finally: + if os.path.exists(temporary): + os.unlink(temporary) + return digest + + def begin(self, record): + input_hash = self.artifact(record["inputs"]) + with self.connect() as db: + db.execute("BEGIN IMMEDIATE") + db.execute("INSERT OR IGNORE INTO runs VALUES (?, 1)", (record["run_id"],)) + for dependency in record["depends_on"]: + found = db.execute( + "SELECT status FROM calls WHERE run_id=? AND step_id=?", + (record["run_id"], dependency), + ).fetchone() + if found != ("succeeded",): + raise ValueError(f"Dependency is not complete: {dependency}") + existing = db.execute( + "SELECT status FROM calls WHERE run_id=? AND step_id=?", + (record["run_id"], record["step_id"]), + ).fetchone() + if existing: + raise ValueError( + f"Step already recorded ({existing[0]}); use a new step_id or inspect the journal" + ) + db.execute( + "INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?, NULL)", + ( + record["run_id"], + record["step_id"], + record["call_id"], + record["parent_call_id"], + record["operation"], + "running", + input_hash, + ), + ) + + def finish(self, record): + event_hash = self.artifact(record) + with self.connect() as db: + cursor = db.execute( + "UPDATE calls SET status=?, event_hash=? WHERE run_id=? AND step_id=? AND call_id=? AND status=?", + ( + record["status"], + event_hash, + record["run_id"], + record["step_id"], + record["call_id"], + "running", + ), + ) + if cursor.rowcount != 1: + raise ValueError("Execution record was not running") diff --git a/TCT/translator_query.py b/TCT/translator_query.py index 0e10d64..ab22530 100644 --- a/TCT/translator_query.py +++ b/TCT/translator_query.py @@ -223,7 +223,8 @@ def parallel_api_query(query_json, select_APIs, APInames, API_predicates,max_wor with ThreadPoolExecutor(max_workers=max_workers) as executor: # copy the query_json for each API to avoid modifying the original query_json query_json_cur = deepcopy(query_json) - future_to_url = {executor.submit(query_KP, API_name_cur, query_json_cur, APInames, API_predicates): API_name_cur for API_name_cur in select_APIs} + from .execution import submit_with_context + future_to_url = {submit_with_context(executor, query_KP, API_name_cur, query_json_cur, APInames, API_predicates): API_name_cur for API_name_cur in select_APIs} for future in as_completed(future_to_url): url = future_to_url[future] diff --git a/tests/test_execution.py b/tests/test_execution.py new file mode 100644 index 0000000..ee9f7c1 --- /dev/null +++ b/tests/test_execution.py @@ -0,0 +1,343 @@ +"""Phase 2 contracts: defaults, isolation, durable outcomes and interfaces.""" + +import asyncio +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +import hashlib +import importlib +import json +import sqlite3 + +import pytest + +from TCT import execution +from TCT.execution import ExecutionRecordingError, capture_trapi, submit_with_context +from TCT.interfaces.invocation import invoke, ToolInvocationError +from TCT.interfaces import observability +from TCT.journal import Journal + + +@capture_trapi("message") +def echo(value): + return value + + +def records(store): + with sqlite3.connect(store / "journal.sqlite3") as db: + rows = db.execute( + "SELECT status, event_hash FROM calls ORDER BY rowid" + ).fetchall() + return [ + ( + status, + json.loads((store / "artifacts" / digest).read_text()) if digest else None, + ) + for status, digest in rows + ] + + +def test_disabled_has_no_storage_or_capture(monkeypatch, tmp_path): + module = importlib.import_module("TCT.execution") + monkeypatch.delenv("TCT_LANGFUSE_ENABLED", raising=False) + + def forbidden(*args, **kwargs): + raise AssertionError("disabled path performed evidence work") + + monkeypatch.setattr(module, "deepcopy", forbidden) + monkeypatch.setattr(module, "uuid4", forbidden) + with execution({"store": str(tmp_path / "unused"), "cache": False}) as run: + assert run is None + assert echo("x") == "x" + assert invoke(echo, "y") == "y" + assert not (tmp_path / "unused").exists() + + +def test_durable_native_identity_and_artifact_checksums(tmp_path): + native = {"message": {"results": []}} + with execution({"record": True, "store": str(tmp_path)}) as run: + assert invoke(echo, native) is native + saved = records(tmp_path) + assert len(run) == len(saved) == 1 + assert saved[0][0] == "succeeded" + assert saved[0][1]["output"] == native + assert saved[0][1]["run_id"] == run.run_id + for path in (tmp_path / "artifacts").iterdir(): + assert path.name == hashlib.sha256(path.read_bytes()).hexdigest() + native.clear() + assert saved[0][1]["output"] == {"message": {"results": []}} + + +def test_tracing_and_recording_are_independent(monkeypatch, tmp_path): + monkeypatch.setenv("TCT_LANGFUSE_ENABLED", "true") + with execution({"record": True, "store": str(tmp_path), "langfuse": False}): + assert not observability.langfuse_enabled() + echo(1) + with execution({"langfuse": True}) as run: + assert run is None + assert observability.langfuse_enabled() + with execution({"langfuse": False}): + assert not observability.langfuse_enabled() + assert observability.langfuse_enabled() + assert len(records(tmp_path)) == 1 + + +def test_nested_disable_and_inherit(tmp_path): + with execution({"record": True, "trapi": True, "store": str(tmp_path)}) as outer: + echo({"one": 1}) + with execution({"record": False, "trapi": False}) as disabled: + assert disabled is None + echo({"two": 2}) + with execution() as inherited: + assert inherited is outer + echo({"three": 3}) + assert len(outer) == len(records(tmp_path)) == 2 + + +def test_one_observation_and_correlated_langfuse(monkeypatch, tmp_path): + from TCT.interfaces import telemetry + + observed = [] + + class Observation: + def update(self, **kwargs): + pass + + @contextmanager + def observe(**kwargs): + observed.append(kwargs["metadata"]) + yield Observation() + + monkeypatch.setattr(telemetry, "observe_tool", observe) + with execution({"record": True, "store": str(tmp_path), "trapi": True}) as run: + invoke(echo, {"results": []}, _interface="mcp") + assert len(observed) == len(run) == len(records(tmp_path)) == 1 + assert observed[0]["tct.execution.call_id"] == run[0]["call_id"] + assert observed[0]["tct.execution.run_id"] == run.run_id + + +def test_explicit_dependencies_and_duplicate_step(tmp_path): + config = {"record": True, "store": str(tmp_path), "run_id": "shared"} + with execution(config | {"step_id": "one"}): + echo(1) + with execution(config | {"step_id": "two", "depends_on": ["one"]}): + echo(2) + with execution(config | {"step_id": "three", "depends_on": ["missing"]}): + with pytest.raises(ExecutionRecordingError, match="Dependency") as caught: + echo(3) + assert caught.value.operation_completed is False + with execution(config | {"step_id": "one"}): + with pytest.raises(ExecutionRecordingError, match="already recorded"): + echo(4) + assert len(records(tmp_path)) == 2 + + +def test_worker_propagation_and_parent_relationship(tmp_path): + @capture_trapi("message") + def parent(): + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [submit_with_context(pool, echo, i) for i in range(4)] + return [future.result() for future in futures] + + with execution({"record": True, "store": str(tmp_path)}) as run: + assert parent() == [0, 1, 2, 3] + assert len(records(tmp_path)) == 5 + root = next(r for r in run if r["operation"] == "parent") + children = [r for r in run if r["operation"] == "echo"] + assert all(r["parent_call_id"] == root["call_id"] for r in children) + assert all(r["depends_on"] == [] for r in children) + + +def test_async_runs_are_isolated(): + async def worker(value): + with execution({"trapi": True}) as run: + await asyncio.sleep(0) + echo(value) + return run + + async def both(): + return await asyncio.gather(worker(1), worker(2)) + + first, second = asyncio.run(both()) + assert first.run_id != second.run_id + assert [r["output"] for r in first] == [1] + assert [r["output"] for r in second] == [2] + + +def test_original_failure_survives_journal_failure(monkeypatch, tmp_path): + error = RuntimeError("original") + + @capture_trapi("message") + def fail(): + raise error + + def broken(*args): + raise OSError("disk unavailable") + + monkeypatch.setattr(Journal, "finish", broken) + with execution({"record": True, "store": str(tmp_path)}): + with pytest.raises(RuntimeError) as caught: + fail() + assert caught.value is error + assert records(tmp_path) == [("running", None)] + + +def test_successful_call_reports_storage_failure(monkeypatch, tmp_path): + calls = [] + + @capture_trapi("message") + def succeed(): + calls.append(1) + return 42 + + def broken(*args): + raise OSError("disk unavailable") + + monkeypatch.setattr(Journal, "finish", broken) + with execution({"record": True, "store": str(tmp_path)}): + with pytest.raises(ExecutionRecordingError) as caught: + succeed() + assert calls == [1] + assert caught.value.operation_completed is True + + +def test_pre_call_failure_does_not_execute(monkeypatch, tmp_path): + calls = [] + + def fail(*args): + raise OSError("no space") + + monkeypatch.setattr(Journal, "begin", fail) + with execution({"record": True, "store": str(tmp_path)}): + with pytest.raises(ToolInvocationError) as caught: + invoke(lambda: calls.append(1)) + assert calls == [] + assert caught.value.cause.operation_completed is False + + +@pytest.mark.parametrize( + "settings", + [ + {"record": True}, + {"cache": True}, + {"trapi": "yes"}, + {"unknown": True}, + {"depends_on": ["x"]}, + {"run_id": ""}, + {"depends_on": ["x", "x"]}, + ], +) +def test_invalid_configuration(settings): + with pytest.raises(ValueError): + with execution(settings): + pytest.fail("invalid config accepted") + + +def test_cli_recording_preserves_stdout(tmp_path, capsys): + from TCT.interfaces.cli import main + + config = tmp_path / "execution.json" + config.write_text( + json.dumps( + { + "record": True, + "store": str(tmp_path / "store"), + "run_id": "cli", + "step_id": "check", + } + ) + ) + workflow = { + "schema_version": 1, + "steps": [{"id": "x", "op": "literal", "value": 1}], + "outputs": {"x": {"$ref": "x"}}, + } + argv = ["validate-query-workflow", "--workflow", json.dumps(workflow)] + assert main(argv) == 0 + baseline = capsys.readouterr().out + assert main(argv + ["--execution", str(config)]) == 0 + assert capsys.readouterr().out == baseline + assert len(records(tmp_path / "store")) == 1 + assert main(argv + ["--execution", str(tmp_path / "missing")]) == 1 + capsys.readouterr() + + +def test_mcp_recording_keeps_native_payload(tmp_path): + from TCT.interfaces.mcp import mcp + + async def run(): + registry = await mcp.get_tools() + workflow = { + "schema_version": 1, + "steps": [{"id": "x", "op": "literal", "value": 1}], + "outputs": {"x": {"$ref": "x"}}, + } + tool = registry["validate_query_workflow"] + baseline = await tool.run({"workflow": workflow}) + recorded = await tool.run( + { + "workflow": workflow, + "execution": {"record": True, "store": str(tmp_path), "run_id": "mcp"}, + } + ) + assert baseline.structured_content == recorded.structured_content + + asyncio.run(run()) + assert len(records(tmp_path)) == 1 + + +def test_concurrent_duplicate_step_executes_once(tmp_path): + config = { + "record": True, + "store": str(tmp_path), + "run_id": "same", + "step_id": "same", + } + called = [] + + def task(): + with execution(config): + try: + invoke(lambda: called.append(1)) + return "executed" + except ToolInvocationError: + return "duplicate" + + # Initialize before contention to isolate transactional step registration. + Journal(tmp_path) + with ThreadPoolExecutor(max_workers=2) as pool: + outcomes = list(pool.map(lambda _: task(), range(2))) + assert sorted(outcomes) == ["duplicate", "executed"] + assert called == [1] + + +def test_interface_recording_requires_run_id(tmp_path): + from TCT.execution import interface_execution + + with pytest.raises(ValueError, match="run_id"): + with interface_execution({"record": True, "store": str(tmp_path)}): + pytest.fail("accepted missing ID") + assert not (tmp_path / "journal.sqlite3").exists() + + +def test_disabled_recording_does_not_reuse_parent_call_id(monkeypatch): + from TCT.interfaces import telemetry + + seen = [] + + @contextmanager + def observe(**kwargs): + seen.append(kwargs["metadata"]) + yield None + + monkeypatch.setattr(telemetry, "observe_tool", observe) + + @capture_trapi("message") + def parent(): + with execution({"trapi": False, "langfuse": True}): + echo("inner") + return "outer" + + with execution({"trapi": True}): + parent() + assert "tct.execution.call_id" in seen[0] + assert "tct.execution.call_id" not in seen[1] diff --git a/tests/test_mcp_contract.py b/tests/test_mcp_contract.py index df8d29e..4e1843f 100644 --- a/tests/test_mcp_contract.py +++ b/tests/test_mcp_contract.py @@ -246,7 +246,20 @@ def test_published_tool_names_signatures_and_docs_are_stable(): assert list(tools) == list(EXPECTED_SIGNATURES) for name, expected_signature in EXPECTED_SIGNATURES.items(): tool = tools[name] - assert str(inspect.signature(tool.fn)) == expected_signature + actual = inspect.signature(tool.fn) + assert actual.parameters["execution"].default is None + assert ( + str( + actual.replace( + parameters=[ + p + for name, p in actual.parameters.items() + if name != "execution" + ] + ) + ) + == expected_signature + ) assert tool.description == inspect.getdoc(tool.fn) assert tool.description @@ -259,6 +272,13 @@ def test_generated_mcp_input_schemas_are_stable(): schema = _without_titles(tools[name].parameters) assert schema["type"] == "object" expected = dict(properties) + expected["execution"] = { + "anyOf": [ + {"type": "object", "additionalProperties": True}, + {"type": "null"}, + ], + "default": None, + } assert schema["properties"] == expected assert schema.get("required", []) == required @@ -283,12 +303,15 @@ def test_workflow_tool_inputs_are_explicit_json_objects(): for name in ("validate_query_workflow", "run_query_workflow"): schema = _without_titles(tools[name].parameters) properties = dict(schema["properties"]) + assert properties.pop("execution")["default"] is None assert properties == { "workflow": {"type": "object", "additionalProperties": True} } assert schema["required"] == ["workflow"] schema = _without_titles(tools["replay_query_workflow"].parameters) - assert schema["properties"] == { + properties = dict(schema["properties"]) + assert properties.pop("execution")["default"] is None + assert properties == { "bundle": {"type": "object", "additionalProperties": True}, "mode": {"type": "string", "default": "snapshot"}, } From 1fbd064d6e4803cb3f94804d05d09307f57dbe73 Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Thu, 1 Oct 2026 09:10:32 -0700 Subject: [PATCH 5/8] Capture toolkit operations and full service evidence without changing returns --- TCT/TCT.py | 6 + TCT/TCT_neighborhood_finder.py | 4 + TCT/TCT_pathfinder.py | 20 +- TCT/execution.py | 28 +- TCT/interfaces/WORKFLOWS.md | 73 ++++- TCT/interfaces/invocation.py | 6 +- TCT/interfaces/tools.py | 14 +- TCT/name_resolver.py | 17 +- TCT/node_normalizer.py | 14 +- TCT/service_capture.py | 48 +++ TCT/translator_kpinfo.py | 5 +- TCT/translator_metakg.py | 38 ++- TCT/translator_query.py | 22 +- TCT/trapi_representation.py | 10 +- tests/test_toolkit_capture.py | 578 +++++++++++++++++++++++++++++++++ 15 files changed, 835 insertions(+), 48 deletions(-) create mode 100644 TCT/service_capture.py create mode 100644 tests/test_toolkit_capture.py diff --git a/TCT/TCT.py b/TCT/TCT.py index c7bb2a6..b81fdb0 100644 --- a/TCT/TCT.py +++ b/TCT/TCT.py @@ -1,3 +1,4 @@ +from .execution import capture_operation from dataclasses import dataclass from typing import Any, Optional, TypeAlias, Union @@ -356,6 +357,7 @@ def select_concept(sub_list,obj_list,metaKG): df2 = metaKG.loc[(metaKG['Subject'].isin(obj_list)) & (metaKG['Object'].isin(sub_list))] df = pd.concat([df1,df2]) return(set(list(df['Predicate']))) +@capture_operation() def sele_predicates_API(input_node1_category,input_node2_category,metaKG, APInames): ''' Selects predicates, APIs, and API URLs for the given input node categories. @@ -964,6 +966,7 @@ def to_dict(self) -> dict[str, Any]: _DEFAULT_TRANSLATOR_RESOURCES: Optional[TranslatorResources] = None +@capture_operation() def get_translator_resources(*, refresh: bool = False) -> TranslatorResources: """ Return cached Translator API metadata, loading it on first use. @@ -1072,6 +1075,7 @@ def _looks_like_curie(value: str) -> bool: return ":" in value and " " not in value +@capture_operation() def _resolve_node( value: str, *, @@ -1128,6 +1132,7 @@ def _resolve_node( ) +@capture_operation() def _resolve_nodes( values: Union[NodeInput, list[NodeInput]], *, @@ -1195,6 +1200,7 @@ def _get_resources( ) +@capture_operation() def _build_finder_result( raw_output: dict[str, Any], *, diff --git a/TCT/TCT_neighborhood_finder.py b/TCT/TCT_neighborhood_finder.py index b5d1677..00151c4 100644 --- a/TCT/TCT_neighborhood_finder.py +++ b/TCT/TCT_neighborhood_finder.py @@ -1,3 +1,4 @@ +from .execution import capture_operation # reviewed by yjzhang, 2026-08-19 from collections import Counter @@ -18,6 +19,7 @@ ) from .TCT_pathfinder import generate_score_results, build_query_graph +@capture_operation() def parse_results_for_neighborhood_finder(start_node_id:str, results:dict, start_node_categories:list|None=None, end_node_categories:list|None=None, get_node_info:bool=True, @@ -126,6 +128,7 @@ def parse_results_for_neighborhood_finder(start_node_id:str, results:dict, return output +@capture_operation() def parse_results_for_neighborhood_finder_multiple_inputs(start_node_ids:list[str], results:dict, start_node_categories:list|None=None, end_node_categories:list|None=None, get_node_info:bool=True, @@ -216,6 +219,7 @@ def parse_results_for_neighborhood_finder_multiple_inputs(start_node_ids:list[st output['knowledge_graph']['nodes'][node_id] = {'name': nn.label, 'categories': nn.types} return output +@capture_operation() def neighborhood_finder( node: Union[NodeInput, list[NodeInput]], neighbor_categories: CategoryList, diff --git a/TCT/TCT_pathfinder.py b/TCT/TCT_pathfinder.py index 435b245..862914c 100644 --- a/TCT/TCT_pathfinder.py +++ b/TCT/TCT_pathfinder.py @@ -1,3 +1,5 @@ +from .service_capture import capture_request +from .execution import capture_operation # TCT Pathfinder... # reviewed by yjzhang, 2026-08-20 import requests @@ -23,6 +25,7 @@ +@capture_operation() def format_query_json_for_pathfinder_with_constraints(subject_ids:str, object_ids:str, subject_categories=None, @@ -114,6 +117,7 @@ def format_query_json_for_pathfinder_with_constraints(subject_ids:str, return q +@capture_operation() def build_query_graph(start_node_id:str, end_node_id:str, start_node_categories=None, end_node_categories=None, constraints_path=None): """ start_node_categories and end_node_categories are lists of categories. @@ -153,6 +157,7 @@ def build_query_graph(start_node_id:str, end_node_id:str, start_node_categories= return q +@capture_operation() def generate_score_results(results:dict, method='infores'): """ Generates a score dict, and a list of "analyses". @@ -191,6 +196,7 @@ def generate_score_results(results:dict, method='infores'): return graph_scores, graph_scores_formatted +@capture_operation() def parse_results_for_pathfinder(start_node_id:str, end_node_id:str, result1:dict, result2:dict, start_node_categories=None, end_node_categories=None, get_node_info=True, @@ -314,6 +320,7 @@ def parse_results_for_pathfinder(start_node_id:str, end_node_id:str, result1:dic # define a function that uses the query_json as an template and change the ids and categories of the nodes +@capture_operation() def format_pathfinder_query(node1_id:str, node1_category:str, node2_id:str, node2_category:str) -> dict: ''' Formats a query to the Pathfinder API. @@ -363,6 +370,7 @@ def format_pathfinder_query(node1_id:str, node1_category:str, node2_id:str, node return query_json +@capture_operation() def query_aragorn_pathfinder(node1_id:str, node1_category:str, node2_id:str, node2_category:str) -> str: """ This queries the ARAGORN Pathfinder API. @@ -380,10 +388,11 @@ def query_aragorn_pathfinder(node1_id:str, node1_category:str, node2_id:str, nod """ aragorn_endpoint = service_url("aragorn") query_current = format_pathfinder_query(node1_id, node1_category, node2_id, node2_category) - response = requests.post(aragorn_endpoint, json=query_current) + response = capture_request(requests.post, aragorn_endpoint, json=query_current) return response +@capture_operation() def query_aragorn_pathfinder_with_constraints(node1_id:str, node1_category:str, node2_id:str, node2_category:str, constraints:list) -> str: """ This queries the ARAGORN Pathfinder API with a list of constraints. @@ -408,9 +417,10 @@ def query_aragorn_pathfinder_with_constraints(node1_id:str, node1_category:str, object_categories=node2_category, constraints=constraints ) - response = requests.post(aragorn_endpoint, json=query_current) + response = capture_request(requests.post, aragorn_endpoint, json=query_current) return response +@capture_operation() def query_arax_pathfinder(node1_id:str, node1_category:str, node2_id:str, node2_category:str) -> str: """ This queries the ARAX Pathfinder API. @@ -428,9 +438,10 @@ def query_arax_pathfinder(node1_id:str, node1_category:str, node2_id:str, node2_ """ ARAX_endpoint = service_url("arax") query_current = format_pathfinder_query(node1_id, node1_category, node2_id, node2_category) - response = requests.post(ARAX_endpoint, json=query_current) + response = capture_request(requests.post, ARAX_endpoint, json=query_current) return response +@capture_operation() def query_arax_pathfinder_with_constraints(node1_id:str, node1_category:str, node2_id:str, node2_category:str, constraints:list) -> str: """ This queries the ARAX Pathfinder API with a list of constraints. @@ -455,9 +466,10 @@ def query_arax_pathfinder_with_constraints(node1_id:str, node1_category:str, nod object_categories=node2_category, constraints=constraints ) - response = requests.post(ARAX_endpoint, json=query_current) + response = capture_request(requests.post, ARAX_endpoint, json=query_current) return response +@capture_operation() def query_TCT_pathfinder( start: NodeInput, end: NodeInput, diff --git a/TCT/execution.py b/TCT/execution.py index 22eb3d7..f1e3bb3 100644 --- a/TCT/execution.py +++ b/TCT/execution.py @@ -239,6 +239,10 @@ def execute(function, args, kwargs, *, interface=None, kind=None): record["status"] = "succeeded" try: record["output"] = deepcopy(result) + if "inputs" in record: + record["inputs_after"] = deepcopy(dict(bound.arguments)) + if record.get("observation_errors"): + raise ValueError("Incomplete service observation") if kind and settings.get("trapi"): record["trapi"] = represent_trapi(result, kind) except Exception as error: @@ -259,7 +263,7 @@ def execute(function, args, kwargs, *, interface=None, kind=None): _active.reset(token) -def capture_trapi(kind): +def capture_trapi(kind=None): """Record separately from the native return; adapters avoid double wrapping.""" def decorate(function): @@ -268,6 +272,28 @@ def captured(*args, **kwargs): return execute(function, args, kwargs, kind=kind) captured.__tct_capture_kind__ = kind + captured.__tct_capture__ = True return captured return decorate + + +# The same boundary also captures operations without a TRAPI result adapter. +capture_operation = capture_trapi + + +def observation_target(): + """Internal capture sink; disabled execution must not inspect responses.""" + return _active.get() if _run.get() is not None else None + + +def observe_evidence(kind, **data): + """Attach explicitly supplied evidence; never infer dependencies from nesting.""" + target = observation_target() + if target is not None: + try: + target.setdefault("observations", []).append( + {"kind": kind, **deepcopy(data)} + ) + except Exception as error: + target.setdefault("observation_errors", []).append(type(error).__name__) diff --git a/TCT/interfaces/WORKFLOWS.md b/TCT/interfaces/WORKFLOWS.md index 573c7a3..ef79251 100644 --- a/TCT/interfaces/WORKFLOWS.md +++ b/TCT/interfaces/WORKFLOWS.md @@ -155,11 +155,11 @@ invocation exposed through the workflow interpreter. ## Next engineering milestones -1. Record name resolution and normalization as explicit versioned operations, - retaining rejected candidates and selection criteria. Version 1 expects - resolved identifiers or caller-supplied literal data. -2. Instrument the existing neighborhood/path finder internals using the same - evidence model, including per-provider failures and actual query rewrites. +1. Promote captured name resolution and normalization into workflow operations + with explicit references. Version 1 recipes still expect resolved identifiers + or caller-supplied literal data. +2. Define explicit data dependencies and query-graph mappings for captured + neighborhood/path finder operations, provider failures and query rewrites. 3. Add operations over TRAPI result bindings, preserving analysis/support graphs while assembling paths; distinguish inference from retrieved assertions. 4. Store large immutable artifacts by reference and expose inspection tools that @@ -217,12 +217,15 @@ With all features disabled, no new persistence, evidence hashing, copying, or SD loading occurs. Existing toolkit resource caching is unchanged. Langfuse failures remain nonfatal. The shared invocation boundary unwraps decorated tools once so interface calls do not produce duplicate observations. Direct decorated Python -calls use that same instrumentation; undecorated direct library calls do not yet -receive automatic instrumentation. +calls use that same instrumentation. All 18 working shared tools and their +resolution, discovery, query, and finder conversion helpers are instrumented. +Other legacy library operations are outside this capture surface. The nineteenth +shared endpoint, `trapi_query_endpoint`, remains explicitly unsupported. With only `trapi: true`, observations remain in memory. Representations are -unvalidated and retain their earlier limitations: merged provider results may -contain only edges, and finder output is TRAPI-style. No full rollup is constructed. +unvalidated: native merged provider results still contain only edges, and finder +output is TRAPI-style. Full provider bodies are retained separately in execution +observations. No full rollup is constructed. Durable recording writes content-addressed JSON observations under `artifacts/` and indexes calls in `journal.sqlite3`. Artifacts are written atomically and the @@ -243,3 +246,55 @@ Thread-pool dispatch uses an explicit copied context for every worker. Context reset on scope exit prevents settings leaking into unrelated calls. No files or full responses are sent to Langfuse merely because local recording was enabled; existing Langfuse input/output capture runs only when tracing is independently on. + +### Phase 3 capture coverage + +`capture_operation()` uses the same execution boundary as `capture_trapi(kind)`; +it does not add parameters or change native results. Interface adapters unwrap +this boundary once. Nested library operations have their own observations and, +when tracing is enabled, their own spans. Trace counts therefore increase even +though the number of public tool calls and service requests does not. + +Each operation records bound inputs before execution, inputs after successful +execution (including legacy mutations), and its native output or exception type. +Buffered service requests retain the requested URL/arguments, HTTP status, exact +body bytes as base64, and decoded JSON when available. Recording reads an already +buffered body; it neither repeats the request nor calls `response.json()` again. +This preserves normalization equivalences and unmatched identifiers, name lookup +candidates and scores, discovery metadata, and full provider envelopes including +nodes, bindings, analyses, qualifiers, sources, and supporting graphs. + +Transport exceptions, non-200 responses, malformed JSON, and valid empty results +remain distinguishable in evidence even where a legacy function returns `None`. +Provider merging records inputs, exclusions, and overwritten edge IDs. Query +optimization, provider selection, finder assembly and scoring retain their actual +input/output snapshots; their existing mutations and completion-order behavior +are preserved. A requested recording failure is surfaced instead of being swallowed +as a provider failure by parallel query execution. + +Workflow execution and snapshot replay continue to use the existing evidence +bundle, with explicit dependencies, filter decisions and join lineage. The wrapper +captures that bundle without introducing a second streaming-response reader or +changing the workflow's size limits. Elsewhere, parent-call links remain nesting +only: snapshots do not constitute complete data lineage or query-graph mappings. +Deterministic semantic composition and schema validation belong to Phase 4. + +Capture starts when enabled. Resource-cache hits record the cached native value; +they cannot recover the original HTTP bodies from a fetch made before capture. +No extra fetch is performed to fill that gap. Missing/incompatible buffered bodies +produce explicit capture diagnostics; durable recording reports a completed-call +capture failure rather than silently claiming complete evidence. + +Compatibility fixtures in `tests/test_toolkit_capture.py` cover all 18 working +boundaries across Python, shared invocation, CLI and registered MCP adapters. +Boundary fixtures isolate delegation, return identity, mutations and exceptions; +synthetic service fixtures execute real normalization, lookup, discovery, metadata +updates, query pruning, provider merging, and finder assembly. The nineteenth +endpoint's existing `TypeError` is retained. These fixtures are not biological +goldens, fresh model evaluations, or tests of MCP's stdio transport. + +Phase 3 acceptance: 235 offline tests passed, including 71 new capture checks. +The single-NLQ frozen benchmark matched its native-output and final-answer goldens +through Python, CLI and registered MCP. A durable Python run retained 11 operation +records and all four service bodies while issuing the same four requests. Live +service tests, real-agent evaluations, semantic rollup and caching were not run. diff --git a/TCT/interfaces/invocation.py b/TCT/interfaces/invocation.py index d1db732..2e676ca 100644 --- a/TCT/interfaces/invocation.py +++ b/TCT/interfaces/invocation.py @@ -34,8 +34,10 @@ def invoke( """Run a tool once and translate its exception for the calling interface.""" try: kind = getattr(tool, "__tct_capture_kind__", None) - function = tool.__wrapped__ if kind is not None else tool - return execute(function, args, kwargs, interface=_interface or "shared", kind=kind) + function = tool.__wrapped__ if getattr(tool, "__tct_capture__", False) else tool + return execute( + function, args, kwargs, interface=_interface or "shared", kind=kind + ) except ToolInvocationError: raise except Exception as error: diff --git a/TCT/interfaces/tools.py b/TCT/interfaces/tools.py index aae51f7..733037b 100644 --- a/TCT/interfaces/tools.py +++ b/TCT/interfaces/tools.py @@ -28,10 +28,11 @@ query_KP, ) from ..trapi import query as trapi_query -from ..execution import capture_trapi +from ..execution import capture_trapi, capture_operation from ..workflow import replay_workflow, run_workflow, validate_workflow +@capture_operation() def get_translator_resources() -> Any: """Load the Translator resources used by the finder tools. @@ -42,6 +43,7 @@ def get_translator_resources() -> Any: return _get_translator_resources() +@capture_operation() def name_lookup( query: str, return_top_response: bool = True, @@ -62,6 +64,7 @@ def name_lookup( return lookup(query, return_top_response, return_synonyms) +@capture_operation() def get_name_synonyms(query: str) -> Any: """Return synonyms and Translator node information for a CURIE. @@ -74,6 +77,7 @@ def get_name_synonyms(query: str) -> Any: return synonyms(query) +@capture_operation() def batch_name_lookup( strings: list[str], size: int = 25, @@ -96,6 +100,7 @@ def batch_name_lookup( return batch_lookup(strings, size, return_top_response, return_synonyms) +@capture_operation() def normalize_nodes( query: str | list[str], return_equivalent_identifiers: bool = False, @@ -123,6 +128,7 @@ def normalize_nodes( ) +@capture_operation() def get_kp_info() -> Any: """Return SmartAPI information for Translator Knowledge Providers. @@ -133,6 +139,7 @@ def get_kp_info() -> Any: return get_translator_kp_info() +@capture_operation() def get_metakg_data(api_names: dict[str, str]) -> Any: """Return MetaKG metadata for a set of Knowledge Providers. @@ -145,6 +152,7 @@ def get_metakg_data(api_names: dict[str, str]) -> Any: return get_KP_metadata(api_names) +@capture_operation() def add_custom_api_to_metakg( api_names: dict[str, str], metakg_df: Any, @@ -179,6 +187,7 @@ def add_custom_api_to_metakg( ) +@capture_operation() def add_plover_apis_to_metakg( api_names: dict[str, str], metakg_df: Any, @@ -195,6 +204,7 @@ def add_plover_apis_to_metakg( return add_plover_API(api_names, metakg_df) +@capture_operation() def get_api_predicates() -> Any: """Return the predicates supported by Translator APIs. @@ -205,6 +215,7 @@ def get_api_predicates() -> Any: return get_translator_API_predicates() +@capture_operation() def optimize_query_for_api( query_json: dict[str, Any], api_name: str, @@ -350,6 +361,7 @@ def path_finder( return result +@capture_operation() def validate_query_workflow(workflow: dict[str, Any]) -> dict[str, Any]: """Validate a composable query recipe without contacting any providers. diff --git a/TCT/name_resolver.py b/TCT/name_resolver.py index d4ebb89..07bdabd 100644 --- a/TCT/name_resolver.py +++ b/TCT/name_resolver.py @@ -3,6 +3,9 @@ API docs: https://name-lookup.ci.transltr.io/docs """ +from .service_capture import capture_request +from .execution import capture_operation + # reviewed by yjzhang, 2026-08-19 import urllib.parse @@ -15,11 +18,12 @@ def status() -> str: """ Returns the status of the Name Resolver API. """ - response = requests.get(urllib.parse.urljoin(service_url("name_resolver"), "status")) + response = capture_request(requests.get, urllib.parse.urljoin(service_url("name_resolver"), "status")) response.raise_for_status() return response.json() +@capture_operation() def lookup(query: str, return_top_response:bool=True, return_synonyms:bool=False, limit:int=10, **kwargs) -> TranslatorNode | list[TranslatorNode]: """ A wrapper around the `lookup` api endpoint. Given a query string, this returns a TranslatorNode object or a list of TranslatorNode objects corresponding to the given name. @@ -60,7 +64,7 @@ def lookup(query: str, return_top_response:bool=True, return_synonyms:bool=False # set autocomplete to be false by default if 'autocomplete' not in kwargs: kwargs['autocomplete'] = False - response = requests.get(path, params={'string': query, 'limit': limit, **kwargs}) + response = capture_request(requests.get, path, params={'string': query, 'limit': limit, **kwargs}) if response.status_code == 200: result = response.json() if len(result) == 0: @@ -78,6 +82,7 @@ def lookup(query: str, return_top_response:bool=True, return_synonyms:bool=False raise requests.RequestException('Response from server had error, code ' + str(response.status_code) + ' ' + str(response)) +@capture_operation() def synonyms(query: str|list, **kwargs) -> dict[str, TranslatorNode]: """ A wrapper around the `synonyms` api endpoint. Given a CURIE or a list of CURIEs, this returns a dict of CURIE id : TranslatorNode for all synonyms for the given query. @@ -95,7 +100,7 @@ def synonyms(query: str|list, **kwargs) -> dict[str, TranslatorNode]: """ path = urllib.parse.urljoin(service_url("name_resolver"), 'synonyms') # set autocomplete to be false by default - response = requests.get(path, params={'preferred_curies': query, **kwargs}) + response = capture_request(requests.get, path, params={'preferred_curies': query, **kwargs}) if response.status_code == 200: result = response.json() if len(result) == 0: @@ -121,6 +126,7 @@ def chunk_list(data:list, size:int) -> list: return chunks +@capture_operation() def batch_lookup(strings:list[str], size: int=25, return_top_response:bool=True, return_synonyms:bool=False, **kwargs) -> dict[str, TranslatorNode|list[TranslatorNode]]: """ A wrapper around the `bulk-lookup` api endpoint. Given a list of query strings, this returns a TranslatorNode object or a list of TranslatorNode objects corresponding to the given name. @@ -156,7 +162,7 @@ def batch_lookup(strings:list[str], size: int=25, return_top_response:bool=True, "strings": chunk, **kwargs } - response = requests.post(path, json = payload) + response = capture_request(requests.post, path, json = payload) if response.status_code == 200: result = response.json() if(len(result) == 0): @@ -180,6 +186,7 @@ def batch_lookup(strings:list[str], size: int=25, return_top_response:bool=True, return curies +@capture_operation() def batch_synonyms(strings:list[str], size:int=50, **kwargs) -> dict[str, TranslatorNode]: """ A wrapper around the `synonyms` API endpoint, using POST. Given a list of CURIEs, this returns a dict of CURIE:TranslatorNode, where each TranslatorNode contains all synonyms for the given CURIE. @@ -200,7 +207,7 @@ def batch_synonyms(strings:list[str], size:int=50, **kwargs) -> dict[str, Transl curies = {} for chunk in chunks: # set autocomplete to be false by default - response = requests.post(path, json={'preferred_curies': chunk, **kwargs}) + response = capture_request(requests.post, path, json={'preferred_curies': chunk, **kwargs}) if response.status_code == 200: result = response.json() if len(result) == 0: diff --git a/TCT/node_normalizer.py b/TCT/node_normalizer.py index 2eefc6b..04e83f2 100644 --- a/TCT/node_normalizer.py +++ b/TCT/node_normalizer.py @@ -3,6 +3,9 @@ API docs: https://nodenorm.transltr.io/docs """ +from .service_capture import capture_request +from .execution import capture_operation + # reviewed by yjzhang, 2026-08-19 import urllib.parse @@ -15,11 +18,12 @@ def status(): """ Returns the status of the Node Normalizer API. """ - response = requests.get(urllib.parse.urljoin(service_url("node_normalizer"), "status")) + response = capture_request(requests.get, urllib.parse.urljoin(service_url("node_normalizer"), "status")) response.raise_for_status() return response.json() +@capture_operation() def get_normalized_nodes(query: str | list[str], return_equivalent_identifiers:bool=False, mode:str='get', @@ -60,9 +64,9 @@ def get_normalized_nodes(query: str | list[str], json_query = query if len(json_query) == 0: return {} - response = requests.post(path, json={'curies': json_query, **kwargs}) + response = capture_request(requests.post, path, json={'curies': json_query, **kwargs}) else: - response = requests.get(path, params={'curie': query, **kwargs}) + response = capture_request(requests.get, path, params={'curie': query, **kwargs}) if response.status_code == 200: result = response.json() normalized_dict = {} @@ -96,6 +100,7 @@ def get_normalized_nodes(query: str | list[str], raise requests.RequestException('Response from server had error, code ' + str(response.status_code)) +@capture_operation() def get_preferred_names(id_list:list[str], batch_limit=500, **kwargs) -> dict[str, str]: """ Converts a list of CURIEs to their preferred names using NodeNorm. This calls get_normalized_nodes. @@ -135,6 +140,7 @@ def get_preferred_names(id_list:list[str], batch_limit=500, **kwargs) -> dict[st return name_map +@capture_operation() def ID_convert_to_preferred_name_nodeNormalizer(id_list: list[str]) -> dict[str, str]: ''' Convert a list of CURIEs to their preferred names using NodeNorm. @@ -162,7 +168,7 @@ def ID_convert_to_preferred_name_nodeNormalizer(id_list: list[str]) -> dict[str, # print(f"id_sublist: {id_sublist}") # Query NodeNorm with https://nodenorm.transltr.io/docs#/default/get_normalized_node_handler_get_normalized_nodes_get - response = requests.post(nodenorm_base_url + '/get_normalized_nodes', json={ + response = capture_request(requests.post, nodenorm_base_url + '/get_normalized_nodes', json={ "curies": id_sublist, "description": False, # Change to True if you want descriptions from any identifiers we know about. "conflate": NODENORM_GENE_PROTEIN_CONFLATION, diff --git a/TCT/service_capture.py b/TCT/service_capture.py new file mode 100644 index 0000000..9d0459a --- /dev/null +++ b/TCT/service_capture.py @@ -0,0 +1,48 @@ +"""Observe existing buffered HTTP calls without replacing their response objects. + +Workflow streaming queries retain their own bounded evidence implementation. +These observations describe transport, not successful biomedical answers. +""" + +from base64 import b64encode +from copy import deepcopy +import json + +from .execution import observation_target + + +def capture_request(request, /, url, **kwargs): + target = observation_target() + if target is None: + return request(url, **kwargs) + observation = {"kind": "service", "status": "pending"} + target.setdefault("observations", []).append(observation) + try: + observation["request"] = { + "method": getattr(request, "__name__", "unknown"), + "url": url, + "arguments": deepcopy(kwargs), + } + except Exception as error: + target.setdefault("observation_errors", []).append(type(error).__name__) + try: + response = request(url, **kwargs) + except BaseException as error: + observation.update(status="transport_error", error_type=type(error).__name__) + raise + try: + observation.update(status="received", status_code=response.status_code) + # All call sites using this helper request buffered responses. Never + # invoke json() again: decoding it twice can alter custom responses. + body = response.content + if not isinstance(body, bytes): + raise TypeError("Response has no buffered byte body") + observation["body_base64"] = b64encode(body).decode("ascii") + try: + observation["body_json"] = json.loads(body) + except (ValueError, UnicodeError): + observation["body_json_status"] = "not_json" + except Exception as error: + observation["capture_error"] = type(error).__name__ + target.setdefault("observation_errors", []).append(type(error).__name__) + return response diff --git a/TCT/translator_kpinfo.py b/TCT/translator_kpinfo.py index b095ec9..f9b2b49 100644 --- a/TCT/translator_kpinfo.py +++ b/TCT/translator_kpinfo.py @@ -1,3 +1,5 @@ +from .service_capture import capture_request +from .execution import capture_operation # used May 30, 2025 @@ -24,6 +26,7 @@ def _select_provider_url( """This is the root URL for the resource.""" +@capture_operation() def get_translator_kp_info() -> tuple[pd.DataFrame, dict[str, str]]: """ Get the SmartAPI Translator KP info from the smart-api.info API. @@ -45,7 +48,7 @@ def get_translator_kp_info() -> tuple[pd.DataFrame, dict[str, str]]: """ # Get x-bte smartapi specs url = service_url("smartapi_registry") - response = requests.get(url) + response = capture_request(requests.get, url) try: response.raise_for_status() except Exception: diff --git a/TCT/translator_metakg.py b/TCT/translator_metakg.py index 497fcc4..1c638aa 100644 --- a/TCT/translator_metakg.py +++ b/TCT/translator_metakg.py @@ -1,3 +1,5 @@ +from .service_capture import capture_request +from .execution import capture_operation import requests import json import pandas as pd @@ -41,6 +43,7 @@ def find_link(name, use_new_url=True): return url +@capture_operation() def get_KP_metadata(APInames:dict[str, str], use_new_url=True) -> pd.DataFrame: ''' This function is used to get the metadata of the KPs in the APInames dictionary. @@ -72,15 +75,15 @@ def get_KP_metadata(APInames:dict[str, str], use_new_url=True) -> pd.DataFrame: for KP in APInames.keys(): json_text ={} if KP == "RTX KG2 - TRAPI 1.5.0": - text =requests.get("https://smart-api.info/api/metakg/consolidated?size=20&q=%28api.x-translator.component%3AKP+AND+api.name%3ARTX+KG2+%5C-+TRAPI+1%5C.4%5C.0%29").text # This works for the previous version + text =capture_request(requests.get, "https://smart-api.info/api/metakg/consolidated?size=20&q=%28api.x-translator.component%3AKP+AND+api.name%3ARTX+KG2+%5C-+TRAPI+1%5C.4%5C.0%29").text # This works for the previous version json_text = json.loads(text) else: - text = requests.get(find_link(KP, use_new_url=use_new_url)).text + text = capture_request(requests.get, find_link(KP, use_new_url=use_new_url)).text json_text = json.loads(text) if 'hits' not in json_text: if use_new_url: #print(KP, '- no hits found in new metakg URL, trying old URL pattern') - text = requests.get(find_link(KP, use_new_url=False)).text + text = capture_request(requests.get, find_link(KP, use_new_url=False)).text json_text = json.loads(text) else: print(KP, '- no hits found') @@ -97,6 +100,7 @@ def get_KP_metadata(APInames:dict[str, str], use_new_url=True) -> pd.DataFrame: return(result_df) +@capture_operation() def add_new_API_for_query(APInames:dict[str, str], metaKG:pd.DataFrame, newAPIname:str, newAPIurl:str, newAPIpredicate:str, newAPIsubject:str, newAPIobject:str): ''' This function is used to add a new API beyond the current list of APIs for query @@ -138,6 +142,7 @@ def add_new_API_for_query(APInames:dict[str, str], metaKG:pd.DataFrame, newAPIna return APInames, metaKG +@capture_operation() def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[str, str], pd.DataFrame]: ''' This function is used to add the Plover APIs developed by the CATRAX team to the APInames and metaKG. @@ -171,7 +176,7 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s url = CI_URL_PREFIX + 'BigGIM_DrugResponse_PerformancePhase/meta_knowledge_graph' url_dev = DEV_URL_PREFIX + 'BigGIM_DrugResponse_PerformancePhase/meta_knowledge_graph' try: - response = requests.get(url, timeout=5) + response = capture_request(requests.get, url, timeout=5) if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): @@ -182,7 +187,7 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s data["edges"][i]['object']) elif response.status_code == 404: # Try the dev URL if the CI URL returns a 404 - response_dev = requests.get(url_dev, timeout=5) + response_dev = capture_request(requests.get, url_dev, timeout=5) if response_dev.status_code == 200: data = response_dev.json() for i in range(len(data["edges"])): @@ -202,14 +207,14 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s url = CI_URL_PREFIX + 'PharmacogenomicsKG/meta_knowledge_graph' url_dev = DEV_URL_PREFIX + 'PharmacogenomicsKG/meta_knowledge_graph' try: - response = requests.get(url) + response = capture_request(requests.get, url) if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): APInames, metaKG = add_new_API_for_query(APInames, metaKG, "CATRAX Pharmacogenomics KP - TRAPI 1.5.0", CI_URL_PREFIX + "PharmacogenomicsKG/query", data["edges"][i]['predicate'], data["edges"][i]['subject'], data["edges"][i]['object']) elif response.status_code == 404: # Try the dev URL if the CI URL returns a 404 - response_dev = requests.get(url_dev, timeout=5) + response_dev = capture_request(requests.get, url_dev, timeout=5) if response_dev.status_code == 200: data = response_dev.json() for i in range(len(data["edges"])): @@ -226,14 +231,14 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s url_dev = DEV_URL_PREFIX + 'ctkp/meta_knowledge_graph' try: - response = requests.get(url) + response = capture_request(requests.get, url) if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): APInames, metaKG = add_new_API_for_query(APInames, metaKG, "Clinical Trials KP - TRAPI 1.5.0", CI_URL_PREFIX + "ctkp/query", data["edges"][i]['predicate'], data["edges"][i]['subject'], data["edges"][i]['object']) elif response.status_code == 404: # Try the dev URL if the CI URL returns a 404 - response_dev = requests.get(url_dev, timeout=5) + response_dev = capture_request(requests.get, url_dev, timeout=5) if response_dev.status_code == 200: data = response_dev.json() for i in range(len(data["edges"])): @@ -250,14 +255,14 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s #url = 'https://multiomics.rtx.ai:9990/dakp/meta_knowledge_graph' try: - response = requests.get(url) + response = capture_request(requests.get, url) if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): APInames, metaKG = add_new_API_for_query(APInames, metaKG, "Drug Approvals KP - TRAPI 1.5.0", CI_URL_PREFIX + "dakp/query", data["edges"][i]['predicate'], data["edges"][i]['subject'], data["edges"][i]['object']) elif response.status_code == 404: # Try the dev URL if the CI URL returns a 404 - response_dev = requests.get(url_dev, timeout=5) + response_dev = capture_request(requests.get, url_dev, timeout=5) if response_dev.status_code == 200: data = response_dev.json() for i in range(len(data["edges"])): @@ -274,14 +279,14 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s url_dev = DEV_URL_PREFIX + 'mokp/meta_knowledge_graph' #url = 'https://multiomics.rtx.ai:9990/mokp/meta_knowledge_graph' try: - response = requests.get(url) + response = capture_request(requests.get, url) if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): APInames, metaKG = add_new_API_for_query(APInames, metaKG, "Multiomics KP - TRAPI 1.5.0", CI_URL_PREFIX + "mokp/query", data["edges"][i]['predicate'], data["edges"][i]['subject'], data["edges"][i]['object']) elif response.status_code == 404: # Try the dev URL if the CI URL returns a 404 - response_dev = requests.get(url_dev, timeout=5) + response_dev = capture_request(requests.get, url_dev, timeout=5) if response_dev.status_code == 200: data = response_dev.json() for i in range(len(data["edges"])): @@ -296,14 +301,14 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s url = CI_URL_PREFIX + 'mbkp/meta_knowledge_graph' url_dev = DEV_URL_PREFIX + 'mbkp/meta_knowledge_graph' try: - response = requests.get(url) + response = capture_request(requests.get, url) if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): APInames, metaKG = add_new_API_for_query(APInames, metaKG, "Microbiome KP - TRAPI 1.5.0", CI_URL_PREFIX + "mbkp/query", data["edges"][i]['predicate'], data["edges"][i]['subject'], data["edges"][i]['object']) elif response.status_code == 404: # Try the dev URL if the CI URL returns a 404 - response_dev = requests.get(url_dev, timeout=5) + response_dev = capture_request(requests.get, url_dev, timeout=5) if response_dev.status_code == 200: data = response_dev.json() for i in range(len(data["edges"])): @@ -318,7 +323,7 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s url = 'https://kg2cploverdb.ci.transltr.io/meta_knowledge_graph' try: - response = requests.get(url) + response = capture_request(requests.get, url) if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): @@ -330,6 +335,7 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s return APInames, metaKG +@capture_operation() def load_translator_resources(use_new_metakg_url=False): """ Load the necessary resources for the Translator. diff --git a/TCT/translator_query.py b/TCT/translator_query.py index ab22530..d74984b 100644 --- a/TCT/translator_query.py +++ b/TCT/translator_query.py @@ -1,9 +1,12 @@ +from .service_capture import capture_request +from .execution import capture_operation, observe_evidence, observation_target, ExecutionRecordingError import requests from copy import deepcopy import pandas from TCT import translator_metakg from TCT import translator_kpinfo +@capture_operation() def get_translator_API_predicates() -> tuple[dict, pandas.DataFrame, dict]: ''' Get the predicates supported by each API. @@ -65,6 +68,7 @@ def build_attribute_constraint(attribute_id:str, operator:str, value, name=None, return output +@capture_operation() def format_query_json(subject_ids:list[str], object_ids:list[str]|None = None, subject_categories:list[str]|None = None, @@ -133,6 +137,7 @@ def format_query_json(subject_ids:list[str], +@capture_operation() def optimize_query_json(query_json, API_name_cur, API_predicates): ''' Optimize the query JSON by removing predicates that are not supported by the selected APIs. @@ -168,6 +173,7 @@ def optimize_query_json(query_json, API_name_cur, API_predicates): return query_json_cur +@capture_operation() def query_KP(API_name_cur, query_json, APInames, API_predicates): """ Query an individual API with a TRAPI 1.5.0 query JSON, @@ -178,7 +184,7 @@ def query_KP(API_name_cur, query_json, APInames, API_predicates): query_copy = deepcopy(query_json) # optimize on our private copy query_json_cur = optimize_query_json(query_copy, API_name_cur, API_predicates) - response = requests.post(API_url_cur, json=query_json_cur) + response = capture_request(requests.post, API_url_cur, json=query_json_cur) if response.status_code == 200: result = response.json().get("message", {}) kg = result.get("knowledge_graph", {}) @@ -193,6 +199,7 @@ def query_KP(API_name_cur, query_json, APInames, API_predicates): #print(f"{API_name_cur}: Warning Code: {response.status_code}") return None +@capture_operation() def parallel_api_query(query_json, select_APIs, APInames, API_predicates,max_workers=1): ''' Queries multiple APIs in parallel and merges the results into a single knowledge graph. @@ -232,8 +239,13 @@ def parallel_api_query(query_json, select_APIs, APInames, API_predicates,max_wor data = future.result() if 'knowledge_graph' in data: result.append(data) - except Exception: + observe_evidence("merge_input", provider=url, message=data) + except ExecutionRecordingError: + # Requested durable capture failure must not become an empty KP. + raise + except Exception as error: no_results_returned.append(url) + observe_evidence("merge_exclusion", provider=url, error_type=type(error).__name__) included_KP_ID = [] for i in range(0,len(result)): @@ -245,6 +257,12 @@ def parallel_api_query(query_json, select_APIs, APInames, API_predicates,max_wor result_merged = {} for i in included_KP_ID: + if observation_target() is not None: + observe_evidence( + "edge_merge", input_index=i, + overwritten_edge_ids=sorted(set(result_merged).intersection(result[i]['knowledge_graph']['edges'])), + policy="later_completion_overwrites", + ) result_merged = {**result_merged, **result[i]['knowledge_graph']['edges']} len(result_merged) diff --git a/TCT/trapi_representation.py b/TCT/trapi_representation.py index 846b76e..eb54cc4 100644 --- a/TCT/trapi_representation.py +++ b/TCT/trapi_representation.py @@ -25,7 +25,10 @@ def add(artifact_kind, content, **metadata): ) else: add("response", {"message": result}) - limitations.append("Original HTTP response envelope was not retained.") + limitations.append( + "This result-only representation omits the HTTP envelope; " + "instrumented provider calls retain it in service observations." + ) elif kind == "finder": add("response", {"message": result.raw}) limitations.append( @@ -35,8 +38,9 @@ def add(artifact_kind, content, **metadata): elif kind == "edges": add("knowledge_graph_fragment", {"edges": result}) limitations.append( - "Only merged edges remain; nodes, result bindings, auxiliary graphs " - "and provider failures cannot be recovered. This is not a TRAPI response." + "This result contains only merged edges. Consult provider observations " + "for nodes, bindings, supporting graphs and failures; no rollup is " + "constructed here. This is not a TRAPI response." ) elif kind == "workflow": for step_id, step in result["steps"].items(): diff --git a/tests/test_toolkit_capture.py b/tests/test_toolkit_capture.py new file mode 100644 index 0000000..4e6113d --- /dev/null +++ b/tests/test_toolkit_capture.py @@ -0,0 +1,578 @@ +"""Phase 3 compatibility matrix and synthetic service evidence fixtures. + +Boundary fixtures intentionally isolate delegation; service fixtures below execute +real conversion/merge code. They are synthetic, not medical goldens. +""" + +from base64 import b64decode +from copy import deepcopy +import json +from types import SimpleNamespace + +import pytest +import requests + +from TCT.execution import execution, ExecutionRecordingError +from TCT.interfaces import tools +from TCT.interfaces.invocation import invoke, ToolInvocationError +from TCT.interfaces.serialization import to_jsonable +from TCT import node_normalizer, name_resolver, translator_query, translator_metakg + +# Every working public boundary; the broken nineteenth endpoint is excluded. +CASES = [ + ("get_translator_resources", "_get_translator_resources", []), + ("name_lookup", "lookup", ["term"]), + ("get_name_synonyms", "synonyms", ["X:1"]), + ("batch_name_lookup", "batch_lookup", [["term"]]), + ("normalize_nodes", "get_normalized_nodes", [["X:1"]]), + ("get_kp_info", "get_translator_kp_info", []), + ("get_metakg_data", "get_KP_metadata", [{"kp": "https://fixture.invalid"}]), + ( + "add_custom_api_to_metakg", + "add_new_API_for_query", + [{}, [], "kp", "url", "p", "s", "o"], + ), + ("add_plover_apis_to_metakg", "add_plover_API", [{}, []]), + ("get_api_predicates", "get_translator_API_predicates", []), + ("optimize_query_for_api", "optimize_query_json", [{}, "kp", {}]), + ("query_knowledge_provider", "query_KP", ["kp", {}, {}, {}]), + ("parallel_query_apis", "parallel_api_query", [{}, [], {}, {}]), + ("neighborhood_finder", "tct_neighborhood_finder", [["X:1"], ["biolink:Gene"]]), + ("path_finder", "query_TCT_pathfinder", ["X:1", "X:2"]), + ("validate_query_workflow", "validate_workflow", [{}]), + ("run_query_workflow", "run_workflow", [{}]), + ("replay_query_workflow", "replay_workflow", [{}]), +] + + +@pytest.mark.parametrize("name,backend,args", CASES) +@pytest.mark.parametrize("interface", ["python", "shared"]) +def test_all_working_boundaries_preserve_identity_arguments_mutations_and_errors( + monkeypatch, name, backend, args, interface +): + monkeypatch.setattr(tools, "_get_translator_resources", lambda: {}) + native = ( + SimpleNamespace(raw={}) + if name in {"path_finder", "neighborhood_finder"} + else {"steps": {}, "native": [1]} + ) + calls = [] + failure = [] + + def delegate(*a, **kw): + calls.append(deepcopy((a, kw))) + if failure: + raise failure[0] + if a and isinstance(a[0], dict): + a[0]["mutated"] = True + return native + + monkeypatch.setattr(tools, backend, delegate) + tool = getattr(tools, name) + call = tool if interface == "python" else lambda *a: invoke(tool, *a) + before = deepcopy(args) + assert call(*before) is native + after = deepcopy(args) + with execution({"trapi": True}) as run: + assert call(*after) is native + assert calls[0] == calls[1] + assert before == after + assert len(run) == 1 # interface adapter unwraps exactly once + assert to_jsonable(run[0]["output"]) == to_jsonable(native) + error = RuntimeError("fixture failure") + failure.append(error) + for enabled in [False, True]: + with execution({"trapi": enabled}) as failed: + with pytest.raises((RuntimeError, ToolInvocationError)) as caught: + call(*deepcopy(args)) + assert ( + caught.value.cause + if isinstance(caught.value, ToolInvocationError) + else caught.value + ) is error + if enabled: + assert failed[0]["status"] == "failed" + assert len(calls) == 4 + + +def response(body, status=200): + value = requests.Response() + value.status_code = status + value._content = body if isinstance(body, bytes) else json.dumps(body).encode() + return value + + +def services(run): + return [o for r in run for o in r.get("observations", []) if o["kind"] == "service"] + + +def test_normalizer_retains_equivalence_and_unmatched_without_extra_calls(monkeypatch): + body = { + "X:1": { + "id": {"identifier": "Y:1", "label": "fixture"}, + "type": ["biolink:Gene"], + "equivalent_identifiers": [{"identifier": "X:1"}], + "information_content": 42, + }, + "X:missing": None, + } + calls = [] + + def get(url, **kwargs): + calls.append(deepcopy(kwargs)) + return response(body) + + monkeypatch.setattr(requests, "get", get) + baseline = node_normalizer.get_normalized_nodes(list(body)) + with execution({"trapi": True}) as run: + captured = node_normalizer.get_normalized_nodes(list(body)) + assert captured == baseline and captured["X:1"].curie_synonyms is None + assert calls[0] == calls[1] and len(calls) == 2 + assert services(run)[0]["body_json"] == body + assert json.loads(b64decode(services(run)[0]["body_base64"])) == body + assert run[0]["inputs"]["return_equivalent_identifiers"] is False + + +@pytest.mark.parametrize( + "operation,args,body", + [ + ( + name_resolver.lookup, + ["term"], + [ + {"curie": "X:1", "types": ["Gene"], "names": ["alias"], "score": 12}, + {"curie": "X:2"}, + ], + ), + ( + name_resolver.synonyms, + ["X:1"], + {"X:1": {"curie": "X:1", "names": ["alias"], "score": 12}}, + ), + ( + name_resolver.batch_lookup, + [["term"]], + {"term": [{"curie": "X:1", "score": 12}]}, + ), + ], +) +def test_resolution_retains_full_candidates(monkeypatch, operation, args, body): + calls = [] + + def request(url, **kwargs): + calls.append(kwargs) + return response(body) + + monkeypatch.setattr(requests, "get", request) + monkeypatch.setattr(requests, "post", request) + baseline = operation(*args) + with execution({"trapi": True}) as run: + assert operation(*args) == baseline + assert len(calls) == 2 and calls[0] == calls[1] + assert services(run)[0]["body_json"] == body + + +def query(): + return translator_query.format_query_json( + ["X:1"], predicates=["biolink:affects", "biolink:related_to"] + ) + + +def test_transform_snapshots_preserve_legacy_mutation(): + original = query() + with execution({"trapi": True}) as run: + result = translator_query.optimize_query_json( + original, "kp", {"kp": ["biolink:affects"]} + ) + assert result["message"] is original["message"] + assert run[0]["inputs"]["query_json"] != run[0]["inputs_after"]["query_json"] + assert run[0]["output"] == original + assert run[0]["depends_on"] == [] + + +@pytest.mark.parametrize( + "status,body", + [ + (200, {"message": {"knowledge_graph": {"edges": {}}}}), + (503, {"error": "fixture unavailable"}), + (200, b"not json"), + ], +) +def test_provider_empty_http_and_parse_failure_remain_distinct( + monkeypatch, status, body +): + monkeypatch.setattr(requests, "post", lambda *a, **kw: response(body, status)) + args = ( + "kp", + query(), + {"kp": "https://fixture.invalid"}, + {"kp": ["biolink:affects"]}, + ) + if isinstance(body, bytes): + with pytest.raises(requests.exceptions.JSONDecodeError): + translator_query.query_KP(*args) + with execution({"trapi": True}) as run: + with pytest.raises(requests.exceptions.JSONDecodeError): + translator_query.query_KP(*args) + assert services(run)[0]["body_json_status"] == "not_json" + else: + assert translator_query.query_KP(*args) is None + with execution({"trapi": True}) as run: + assert translator_query.query_KP(*args) is None + assert services(run)[0]["body_json"] == body + assert services(run)[0]["status_code"] == status + + +def test_parallel_capture_retains_envelopes_and_collision_decisions(monkeypatch): + import concurrent.futures + + # Fix the completion schedule in this subcase; production ordering is unchanged. + monkeypatch.setattr( + concurrent.futures, "as_completed", lambda futures: list(futures) + ) + bodies = {} + calls = [] + for provider in ["a", "b"]: + bodies[provider] = { + "status": "OK", + "message": { + "knowledge_graph": { + "nodes": {"X:1": {}}, + "edges": { + "collision": { + "subject": "X:1", + "object": "X:2", + "sources": [{"resource_id": provider}], + "qualifiers": [ + { + "qualifier_type_id": "fixture", + "qualifier_value": provider, + } + ], + } + }, + }, + "results": [ + { + "analyses": [ + { + "resource_id": provider, + "edge_bindings": {"e": [{"id": "collision"}]}, + } + ] + } + ], + "auxiliary_graphs": {"support": {"edges": ["collision"]}}, + }, + } + + def post(url, **kwargs): + calls.append((url, deepcopy(kwargs))) + if url == "failed": + raise requests.ConnectionError("fixture") + return response(bodies[url]) + + monkeypatch.setattr(requests, "post", post) + args = ( + query(), + ["a", "b", "failed"], + {p: p for p in ["a", "b", "failed"]}, + {p: ["biolink:affects"] for p in ["a", "b", "failed"]}, + ) + baseline = translator_query.parallel_api_query(*args) + with execution({"trapi": True}) as run: + captured = translator_query.parallel_api_query(*args) + # One worker preserves provider completion order for this fixture. + assert captured == baseline == bodies["b"]["message"]["knowledge_graph"]["edges"] + assert len(calls) == 6 and calls[:3] == calls[3:] + assert [o.get("body_json") for o in services(run)] == [ + bodies["a"], + bodies["b"], + None, + ] + assert services(run)[-1]["status"] == "transport_error" + merges = [o for o in run[0]["observations"] if o["kind"] == "edge_merge"] + assert merges[1]["overwritten_edge_ids"] == ["collision"] + assert all(r["depends_on"] == [] for r in run) + + +def test_disabled_does_not_read_body(monkeypatch): + class NativeResponse: + status_code = 200 + + @property + def content(self): + raise AssertionError("disabled capture read response") + + def json(self): + return {"X:1": None} + + monkeypatch.setattr(requests, "get", lambda *a, **kw: NativeResponse()) + assert node_normalizer.get_normalized_nodes("X:1") is None + + +def test_incomplete_requested_capture_surfaces_after_native_operation( + monkeypatch, tmp_path +): + monkeypatch.setattr( + requests, + "get", + lambda *a, **kw: SimpleNamespace(status_code=200, json=lambda: {"X:1": None}), + ) + with execution({"record": True, "store": str(tmp_path)}) as run: + with pytest.raises(ExecutionRecordingError) as caught: + node_normalizer.get_normalized_nodes("X:1") + assert caught.value.operation_completed is True + assert run[0]["status"] == "capture_failed" + + +def test_working_surface_coverage_and_unsupported_endpoint(): + assert {c[0] for c in CASES} == {f.__name__ for f in tools.TOOLS} - { + "trapi_query_endpoint" + } + assert len(CASES) == 18 + with pytest.raises(TypeError): + tools.trapi_query_endpoint("https://fixture.invalid") + + +def test_discovery_metadata_and_cached_resources_keep_full_observations(monkeypatch): + import pandas as pd + from TCT import TCT as core + from TCT.config import service_url + + calls = [] + registry = { + "hits": [ + { + "_id": "synthetic", + "info": {"title": "fixture"}, + "servers": [ + {"url": "https://fixture.invalid", "x-maturity": "production"} + ], + "extra": {"provenance": "synthetic"}, + } + ] + } + metadata = { + "hits": [{"_id": "Gene-affects-Gene", "extra": {"evidence": "synthetic"}}] + } + plover = { + "nodes": {"biolink:Gene": {"id_prefixes": ["X"]}}, + "edges": [], + "extra": "synthetic", + } + + def get(url, **kwargs): + calls.append((url, kwargs)) + if url == service_url("smartapi_registry"): + return response(registry) + if "/metakg" in url: + return response(metadata) + if url.endswith("/meta_knowledge_graph"): + return response(plover) + raise AssertionError(f"Unexpected request {url}") + + monkeypatch.setattr(requests, "get", get) + core.clear_translator_resource_cache() + try: + baseline = tools.get_translator_resources() + baseline_calls = deepcopy(calls) + core.clear_translator_resource_cache() + calls.clear() + with execution({"trapi": True}) as run: + captured = tools.get_translator_resources() + assert tools.get_translator_resources() is captured + assert captured.api_names == baseline.api_names + pd.testing.assert_frame_equal(captured.meta_kg, baseline.meta_kg) + assert captured.api_predicates == baseline.api_predicates + assert calls == baseline_calls + bodies = [o["body_json"] for o in services(run)] + assert registry in bodies and metadata in bodies and plover in bodies + assert len(bodies) == len(baseline_calls) + finally: + core.clear_translator_resource_cache() + + +def test_custom_metadata_captures_mapping_mutation_and_dataframe(monkeypatch): + import pandas as pd + + api_names = {} + metadata = pd.DataFrame(columns=["API", "Predicate", "Subject", "Object", "URL"]) + with execution({"trapi": True}) as run: + result = tools.add_custom_api_to_metakg( + api_names, + metadata, + "fixture", + "https://fixture.invalid", + "biolink:affects", + "biolink:ChemicalEntity", + "biolink:Gene", + ) + assert result[0] is api_names + assert len(metadata) == 0 and len(result[1]) == 1 + assert run[0]["inputs"]["api_names"] == {} + assert run[0]["inputs_after"]["api_names"] == {"fixture": "https://fixture.invalid"} + assert to_jsonable(result[1]) == [ + { + "API": "fixture", + "Predicate": "biolink:affects", + "Subject": "biolink:ChemicalEntity", + "Object": "biolink:Gene", + "URL": "https://fixture.invalid", + } + ] + + +def test_metakg_fallback_preserves_both_attempts(monkeypatch): + calls = [] + bodies = [{"unavailable": True}, {"hits": [{"_id": "Gene-affects-Gene"}]}] + + def get(url, **kwargs): + calls.append(url) + return response(bodies[len(calls) - 1]) + + monkeypatch.setattr(requests, "get", get) + with execution({"trapi": True}) as run: + result = translator_metakg.get_KP_metadata( + {"fixture": "https://fixture.invalid"} + ) + assert len(calls) == 2 and len(result) == 1 + assert [o["body_json"] for o in services(run)] == bodies + + +def test_finder_assembly_preserves_evidence_and_native_extensions(): + from TCT.TCT_pathfinder import parse_results_for_pathfinder + from TCT.TCT_neighborhood_finder import parse_results_for_neighborhood_finder + + left = { + "a": { + "subject": "X:chemical", + "object": "X:gene", + "predicate": "biolink:affects", + "attributes": [], + "sources": [{"resource_id": "infores:synthetic"}], + } + } + right = { + "b": { + "subject": "X:gene", + "object": "X:disease", + "predicate": "biolink:gene_associated_with_condition", + "attributes": [], + "sources": [{"resource_id": "infores:synthetic"}], + } + } + expected = parse_results_for_pathfinder( + "X:chemical", "X:disease", deepcopy(left), deepcopy(right), get_node_info=False + ) + with execution({"trapi": True}) as run: + actual = parse_results_for_pathfinder( + "X:chemical", + "X:disease", + deepcopy(left), + deepcopy(right), + get_node_info=False, + ) + assert actual == expected + assert set(actual["knowledge_graph"]["edges"]) == {"a", "b"} + assert actual["auxiliary_graphs"] == {"aux_1_X:gene": ["a", "b"]} + assert run[0]["inputs"]["result1"] == left + assert run[0]["inputs"]["result2"] == right + baseline = parse_results_for_neighborhood_finder( + "X:chemical", deepcopy(left), get_node_info=False + ) + with execution({"trapi": True}) as run: + captured = parse_results_for_neighborhood_finder( + "X:chemical", deepcopy(left), get_node_info=False + ) + assert baseline == captured + assert captured["knowledge_graph"]["edges"] == left + + +def test_parallel_does_not_swallow_requested_recording_failure(monkeypatch, tmp_path): + from TCT.journal import Journal + + original = Journal.finish + + def finish(self, record): + if record["operation"] == "query_KP": + raise OSError("fixture disk failure") + return original(self, record) + + monkeypatch.setattr(Journal, "finish", finish) + monkeypatch.setattr( + requests, + "post", + lambda *a, **kw: response({"message": {"knowledge_graph": {"edges": {}}}}), + ) + with execution({"record": True, "store": str(tmp_path)}): + with pytest.raises(ExecutionRecordingError): + translator_query.parallel_api_query( + query(), + ["kp"], + {"kp": "https://fixture.invalid"}, + {"kp": ["biolink:affects"]}, + ) + + +@pytest.mark.parametrize("name,backend,args", CASES) +def test_cli_and_mcp_payloads_with_durable_capture( + monkeypatch, tmp_path, capsys, name, backend, args +): + import asyncio + import inspect + from typing import Any, get_type_hints + from TCT.interfaces.cli import main + from TCT.interfaces.mcp import mcp + + monkeypatch.setattr(tools, "_get_translator_resources", lambda: {}) + native = ( + SimpleNamespace(raw={}) + if name in {"path_finder", "neighborhood_finder"} + else {"steps": {}, "fixture": [1]} + ) + calls = [] + + def delegate(*a, **kw): + calls.append(deepcopy((a, kw))) + return native + + monkeypatch.setattr(tools, backend, delegate) + tool = getattr(tools, name) + arguments = dict(inspect.signature(tool).bind(*args).arguments) + argv = [name.replace("_", "-")] + hints = get_type_hints(tool) + for key, value in arguments.items(): + argv.append("--" + key.replace("_", "-")) + if isinstance(value, list) and hints[key] is not Any: + argv.extend(value or ["fixture"]) + elif isinstance(value, (dict, list)): + argv.append(json.dumps(value)) + else: + argv.append(str(value)) + assert main(argv) == 0 + before = capsys.readouterr() + config = { + "record": True, + "store": str(tmp_path / "store"), + "run_id": name, + "trapi": True, + } + path = tmp_path / "execution.json" + path.write_text(json.dumps(config)) + assert main(argv + ["--execution", str(path)]) == 0 + after = capsys.readouterr() + assert before == after + assert calls[0] == calls[1] + + async def invoke_mcp(): + registered = await mcp.get_tool(name) + baseline = await registered.run(deepcopy(arguments)) + captured = await registered.run(deepcopy(arguments) | {"execution": config}) + assert to_jsonable(baseline.content) == to_jsonable(captured.content) + assert to_jsonable(baseline.structured_content) == to_jsonable( + captured.structured_content + ) + + asyncio.run(invoke_mcp()) + assert calls[2] == calls[3] From ad16ce919e89b3223affe0a7230cba71fa89b4f3 Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Thu, 1 Oct 2026 10:21:56 -0700 Subject: [PATCH 6/8] Add deterministic evidence rollup and pinned offline TRAPI validation --- TCT/evidence.py | 784 ++++++++++ TCT/interfaces/WORKFLOWS.md | 105 ++ TCT/interfaces/tools.py | 35 + TCT/schemas/LICENSE | 202 +++ TCT/schemas/NOTICE | 5 + TCT/schemas/manifest.json | 14 + TCT/schemas/trapi-1.5.0.yaml | 1625 +++++++++++++++++++ TCT/schemas/trapi-2.0.0-beta2.yaml | 1814 ++++++++++++++++++++++ TCT/trapi_validation.py | 82 + pyproject.toml | 2 + tests/fixtures/rollup/README.md | 8 + tests/fixtures/rollup/case.json | 90 ++ tests/fixtures/rollup/chemical.json | 154 ++ tests/fixtures/rollup/composition.json | 180 +++ tests/fixtures/rollup/gene.json | 111 ++ tests/fixtures/rollup/golden.json | 400 +++++ tests/fixtures/rollup/normalization.json | 17 + tests/fixtures/rollup/predicates.json | 8 + tests/fixtures/rollup/providers.json | 4 + tests/fixtures/rollup/registry.json | 28 + tests/fixtures/rollup/rollup-golden.json | 15 + tests/test_evidence.py | 649 ++++++++ tests/test_mcp_contract.py | 6 + tests/test_toolkit_capture.py | 3 +- tests/test_tools.py | 3 +- uv.lock | 4 + 26 files changed, 6346 insertions(+), 2 deletions(-) create mode 100644 TCT/evidence.py create mode 100644 TCT/schemas/LICENSE create mode 100644 TCT/schemas/NOTICE create mode 100644 TCT/schemas/manifest.json create mode 100644 TCT/schemas/trapi-1.5.0.yaml create mode 100644 TCT/schemas/trapi-2.0.0-beta2.yaml create mode 100644 TCT/trapi_validation.py create mode 100644 tests/fixtures/rollup/README.md create mode 100644 tests/fixtures/rollup/case.json create mode 100644 tests/fixtures/rollup/chemical.json create mode 100644 tests/fixtures/rollup/composition.json create mode 100644 tests/fixtures/rollup/gene.json create mode 100644 tests/fixtures/rollup/golden.json create mode 100644 tests/fixtures/rollup/normalization.json create mode 100644 tests/fixtures/rollup/predicates.json create mode 100644 tests/fixtures/rollup/providers.json create mode 100644 tests/fixtures/rollup/registry.json create mode 100644 tests/fixtures/rollup/rollup-golden.json create mode 100644 tests/test_evidence.py diff --git a/TCT/evidence.py b/TCT/evidence.py new file mode 100644 index 0000000..bc3b74c --- /dev/null +++ b/TCT/evidence.py @@ -0,0 +1,784 @@ +"""Deterministic reduction of captured observations; no live service execution. + +Evidence is a set of immutable events. Composition is an explicit, closed recipe. +Call nesting is never interpreted as data lineage. +""" + +from copy import deepcopy +import hashlib +import json +from pathlib import Path +import sqlite3 + +from .interfaces.serialization import to_jsonable + +REDUCER_VERSION = 1 +MAX_EVENTS = 10000 + + +class EvidenceError(ValueError): + def __init__(self, message, report=None): + self.report = report + super().__init__(message) + + +def canonical(value): + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ) + + +def digest(value): + return hashlib.sha256(canonical(value).encode()).hexdigest() + + +def accumulate(*collections): + """Set union: associative, commutative and idempotent; conflicting IDs fail.""" + events = {} + identities = {} + for collection in collections: + if not isinstance(collection, list): + raise EvidenceError("events must be a list of execution records") + for native in collection: + event = json.loads(canonical(to_jsonable(native))) + if not isinstance(event, dict) or event.get("format_version") != 1: + raise EvidenceError("Unsupported execution event format") + key = digest(event) + identity = (event.get("run_id"), event.get("call_id")) + if not all(isinstance(x, str) and x for x in identity): + raise EvidenceError("Events require run_id and call_id") + if identity in identities and identities[identity] != key: + raise EvidenceError(f"Conflicting event identity: {identity}") + identities[identity] = key + events[key] = event + if len(events) > MAX_EVENTS: + raise EvidenceError("Evidence exceeds 10000 unique events") + return [events[key] for key in sorted(events)] + + +def read_run(store, run_id): + """Read a journal snapshot without creating files or trusting artifact names.""" + root = Path(store).expanduser().resolve() + database = root / "journal.sqlite3" + if not database.is_file(): + raise EvidenceError("Execution journal does not exist") + records = [] + connection = sqlite3.connect(database.as_uri() + "?mode=ro", uri=True) + try: + rows = connection.execute( + "SELECT call_id, step_id, status, event_hash FROM calls WHERE run_id=? ORDER BY step_id", + (run_id,), + ).fetchall() + if not rows: + raise EvidenceError("Run does not exist or has no recorded calls") + for call_id, step_id, status, checksum in rows: + if not checksum: + records.append( + { + "format_version": 1, + "run_id": run_id, + "call_id": call_id, + "step_id": step_id, + "status": status, + } + ) + continue + if len(checksum) != 64 or any( + x not in "0123456789abcdef" for x in checksum + ): + raise EvidenceError("Invalid artifact digest") + raw = (root / "artifacts" / checksum).read_bytes() + if hashlib.sha256(raw).hexdigest() != checksum: + raise EvidenceError("Execution artifact checksum mismatch") + record = json.loads(raw) + if ( + record.get("run_id"), + record.get("call_id"), + record.get("step_id"), + record.get("status"), + ) != (run_id, call_id, step_id, status): + raise EvidenceError("Journal index and execution artifact disagree") + records.append(record) + finally: + connection.close() + return accumulate(records) + + +def inventory(events): + sources, issues, workflows = {}, [], {} + for record in accumulate(events): + event_id = digest(record) + if ( + record.get("status") != "succeeded" + or record.get("capture_error") + or record.get("observation_errors") + ): + issues.append( + { + "event_id": event_id, + "status": record.get("status"), + "operation": record.get("operation"), + "capture_error": record.get("capture_error"), + } + ) + observations = list(record.get("observations", [])) + output = record.get("output") + if ( + isinstance(output, dict) + and output.get("schema_version") == 1 + and "steps" in output + ): + workflows[event_id] = output + for step_id, step in output["steps"].items(): + if step.get("op") == "query": + observations.append( + { + "kind": "service", + "status": "received", + "status_code": step.get("transport", {}).get("status_code"), + "body_json": step.get("response"), + "request": { + "url": step.get("inputs", {}).get("url"), + "arguments": { + "json": step.get("inputs", {}).get("query") + }, + }, + "workflow_step": step_id, + "eligible": step.get("status") == "succeeded", + } + ) + for index, observation in enumerate(observations): + if observation.get("kind") != "service": + continue + response = observation.get("body_json") + request = observation.get("request", {}) + query = request.get("arguments", {}).get("json") + if observation.get("eligible") is False: + issues.append( + { + "event_id": event_id, + "observation": index, + "status": "workflow_step_failed", + } + ) + if ( + observation.get("status") != "received" + or observation.get("status_code") != 200 + ): + issues.append( + { + "event_id": event_id, + "observation": index, + "status": observation.get("status"), + "status_code": observation.get("status_code"), + } + ) + continue + if not isinstance(response, dict) or not isinstance( + response.get("message"), dict + ): + if isinstance(query, dict) and "message" in query: + issues.append( + { + "event_id": event_id, + "observation": index, + "status": "invalid_trapi_body", + } + ) + continue # Normalization and discovery are retained events, not KG responses. + content = {"url": request.get("url"), "query": query, "response": response} + source_id = "source:" + digest(content) + source = sources.setdefault( + source_id, content | {"origins": [], "eligible": False} + ) + source["eligible"] |= ( + record.get("status") == "succeeded" + and not record.get("capture_error") + and not record.get("observation_errors") + and observation.get("eligible", True) + ) + source["origins"].append( + { + "event_id": event_id, + "observation": index, + "call_id": record["call_id"], + } + ) + for source in sources.values(): + source["origins"] = sorted( + {canonical(o): o for o in source["origins"]}.values(), key=canonical + ) + return sources, sorted(issues, key=canonical), workflows + + +def _pointer(value, pointer): + if not isinstance(pointer, str) or (pointer and not pointer.startswith("/")): + raise EvidenceError("Selection paths must be JSON pointers") + for part in pointer.split("/")[1:]: + part = part.replace("~1", "/").replace("~0", "~") + value = value[int(part)] if isinstance(value, list) else value[part] + return value + + +def _attrs(value, path, additions): + if "attributes" not in value: + value["attributes"] = [] + additions.append(path + "/attributes") + + +def _prepare(source, source_id, mapping, query_graph, version): + """Normalize structural omissions only; never invent biomedical statements.""" + if not source["eligible"]: + raise EvidenceError( + f"{source_id}: source capture did not complete successfully" + ) + response = deepcopy(source["response"]) + message = response["message"] + request = source.get("query") or {} + local = message.get("query_graph") or request.get("message", {}).get("query_graph") + sent_graph = request.get("message", {}).get("query_graph") + if ( + message.get("query_graph") + and sent_graph + and canonical(local) != canonical(sent_graph) + ): + raise EvidenceError( + "Provider query-graph rewrite requires an explicit conversion" + ) + if not isinstance(local, dict): + raise EvidenceError(f"{source_id}: explicit provider query graph is missing") + node_map, edge_map = mapping.get("nodes", {}), mapping.get("edges", {}) + path_map = mapping.get("paths", {}) + for field, rename in [ + ("nodes", node_map), + ("edges", edge_map), + ("paths", path_map), + ]: + if set(rename) != set(local.get(field, {})) or len(set(rename.values())) != len( + rename + ): + raise EvidenceError( + f"{source_id}: {field} mapping must map each local ID once without collapsing IDs" + ) + if not set(rename.values()) <= set(query_graph.get(field, {})): + raise EvidenceError(f"{source_id}: unknown target {field} mapping") + for field, rename in [("edges", edge_map), ("paths", path_map)]: + for key, target in rename.items(): + item, combined = local[field][key], query_graph[field][target] + if any( + node_map.get(item[end]) != combined[end] + for end in ["subject", "object"] + ): + raise EvidenceError( + f"{source_id}: mapping changes {field} direction/endpoints" + ) + if response.get("status") not in (None, "Success", "OK"): + raise EvidenceError(f"{source_id}: provider reported {response['status']}") + kg = message.get("knowledge_graph") + if ( + not isinstance(kg, dict) + or not isinstance(kg.get("nodes"), dict) + or not isinstance(kg.get("edges"), dict) + ): + raise EvidenceError(f"{source_id}: incomplete knowledge graph") + additions = [] + if version == "1.5.0": + if local.get("paths"): + raise EvidenceError("Path conversion to TRAPI 1.5.0 is unsupported") + for node_id, node in kg["nodes"].items(): + _attrs(node, f"/knowledge_graph/nodes/{node_id}", additions) + for aux_id, aux in (message.get("auxiliary_graphs") or {}).items(): + if not isinstance(aux, dict): + raise EvidenceError( + "Legacy list-valued auxiliary graphs require an explicit conversion" + ) + _attrs(aux, f"/auxiliary_graphs/{aux_id}", additions) + for i, result in enumerate((message.get("results") or [])): + for bindings in [ + result.get("node_bindings", {}), + *(a.get("edge_bindings", {}) for a in result.get("analyses", [])), + ]: + for key, values in bindings.items(): + for j, binding in enumerate(values): + _attrs(binding, f"/results/{i}/bindings/{key}/{j}", additions) + # Full local responses are validated before any remapping. + from .trapi_validation import validate + + checked = validate(response, version) + if not checked["valid"]: + raise EvidenceError(f"{source_id}: invalid source response", checked) + return message, additions + + +def _ids(binding, version): + return [b["id"] for b in binding] if version == "1.5.0" else binding["ids"] + + +def _bindings(ids, version): + return ( + [{"id": item, "attributes": []} for item in sorted(set(ids))] + if version == "1.5.0" + else {"ids": sorted(set(ids))} + ) + + +def _rewrite_support(attributes, auxiliary): + for attr in attributes or []: + if attr.get("attribute_type_id") == "biolink:support_graphs": + if not isinstance(attr.get("value"), list): + raise EvidenceError("support_graphs attribute must contain a list") + attr["value"] = [auxiliary[x] for x in attr["value"]] + _rewrite_support(attr.get("attributes", []), auxiliary) + + +def _merge(mapped, version): + node_variants = {} + nodes, edges, auxiliary, source_results, references, variants = ( + {}, + {}, + {}, + {}, + {}, + {}, + ) + for source_id in sorted(mapped): + message, mapping = mapped[source_id] + kg = message["knowledge_graph"] + edge_ids = {key: "edge:" + digest([source_id, key]) for key in kg["edges"]} + aux_ids = { + key: "aux:" + digest([source_id, key]) + for key in (message.get("auxiliary_graphs") or {}) + } + references[source_id] = {"edges": edge_ids, "auxiliary_graphs": aux_ids} + for node_id, node in kg["nodes"].items(): + node = deepcopy(node) + _rewrite_support(node.get("attributes", []), aux_ids) + variants.setdefault(node_id, {})[digest(node)] = node + node_variants.setdefault(node_id, {})[source_id] = node + for key, edge in kg["edges"].items(): + value = deepcopy(edge) + if ( + value["subject"] not in kg["nodes"] + or value["object"] not in kg["nodes"] + ): + raise EvidenceError(f"{source_id}: edge {key} refers to an absent node") + _rewrite_support(value.get("attributes", []), aux_ids) + edges[edge_ids[key]] = value + for key, graph in (message.get("auxiliary_graphs") or {}).items(): + value = deepcopy(graph) + value["edges"] = [edge_ids[x] for x in value["edges"]] + _rewrite_support(value.get("attributes", []), aux_ids) + auxiliary[aux_ids[key]] = value + results = deepcopy(message.get("results") or []) + for result in results: + result["node_bindings"] = { + mapping["nodes"][key]: value + for key, value in result["node_bindings"].items() + } + for values in result["node_bindings"].values(): + if not set(_ids(values, version)) <= set(kg["nodes"]): + raise EvidenceError( + f"{source_id}: node binding refers to an absent node" + ) + if version == "1.5.0": + for binding in values: + _rewrite_support(binding.get("attributes", []), aux_ids) + for analysis in result.get("analyses") or []: + remapped = {} + for key, values in analysis.get("edge_bindings", {}).items(): + if version == "1.5.0": + for binding in values: + binding["id"] = edge_ids[binding["id"]] + _rewrite_support(binding.get("attributes", []), aux_ids) + remapped[mapping["edges"][key]] = values + else: + remapped[mapping["edges"][key]] = { + "ids": [edge_ids[x] for x in values["ids"]] + } + if "edge_bindings" in analysis: + analysis["edge_bindings"] = remapped + if "path_bindings" in analysis: + analysis["path_bindings"] = { + mapping["paths"][k]: {"ids": [aux_ids[x] for x in v["ids"]]} + for k, v in analysis["path_bindings"].items() + } + if "support_graphs" in analysis: + analysis["support_graphs"] = [ + aux_ids[x] for x in analysis["support_graphs"] + ] + _rewrite_support(analysis.get("attributes", []), aux_ids) + source_results[source_id] = results + for node_id, choices in variants.items(): + values = [choices[k] for k in sorted(choices)] + node = deepcopy(values[0]) + node["categories"] = sorted({c for v in values for c in v["categories"]}) + attrs = {canonical(a): a for v in values for a in (v.get("attributes") or [])} + if attrs or version == "1.5.0": + node["attributes"] = [attrs[k] for k in sorted(attrs)] + # Node disallows arbitrary properties in 1.5. Preserve variants in the + # top-level provenance extension, associated with their source IDs. + nodes[node_id] = node + return ( + {"nodes": nodes, "edges": edges}, + auxiliary, + source_results, + references, + node_variants, + ) + + +def _transform(composition, sources): + from .workflow import run_workflow + + recipe = deepcopy(composition["workflow"]) + for step in recipe.get("steps", []): + if step.get("op") == "query": + raise EvidenceError("Evidence replay does not execute network queries") + if step.get("op") == "source": + if ( + set(step) != {"id", "op", "source_id"} + or step["source_id"] not in sources + ): + raise EvidenceError( + "source steps must reference an explicitly mapped source" + ) + step.update( + op="literal", value=deepcopy(sources[step.pop("source_id")]["response"]) + ) + bundle = run_workflow(recipe) + if bundle["status"] != "succeeded": + raise EvidenceError("Composition workflow failed", bundle) + output_name = composition["selection"]["output"] + rows = bundle["outputs"].get(output_name) + if not isinstance(rows, list): + raise EvidenceError("Selected workflow output must be a list of answer rows") + # Exclude wall-clock metadata from the semantic hash; retain decisions and lineage. + trace = { + key: { + field: value + for field, value in step.items() + if field + in ( + "op", + "status", + "inputs", + "output", + "row_lineage", + "decisions", + "dependencies", + "depends_on", + ) + } + for key, step in bundle["steps"].items() + } + return rows, trace + + +def _select(rows, selection, query_graph, kg, auxiliary, refs, mappings, version): + results = {} + if set(selection.get("nodes", {})) != set(query_graph["nodes"]): + raise EvidenceError("Selection must bind every query node explicitly") + if set(selection.get("edges", {})) != set(query_graph.get("edges", {})): + raise EvidenceError("Selection must bind every query edge explicitly") + if set(selection.get("paths", {})) != set(query_graph.get("paths", {})): + raise EvidenceError("Selection must bind every query path explicitly") + for row_index, row in enumerate(rows): + nodes = {key: _pointer(row, path) for key, path in selection["nodes"].items()} + for key, node_id in nodes.items(): + if not isinstance(node_id, str) or node_id not in kg["nodes"]: + raise EvidenceError( + f"Selected node {key} is missing from accumulated evidence" + ) + qnode = query_graph["nodes"][key] + if qnode.get("ids") and node_id not in qnode["ids"]: + raise EvidenceError( + f"Selected node {key} violates the query identifier restriction" + ) + if qnode.get("categories") and not set(qnode["categories"]).intersection( + kg["nodes"][node_id]["categories"] + ): + raise EvidenceError( + f"Selected node {key} has no exact requested category; ancestor inference is unsupported" + ) + edge_bindings, path_bindings = {}, {} + origins = [] + for field, target in [("edges", edge_bindings), ("paths", path_bindings)]: + for key, spec in selection.get(field, {}).items(): + source_id = spec["source_id"] + if ( + source_id not in refs + or key not in mappings[source_id].get(field, {}).values() + ): + raise EvidenceError( + f"Selection {key} has no corresponding source query mapping" + ) + original = _pointer(row, spec["path"]) + identifiers = original if isinstance(original, list) else [original] + if not identifiers: + raise EvidenceError("Empty answer bindings are unsupported") + table = refs[source_id][ + "edges" if field == "edges" else "auxiliary_graphs" + ] + ids = [table[x] for x in identifiers] + target[key] = _bindings(ids, version) + origins.append( + { + "query_id": key, + "source_id": source_id, + "original_ids": identifiers, + } + ) + qedge = query_graph[field][key] + if field == "edges": + for edge_id in ids: + edge = kg["edges"][edge_id] + if ( + edge["subject"] != nodes[qedge["subject"]] + or edge["object"] != nodes[qedge["object"]] + ): + raise EvidenceError( + f"Selected edge {key} violates query direction/endpoints" + ) + if ( + qedge.get("predicates") + and edge["predicate"] not in qedge["predicates"] + ): + raise EvidenceError( + f"Selected edge {key} violates the requested predicate" + ) + else: + for aux_id in ids: + # Directed reachability within each declared supporting graph. + frontier = {nodes[qedge["subject"]]} + for _ in range(len(auxiliary[aux_id]["edges"]) + 1): + frontier |= { + kg["edges"][e]["object"] + for e in auxiliary[aux_id]["edges"] + if kg["edges"][e]["subject"] in frontier + } + if nodes[qedge["object"]] not in frontier: + raise EvidenceError( + "Selected path does not connect the requested endpoints" + ) + analysis = {"resource_id": selection["resource_id"]} + if edge_bindings or version == "1.5.0": + analysis["edge_bindings"] = edge_bindings + if path_bindings: + analysis["path_bindings"] = path_bindings + result = { + "node_bindings": { + key: _bindings([value], version) for key, value in nodes.items() + }, + "analyses": [analysis], + "x-tct-evidence": sorted(origins, key=canonical), + } + result = results.setdefault( + canonical(result), result | {"x-tct-selection-rows": []} + ) + result["x-tct-selection-rows"].append(row_index) + return [results[key] for key in sorted(results)] + + +def reduce_evidence(events, composition, version="1.5.0"): + """Return a validated report or an explicit invalid draft; never perform I/O.""" + from .trapi_validation import validate + + sources, issues, _ = inventory(events) + report = {"reducer_version": REDUCER_VERSION, "status": "invalid", "issues": issues} + try: + composition = deepcopy(composition) + if not isinstance(composition, dict) or set(composition) - { + "closed", + "query_graph", + "sources", + "workflow", + "selection", + }: + raise EvidenceError( + "Unknown composition settings; ranking and automatic query reformulation are unsupported" + ) + selection = composition.get("selection", {}) + if not isinstance(selection, dict) or set(selection) - { + "output", + "resource_id", + "nodes", + "edges", + "paths", + }: + raise EvidenceError("Unknown selection settings") + if version not in {"1.5.0", "2.0.0-beta2"}: + raise EvidenceError( + "Unsupported target; final TRAPI 2.0.0 is not available in the pinned schemas" + ) + if composition.get("closed") is not True: + raise EvidenceError( + "Final selection/exclusion requires closed=true and a complete explicit source set" + ) + if any(item.get("status") == "running" for item in issues): + raise EvidenceError( + "Cannot finalize while recorded inputs are still running" + ) + query_graph = deepcopy(composition["query_graph"]) + if version == "1.5.0" and query_graph.get("paths"): + raise EvidenceError( + "Path exports require the explicitly prerelease 2.0.0-beta2 target" + ) + # Refuse constraints the local selector cannot evaluate. + for node in query_graph["nodes"].values(): + if set(node) - {"ids", "categories"}: + raise EvidenceError("Unsupported query-node constraint conversion") + for edge in query_graph.get("edges", {}).values(): + if ( + set(edge) - {"subject", "object", "predicates", "knowledge_type"} + or edge.get("knowledge_type", "lookup") != "lookup" + ): + raise EvidenceError( + "Unsupported query-edge constraint/inference conversion" + ) + for path in query_graph.get("paths", {}).values(): + if set(path) - {"subject", "object"}: + raise EvidenceError("Constrained path conversion is unsupported") + mappings = composition["sources"] + if not isinstance(mappings, dict) or not mappings: + raise EvidenceError("Composition requires explicit source mappings") + selected_sources = {} + mapped, additions = {}, {} + for source_id, mapping in sorted(mappings.items()): + if not isinstance(mapping, dict) or set(mapping) - { + "nodes", + "edges", + "paths", + }: + raise EvidenceError("Unknown query mapping settings") + if source_id not in sources: + raise EvidenceError(f"Missing closed input: {source_id}") + selected_sources[source_id] = sources[source_id] + message, added = _prepare( + sources[source_id], source_id, mapping, query_graph, version + ) + mapped[source_id] = (message, mapping) + additions[source_id] = added + kg, auxiliary, original_results, references, node_variants = _merge( + mapped, version + ) + rows, transformations = _transform(composition, selected_sources) + results = _select( + rows, + composition["selection"], + query_graph, + kg, + auxiliary, + references, + mappings, + version, + ) + response = { + "schema_version": version, + "message": { + "query_graph": query_graph, + "knowledge_graph": kg, + "auxiliary_graphs": auxiliary, + "results": results, + }, + "x-tct": { + "reducer_version": REDUCER_VERSION, + "source_mappings": mappings, + "source_results": original_results, + "node_variants": node_variants, + "structural_additions": additions, + "selection": composition["selection"], + "recipe": composition["workflow"], + "transformations": transformations, + "coverage": { + "scope": "Closed over the explicitly mapped responses only; not exhaustive biomedical coverage.", + "unmapped_source_ids": sorted(set(sources) - set(mappings)), + "observed_issue_count": len(issues), + }, + }, + } + report["draft"] = response + report["validation"] = validate(response, version) + if report["validation"]["valid"]: + report.update( + status="valid", rollup_hash=digest(response), references=references + ) + except (ValueError, KeyError, TypeError, IndexError, AttributeError) as error: + report["error"] = str(error) + if isinstance(error, EvidenceError) and error.report is not None: + report["details"] = error.report + return report + + +def evidence(request): + """One operation for inventory, deterministic replay and validated export.""" + allowed = {"action", "events", "store", "run_id", "composition", "version"} + if not isinstance(request, dict) or set(request) - allowed: + raise EvidenceError("Unknown evidence request settings") + action = request.get("action", "inspect") + if action not in {"inspect", "replay", "export"}: + raise EvidenceError("action must be inspect, replay or export") + if "events" in request: + if "store" in request or "run_id" in request: + raise EvidenceError("Choose events or a store/run_id, not both") + events = accumulate(request["events"]) + else: + if not all( + isinstance(request.get(k), str) and request[k] for k in ("store", "run_id") + ): + raise EvidenceError("Provide events or both store and run_id") + events = read_run(request["store"], request["run_id"]) + # An export may itself be recorded into the run being read. Its current + # journal row is not an unfinished provider input to its own composition. + from .execution import observation_target + + active = observation_target() + if active and active.get("operation") == "evidence": + events = [ + e + for e in events + if (e["run_id"], e["call_id"]) != (active["run_id"], active["call_id"]) + ] + sources, issues, workflows = inventory(events) + if action == "inspect": + return { + "event_count": len(events), + "sources": { + key: { + "url": source["url"], + "query_graph": (source.get("query") or {}) + .get("message", {}) + .get("query_graph"), + "origins": source["origins"], + "eligible": source["eligible"], + "node_count": len( + ( + source["response"]["message"].get("knowledge_graph") or {} + ).get("nodes", {}) + ), + "edge_count": len( + ( + source["response"]["message"].get("knowledge_graph") or {} + ).get("edges", {}) + ), + } + for key, source in sorted(sources.items()) + }, + "issues": issues, + "workflow_events": sorted(workflows), + } + report = reduce_evidence( + events, request["composition"], request.get("version", "1.5.0") + ) + if action == "replay": + return report + if report["status"] != "valid": + raise EvidenceError( + "TRAPI export rejected: " + + canonical(report.get("validation", report.get("error", report))), + report, + ) + return report["draft"] diff --git a/TCT/interfaces/WORKFLOWS.md b/TCT/interfaces/WORKFLOWS.md index ef79251..eaacdba 100644 --- a/TCT/interfaces/WORKFLOWS.md +++ b/TCT/interfaces/WORKFLOWS.md @@ -298,3 +298,108 @@ The single-NLQ frozen benchmark matched its native-output and final-answer golde through Python, CLI and registered MCP. A durable Python run retained 11 operation records and all four service bodies while issuing the same four requests. Live service tests, real-agent evaluations, semantic rollup and caching were not run. + +### Phase 4: deterministic evidence composition + +The new `evidence(request)` tool is shared by Python, CLI and MCP. Existing tool +returns and ordinary execution are unchanged. The registry now has 19 working +operations plus the existing unsupported `trapi_query_endpoint` placeholder. +The request is one JSON object; it does not add a family of CLI flags. + +```python +from TCT.interfaces.tools import evidence + +source = {"store": "./evidence", "run_id": "experiment-001"} +inventory = evidence(source | {"action": "inspect"}) +# Use the returned source IDs and local query graphs to define composition. +report = evidence(source | {"action": "replay", "composition": composition}) +response = evidence(source | {"action": "export", "composition": composition}) +``` + +CLI uses `tct evidence --request ''`; MCP uses the same tool with a +`request` object. Alternatively, provide `events` (an execution observation list) +instead of `store`/`run_id`. Inspection lists sources, call/event origins, query +graphs, counts, eligibility and failures. Replay returns the draft, diagnostics, +identifier mappings and semantic rollup hash. Export returns only the validated +TRAPI response, with provenance inside its `x-tct` extension. Invalid exports +raise an error; inspect the replay report for details. None of these operations +contacts a provider. Reading a journal checks its artifact digests and index +identities without creating a missing store. An export may itself be recorded +into the same run; its own pending journal row is excluded from its input set. + +A composition object has these fields: + +- `closed: true`: the caller declares its explicit source set complete. Referenced + sources must be present and successfully captured. Other unfinished journal + calls prevent finalization; failures and unmapped sources remain disclosed. +- `query_graph`: the intended combined query, retaining edge direction and lookup + semantics. Exact node IDs, categories and predicates are checked for selected + answers. Biolink ancestor reasoning and inferred edges are not implemented. +- `sources`: map each inspected source ID to `nodes`, `edges`, and optionally + `paths` dictionaries mapping local query IDs to combined query IDs. Every local + ID must be mapped once; mappings cannot collapse local IDs or reverse endpoints. +- `workflow`: the existing version-1 DAG recipe. A `source` step with `id` and + `source_id` supplies a frozen response. `rows`, `filter`, `project`, `pluck` and + `join` use explicit references and run in dependency order. Network `query` + steps are forbidden. Other operations, including ranking, are unsupported. +- `selection`: names a workflow `output`, the producing `resource_id`, `nodes` + (query node to row JSON pointer), and `edges`/`paths` (query ID to + `{source_id, path}`). Each query element must be explicitly bound. Supply an + appropriate producer identifier; the synthetic fixture's `infores:tct` is an + example identifier, not a claim of registration in the InfoRes catalogue. + +`tests/fixtures/rollup/composition.json` is a complete example for the original +Alzheimer's NLQ. Its service fixtures and native-output goldens are unchanged +copies from the Phase 1 worktree, with the attachment hash retained in `case.json`. +The recipe uses recorded-source references and the same filters/join. It selects +one chemical → gene → disease answer; all five retrieved edges, including the +three distractors, remain in the accumulated knowledge graph. + +Accumulation is set union over canonical JSON event content: duplicate events do +not multiply evidence, and insertion/completion order does not affect the result. +Conflicting content for the same run/call identity is rejected. The semantic hash +excludes capture IDs and workflow timestamps. It includes the explicit recipe, +source mappings, selected answers, retained evidence, filter decisions and row +lineage. This hash identifies a Phase 4 artifact; it is not the execution/cache +identity planned for Phase 5. Reordering equivalent recipe definitions can still +change this hash; no semantic query equivalences are assumed. + +Provider-local edge and auxiliary-graph IDs are deterministically namespaced. +Bindings, supporting-graph references and known support-graph attributes are +rewritten together. CURIE node IDs retain their identity. Conflicting node metadata +has a deterministic display value, combined categories/attributes, and complete +source-specific variants under `x-tct.node_variants`. Provider analyses, scores, +node/edge/path bindings and support graphs survive under `x-tct.source_results`; +they are retained as source evidence, not falsely presented as complete analyses +of the combined query. Selected answers get an explicit composition analysis, +source references and the workflow output row indices that produced them. + +Validation uses unmodified, checksum-pinned upstream schemas bundled under +`TCT/schemas`, with OpenAPI 3.0 `nullable` adapted to JSON Schema at load time. +Ordinary exports target TRAPI **1.5.0**. Required empty attribute arrays omitted +by legacy responses are added structurally and listed in `structural_additions`. +No predicates, evidence sources or knowledge-level classifications are invented. +Validation proves structural conformance, with additional reducer reference and +selection checks; it does not establish Biolink conformance or biological truth. + +**Version limitation:** the official repository had no final 2.0.0 tag when these +schemas were pinned. Native path exports can explicitly target **2.0.0-beta2**; +output and validation metadata retain that prerelease label. Requesting `2.0.0` +is rejected. Automatic conversion from 1.5.0 or legacy finder path extensions to +2.0 is unsupported, as are constrained paths, unsupported query constraints and +provider query-graph rewrites. External schema references (such as a provider's +workflow extension) that are not vendored produce an explicit validation error; +validation never fetches them. Full response bodies remain in the execution +journal even if no supported rollup can be produced. + +The endpoint remains opt-in: capture alone does not automatically compose answers, +select mappings or enable caching. Phase 5 remains a separate trigger. + +Phase 4 acceptance: **260 offline tests passed**. The original single-NLQ service +and native-output fixtures produce one validated, traceable TRAPI 1.5.0 answer. +`rollup-golden.json` pins its semantic hash; duplicate events and eight shuffled +completion orders give identical reports. Tests also cover collision remapping, +retained provider analyses/support, dependency order, failed/missing inputs, +journal checksums, same-run recording, schema failures, beta2 native paths and +CLI/MCP/Python output parity. The built wheel includes both pinned schemas, +manifest and license. No live provider or paid model calls were made. diff --git a/TCT/interfaces/tools.py b/TCT/interfaces/tools.py index 733037b..1accbda 100644 --- a/TCT/interfaces/tools.py +++ b/TCT/interfaces/tools.py @@ -30,6 +30,7 @@ from ..trapi import query as trapi_query from ..execution import capture_trapi, capture_operation from ..workflow import replay_workflow, run_workflow, validate_workflow +from ..evidence import evidence as _evidence @capture_operation() @@ -441,6 +442,39 @@ def replay_query_workflow( return result +@capture_operation() +def evidence(request: dict[str, Any]) -> dict[str, Any]: + """Inspect captured evidence, replay composition, or export validated TRAPI. + + Inspect first to obtain source IDs and local query graphs. Composition requires + closed=true, query_graph, sources, workflow and selection. sources maps each + source ID to nodes/edges/paths dictionaries mapping local query IDs to global + query IDs. Direction must be preserved; all local IDs need explicit mappings. + + workflow uses the version-1 recipe described by validate_query_workflow, plus + source steps {id, op: "source", source_id} that provide recorded responses. + Network query steps are forbidden. Filters and joins use explicit references; + for example, rows can read a source at /message/knowledge_graph/edges. + + selection has output (workflow output name), resource_id (composition producer), + nodes (query node ID to row JSON pointer), and edges/paths (query ID to an object + with source_id and path, a row pointer to original edge/auxiliary-graph IDs). + Each query element must be bound. The operation retains retrieved evidence + separately from selected answers. Schema validation does not establish + biomedical correctness or complete source coverage. + + Args: + request: Object with action (inspect, replay, export), either events or + store/run_id, and an explicit composition for replay/export. Version + defaults to 1.5.0; 2.0.0-beta2 is an explicit prerelease target. + + Returns: + Inventory, replay diagnostics, or a validated TRAPI response. No network + calls occur. Invalid or unsupported exports raise an error. + """ + return _evidence(request) + + TOOLS: tuple[Callable[..., Any], ...] = ( get_translator_resources, name_lookup, @@ -461,6 +495,7 @@ def replay_query_workflow( validate_query_workflow, run_query_workflow, replay_query_workflow, + evidence, ) __all__ = [tool.__name__ for tool in TOOLS] + ["TOOLS"] diff --git a/TCT/schemas/LICENSE b/TCT/schemas/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/TCT/schemas/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/TCT/schemas/NOTICE b/TCT/schemas/NOTICE new file mode 100644 index 0000000..b291be1 --- /dev/null +++ b/TCT/schemas/NOTICE @@ -0,0 +1,5 @@ +TRAPI schemas: NCATSTranslator/ReasonerAPI, Apache 2.0 (as declared in +info.license in each upstream document). Original YAML is retained unmodified. +manifest.json pins source URLs, commits, checksums, and prerelease status. +The OpenAPI 3.0 nullable keyword is adapted at validation time only. +No final TRAPI 2.0.0 release was available when pinned; beta2 is explicitly opt-in. diff --git a/TCT/schemas/manifest.json b/TCT/schemas/manifest.json new file mode 100644 index 0000000..c553b31 --- /dev/null +++ b/TCT/schemas/manifest.json @@ -0,0 +1,14 @@ +{ + "1.5.0": { + "url": "https://raw.githubusercontent.com/NCATSTranslator/ReasonerAPI/4edf6c6ea31c9da29d344abb876998c9d19e4ba5/TranslatorReasonerAPI.yaml", + "commit": "4edf6c6ea31c9da29d344abb876998c9d19e4ba5", + "sha256": "92ea525f3a931a9b7e0952e2e17b2e4f71405765664ad802ff82b18f32ada147", + "prerelease": false + }, + "2.0.0-beta2": { + "url": "https://raw.githubusercontent.com/NCATSTranslator/ReasonerAPI/14638b662fdad1b131487d8b7d4001b70f98e956/TranslatorReasonerAPI.yaml", + "commit": "14638b662fdad1b131487d8b7d4001b70f98e956", + "sha256": "7a43d77303824b9e419ce5f32484b4c76daac564d94fba6960392ce5542da41e", + "prerelease": true + } +} diff --git a/TCT/schemas/trapi-1.5.0.yaml b/TCT/schemas/trapi-1.5.0.yaml new file mode 100644 index 0000000..0b5aad8 --- /dev/null +++ b/TCT/schemas/trapi-1.5.0.yaml @@ -0,0 +1,1625 @@ +openapi: 3.0.1 +info: + description: INSERT-DESCRIPTION-OF-YOUR-SERVICE-HERE + version: INSERT-VERSION-OF-YOUR-SERVICE-HERE + title: INSERT-TITLE-OF-YOUR-SERVICE-HERE + contact: + email: INSERT@EMAIL-ADDRESS-OF-IMPLEMENTER-HERE.ORG + license: + name: Apache 2.0 + url: http://www.apache.org/licenses/LICENSE-2.0.html + termsOfService: INSERT-URL-HERE + x-translator: + component: INSERT-EITHER-ARA-OR-KP-OR-ARS-OR-Utility-HERE + team: + - INSERT-ONE-TRANSLATOR-TEAM-NAME-PER-ELEMENT-HERE + biolink-version: INSERT-BIOLINK-MODEL-VERSION-HERE + infores: INSERT-INFORES-CURIE-OF-THIS-RESOURCE-HERE + externalDocs: + description: >- + The values for component and team are restricted according to this + external JSON schema. See schema and examples at url + url: https://github.com/NCATSTranslator/translator_extensions/blob/\ + production/x-translator/ + x-trapi: + version: 1.5.0 + multicuriequery: REPLACE-WITH-true-IF-multicurie-queries-ARE-IMPLEMENTED-ELSE-false + asyncquery: REPLACE-WITH-true-IF-/asyncquery-IS-IMPLEMENTED-ELSE-false + pathfinderquery: REPLACE-WITH-true-IF-pathfinder-queries-ARE-IMPLEMENTED-ELSE-false + operations: + - LIST-ALL-SUPPORTED-OPERATION-IDS-HERE-OR-REMOVE-THIS-SECTION + batch_size_limit: ADD-BATCH-SIZE-LIMIT-OR-REMOVE-IF-NO-LIMIT + rate_limit: ADD-REQUEST-RATE-LIMIT-OR-REMOVE-IF-NO-LIMIT + test_data_location: ADD-URL-THAT-RESOLVES-TO-SRI-TESTING-HARNESS-DATA + externalDocs: + description: >- + The values for version are restricted according to the regex in + this external JSON schema. See schema and examples at url + url: https://github.com/NCATSTranslator/translator_extensions/blob/\ + production/x-trapi/ +servers: + - url: INSERT-URL-HERE + description: INSERT-SERVER-DESCRIPTION-HERE +externalDocs: + description: >- + Documentation for the NCATS Biomedical Translator Reasoners web services + url: https://github.com/NCATSTranslator/ReasonerAPI +tags: + - name: meta_knowledge_graph + description: >- + Retrieve the meta knowledge graph representation of this + TRAPI web service. KPs MUST provide all subject category - predicate - + object category triplets that are supported by the service, + NOT including all implied ancestor relationships. ARAs SHOULD provide + the union of all meta knowledge graphs of all the KPs that they can + consult. + externalDocs: + description: >- + Documentation for the reasoner meta_knowledge_graph function + url: INSERT-URL-HERE-OR-REMOVE-EXTERNALDOCS-IF-NA + - name: query + description: Initiate a query and wait to receive the response + externalDocs: + description: Documentation for the reasoner query function + url: INSERT-URL-HERE-OR-REMOVE-EXTERNALDOCS-IF-NA + - name: asyncquery + description: Initiate a query with a callback to receive the response + externalDocs: + description: Documentation for the reasoner asynchquery function + url: INSERT-URL-HERE-OR-REMOVE-EXTERNALDOCS-IF-NA + - name: asyncquery_status + description: >- + Retrieve the current status of a previously submitted + asyncquery given its job_id + - name: translator + description: Required for SmartAPI validation of x-translator + - name: trapi + description: Required for SmartAPI validation of x-trapi +paths: + /meta_knowledge_graph: + get: + tags: + - meta_knowledge_graph + summary: Meta knowledge graph representation of this TRAPI web service. + responses: + '200': + description: >- + Returns meta knowledge graph representation of this TRAPI web + service. + content: + application/json: + schema: + $ref: '#/components/schemas/MetaKnowledgeGraph' + /query: + post: + tags: + - query + summary: Initiate a query and wait to receive a Response + description: '' + requestBody: + description: Query information to be submitted + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/Query' + responses: + '200': + description: >- + OK. There may or may not be results. Note that some of the provided + identifiers may not have been recognized. + content: + application/json: + schema: + $ref: '#/components/schemas/Response' + '400': + description: >- + Bad request. The request is invalid according to this OpenAPI + schema OR a specific identifier is believed to be invalid somehow + (not just unrecognized). + content: + application/json: + schema: + type: string + '413': + description: >- + Payload too large. Indicates that batch size was over the limit + specified in x-trapi. + content: + application/json: + schema: + type: string + '429': + description: >- + Too many requests. Indicates that the client issued requests that + exceed the rate limit specified in x-trapi. + content: + application/json: + schema: + type: string + '500': + description: >- + Internal server error. + content: + application/json: + schema: + type: string + '501': + description: >- + Not implemented. + content: + application/json: + schema: + type: string + /asyncquery: + post: + tags: + - asyncquery + summary: Initiate a query with a callback to receive the response + description: '' + requestBody: + description: Query information to be submitted + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/AsyncQuery' + responses: + '200': + description: >- + The query is accepted for processing and the Response will be + sent to the callback url when complete. + content: + application/json: + schema: + $ref: '#/components/schemas/AsyncQueryResponse' + '400': + description: >- + Bad request. The request is invalid according to this OpenAPI + schema OR a specific identifier is believed to be invalid somehow + (not just unrecognized). + content: + application/json: + schema: + type: string + '413': + description: >- + Payload too large. Indicates that batch size was over the limit + specified in x-trapi. + content: + application/json: + schema: + type: string + '429': + description: >- + Too many requests. Indicates that the client issued requests that + exceed the rate limit specified in x-trapi. + content: + application/json: + schema: + type: string + '500': + description: >- + Internal server error. + content: + application/json: + schema: + type: string + '501': + description: >- + Not implemented. + content: + application/json: + schema: + type: string + /asyncquery_status/{job_id}: + get: + tags: + - asyncquery_status + summary: >- + Retrieve the current status of a previously submitted + asyncquery given its job_id + operationId: asyncquery_status + parameters: + - in: path + name: job_id + description: Identifier of the job for status request + example: rXEOAosN3L + required: true + schema: + type: string + responses: + '200': + description: >- + Returns the status and current logs of a previously + submitted asyncquery. + content: + application/json: + schema: + $ref: '#/components/schemas/AsyncQueryStatusResponse' + '404': + description: job_id not found + '501': + description: >- + Return code 501 indicates that this endpoint has not been + implemented at this site. Sites that implement /asyncquery + MUST implement /asyncquery_status/{job_id}, but those that + do not implement /asyncquery SHOULD NOT implement + /asyncquery_status. + content: + application/json: + schema: + type: string +components: + schemas: + Query: + description: >- + The Query class is used to package a user request for information. A + Query object consists of a required Message object with optional + additional properties. Additional properties are intended to convey + implementation-specific or query-independent parameters. For example, + an additional property specifying a log level could allow a user to + override the default log level in order to receive more fine-grained + log information when debugging an issue. + x-body-name: request_body + type: object + properties: + message: + oneOf: + - $ref: '#/components/schemas/Message' + nullable: false + description: >- + The query Message is a serialization of the user request. Content + of the Message object depends on the intended TRAPI operation. For + example, the fill operation requires a non-empty query_graph field + as part of the Message, whereas other operations, e.g. overlay, + require non-empty results and knowledge_graph fields. + log_level: + description: The least critical level of logs to return + oneOf: + - $ref: '#/components/schemas/LogLevel' + nullable: true + workflow: + description: List of workflow steps to be executed. + oneOf: + - $ref: https://standards.ncats.io/workflow/1.3.5/schema + nullable: true + submitter: + type: string + description: >- + Any string for self-identifying the submitter of a query. The + purpose of this optional field is to aid in the tracking of + the source of queries for development and issue resolution. + nullable: true + bypass_cache: + type: boolean + default: false + description: >- + Set to true in order to request that the agent obtain + fresh information from its sources in all cases where + it has a viable choice between requesting fresh information + in real time and using cached information. The agent + receiving this flag MUST also include it in TRAPI sent to + downstream sources (e.g., ARS -> ARAs -> KPs). + additionalProperties: true + required: + - message + AsyncQuery: + description: >- + The AsyncQuery class is effectively the same as the Query class but + it requires a callback property. + x-body-name: request_body + type: object + properties: + callback: + type: string + nullable: false + format: uri + pattern: ^https?:// + description: >- + Upon completion, this server will send a POST request to the + callback URL with `Content-Type: application/json` header and + request body containing a JSON-encoded `Response` object. + The server MAY POST `Response` objects before work is fully + complete to provide interim results with a Response.status + value of 'Running'. If a POST operation to the callback URL + does not succeed, the server SHOULD retry the POST at least + once. + message: + oneOf: + - $ref: '#/components/schemas/Message' + nullable: false + description: >- + The query Message is a serialization of the user request. Content + of the Message object depends on the intended TRAPI operation. For + example, the fill operation requires a non-empty query_graph field + as part of the Message, whereas other operations, e.g. overlay, + require non-empty results and knowledge_graph fields. + log_level: + description: The least critical level of logs to return + oneOf: + - $ref: '#/components/schemas/LogLevel' + nullable: true + workflow: + description: List of workflow steps to be executed. + oneOf: + - $ref: https://standards.ncats.io/workflow/1.3.5/schema + nullable: true + submitter: + type: string + description: >- + Any string for self-identifying the submitter of a query. The + purpose of this optional field is to aid in the tracking of + the source of queries for development and issue resolution. + nullable: true + bypass_cache: + type: boolean + default: false + description: >- + Set to true in order to request that the agent obtain + fresh information from its sources in all cases where + it has a viable choice between requesting fresh information + in real time and using cached information. The agent + receiving this flag MUST also include it in TRAPI sent to + downstream sources (e.g., ARS -> ARAs -> KPs). + additionalProperties: true + required: + - callback + - message + AsyncQueryResponse: + type: object + description: >- + The AsyncQueryResponse object contains a payload that must be + returned from a submitted async_query. + properties: + status: + description: >- + One of a standardized set of short codes: + e.g. Accepted, QueryNotTraversable, KPsNotAvailable + type: string + example: Accepted + nullable: true + description: + description: >- + A brief human-readable description of the result of the + async_query submission. + type: string + example: Async_query has been queued + nullable: true + job_id: + description: >- + An identifier for the submitted job that can be used with + /async_query_status to receive an update on the status of + the job. + type: string + example: rXEOAosN3L + nullable: false + additionalProperties: true + required: + - job_id + AsyncQueryStatusResponse: + type: object + description: >- + The AsyncQueryStatusResponse object contains a payload that describes + the current status of a previously submitted async_query. + properties: + status: + description: >- + One of a standardized set of short codes: + Queued, Running, Completed, Failed + type: string + example: Running + nullable: false + description: + description: >- + A brief human-readable description of the current state + or summary of the problem if the status is Failed. + type: string + example: Callback URL returned 500 + nullable: false + logs: + description: >- + A list of LogEntry items, containing errors, warnings, debugging + information, etc. List items MUST be in chronological order with + earliest first. The most recent entry should be last. Its timestamp + will be compared against the current time to see if there is + still activity. + type: array + items: + $ref: '#/components/schemas/LogEntry' + minItems: 1 + nullable: false + response_url: + description: >- + Optional URL that can be queried to restrieve the full TRAPI + Response. + type: string + example: https://arax.ncats.io/api/arax/v1.3/response/116481 + nullable: true + additionalProperties: true + required: + - status + - description + - logs + Response: + type: object + description: >- + The Response object contains the main payload when a TRAPI query + endpoint interprets and responds to the submitted query successfully + (i.e., HTTP Status Code 200). The message property contains the + knowledge of the response (query graph, knowledge graph, and results). + The status, description, and logs properties provide additional details + about the response. + properties: + message: + description: >- + Contains the knowledge of the response (query graph, knowledge + graph, and results). + oneOf: + - $ref: '#/components/schemas/Message' + nullable: false + status: + description: >- + One of a standardized set of short codes, + e.g. Success, QueryNotTraversable, KPsNotAvailable + type: string + example: Success + nullable: true + description: + description: A brief human-readable description of the outcome + type: string + example: Success. 42 results found. + nullable: true + logs: + description: >- + A list of LogEntry items, containing errors, warnings, debugging + information, etc. List items MUST be in chronological order with + earliest first. + type: array + items: + $ref: '#/components/schemas/LogEntry' + minItems: 0 + nullable: false + workflow: + description: List of workflow steps that were executed. + oneOf: + - $ref: https://standards.ncats.io/workflow/1.3.5/schema + nullable: true + schema_version: + type: string + example: 1.4.0 + description: Version label of the TRAPI schema used in this document + nullable: true + biolink_version: + type: string + example: 3.1.2 + description: Version label of the Biolink model used in this document + nullable: true + additionalProperties: true + required: + - message + Message: + description: >- + The message object holds the main content of a Query or a Response in + three properties: query_graph, results, and knowledge_graph. + The query_graph property contains the query configuration, the results + property contains any answers that are returned by the service, + and knowledge_graph property contains lists of edges and nodes in the + thought graph corresponding to this message. The content of these + properties is context-dependent to the encompassing object and + the TRAPI operation requested. + type: object + properties: + results: + description: >- + List of all returned Result objects for the query posed. + The list SHOULD NOT be assumed to be ordered. The 'score' property, + if present, MAY be used to infer result rankings. If Results are + not expected (such as for a query Message), this property SHOULD + be null or absent. If Results are expected (such as for a response + Message) and no Results are available, this property SHOULD be an + array with 0 Results in it. + type: array + items: + $ref: '#/components/schemas/Result' + nullable: true + minItems: 0 + query_graph: + description: >- + QueryGraph object that contains a serialization of a query in the + form of a graph + oneOf: + - $ref: '#/components/schemas/QueryGraph' + nullable: true + knowledge_graph: + description: >- + KnowledgeGraph object that contains lists of nodes and edges + in the thought graph corresponding to the message + oneOf: + - $ref: '#/components/schemas/KnowledgeGraph' + nullable: true + auxiliary_graphs: + type: object + description: >- + Dictionary of AuxiliaryGraph instances that are used by Knowledge + Graph Edges and Result Analyses. These are referenced elsewhere by + the dictionary key. + additionalProperties: + $ref: '#/components/schemas/AuxiliaryGraph' + nullable: true + additionalProperties: false + LogEntry: + description: >- + The LogEntry object contains information useful for tracing + and debugging across Translator components. Although an + individual component (for example, an ARA or KP) may have its + own logging and debugging infrastructure, this internal + information is not, in general, available to other components. + In addition to a timestamp and logging level, LogEntry + includes a string intended to be read by a human, along with + one of a standardized set of codes describing the condition of + the component sending the message. + type: object + properties: + timestamp: + type: string + format: date-time + description: >- + Timestamp in ISO 8601 format, providing the LogEntry time + either in univeral coordinated time (UTC) using the 'Z' tag + (e.g 2020-09-03T18:13:49Z), or, if local time is provided, + the timezone offset must be provided + (e.g. 2020-09-03T18:13:49-04:00). + example: '2020-09-03T18:13:49+00:00' + nullable: false + level: + oneOf: + - $ref: '#/components/schemas/LogLevel' + nullable: true + code: + type: string + description: >- + One of a standardized set of short codes + e.g. QueryNotTraversable, KPNotAvailable, KPResponseMalformed + nullable: true + message: + type: string + description: A human-readable log message + nullable: false + additionalProperties: true + required: + - timestamp + - message + LogLevel: + type: string + description: Logging level + enum: + - ERROR + - WARNING + - INFO + - DEBUG + Result: + type: object + description: >- + A Result object specifies the nodes and edges in the knowledge graph + that satisfy the structure or conditions of a user-submitted query + graph. It must contain a NodeBindings object (list of query graph node + to knowledge graph node mappings) and a list of Analysis objects. + properties: + node_bindings: + type: object + description: >- + The dictionary of Input Query Graph to Result Knowledge Graph node + bindings where the dictionary keys are the key identifiers of the + Query Graph nodes and the associated values of those keys are + instances of NodeBinding schema type (see below). This value is an + array of NodeBindings since a given query node may have multiple + knowledge graph Node bindings in the result. + additionalProperties: + type: array + items: + $ref: '#/components/schemas/NodeBinding' + minItems: 1 + nullable: false + analyses: + type: array + description: >- + The list of all Analysis components that contribute to the result. + See below for Analysis components. + items: + $ref: '#/components/schemas/Analysis' + minItems: 0 + nullable: false + additionalProperties: true + required: + - node_bindings + - analyses + NodeBinding: + type: object + description: >- + An instance of NodeBinding is a single KnowledgeGraph Node mapping, + identified by the corresponding 'id' object key identifier of the + Node within the Knowledge Graph. Instances of NodeBinding may + include extra annotation in the form of additional properties. + (such annotation is not yet fully standardized). Each Node + Binding must bind directly to node in the original Query Graph. + properties: + id: + oneOf: + - $ref: '#/components/schemas/CURIE' + nullable: false + description: >- + The CURIE of a Node within the Knowledge Graph. + query_id: + oneOf: + - $ref: '#/components/schemas/CURIE' + description: >- + An optional property to provide the CURIE in the QueryGraph to + which this binding applies. If the bound QNode does not have an + an 'id' property or if it is empty, then this query_id MUST be + null or absent. If the bound QNode has one or more CURIEs + as an 'id' and this NodeBinding's 'id' refers to a QNode 'id' + in a manner where the CURIEs are different (typically due to + the NodeBinding.id being a descendant of a QNode.id), then + this query_id MUST be provided. In other cases, there is no + ambiguity, and this query_id SHOULD NOT be provided. + nullable: true + attributes: + type: array + description: >- + A list of attributes providing further information about the + node binding. This is not intended for capturing node attributes + and should only be used for properties that vary from result to + result. + items: + $ref: '#/components/schemas/Attribute' + minItems: 0 + nullable: false + additionalProperties: true + required: + - id + - attributes + Analysis: + type: object + description: >- + An analysis is a dictionary that contains information about + the result tied to a particular service. Each Analysis is + generated by a single reasoning service, and describes the + outputs of analyses performed by the reasoner on a particular + Result (e.g. a result score), along with provenance information + supporting the analysis (e.g. method or data that supported + generation of the score). + properties: + resource_id: + $ref: '#/components/schemas/CURIE' + description: The id of the resource generating this Analysis + score: + type: number + format: float + example: 163.233 + description: >- + A numerical score associated with this result indicating the + relevance or confidence of this result relative to others in the + returned set. Higher MUST be better. + nullable: true + edge_bindings: + type: object + description: >- + The dictionary of input Query Graph to Knowledge Graph edge + bindings where the dictionary keys are the key identifiers of the + Query Graph edges and the associated values of those keys are + instances of EdgeBinding schema type (see below). This value is an + array of EdgeBindings since a given query edge may resolve to + multiple Knowledge Graph Edges. + additionalProperties: + type: array + items: + $ref: '#/components/schemas/EdgeBinding' + support_graphs: + type: array + description: >- + This is a list of references to Auxiliary Graph instances + that supported the analysis of a Result as performed by the + reasoning service. Each item in the list is the key of a + single Auxiliary Graph. + nullable: true + items: + type: string + scoring_method: + type: string + description: >- + An identifier and link to an explanation for the method used + to generate the score + nullable: true + attributes: + type: array + description: >- + The attributes of this particular Analysis. + items: + $ref: '#/components/schemas/Attribute' + nullable: true + additionalProperties: true + required: + - resource_id + - edge_bindings + EdgeBinding: + type: object + description: >- + A instance of EdgeBinding is a single KnowledgeGraph Edge mapping, + identified by the corresponding 'id' object key identifier of the + Edge within the Knowledge Graph. Instances of EdgeBinding may include + extra annotation (such annotation is not yet fully standardized). + Edge bindings are captured within a specific reasoner's Analysis + object because the Edges in the Knowledge Graph that get bound to + the input Query Graph may differ between reasoners. + properties: + id: + type: string + description: The key identifier of a specific KnowledgeGraph Edge. + nullable: false + attributes: + type: array + description: >- + A list of attributes providing further information about the + edge binding. This is not intended for capturing edge attributes + and should only be used for properties that vary from result to + result. + items: + $ref: '#/components/schemas/Attribute' + minItems: 0 + nullable: false + additionalProperties: true + required: + - id + - attributes + AuxiliaryGraph: + type: object + description: >- + A single AuxiliaryGraph instance that is used by Knowledge Graph + Edges and Result Analyses. Edges comprising an Auxiliary Graph + are a subset of the Knowledge Graph in the message. Data creators + can create an AuxiliaryGraph to assemble a specific collections + of edges from the Knowledge Graph into a named graph that can be + referenced from an Edge as evidence/explanation supporting that Edge, + or from a Result Analysis as information used to generate a score. + properties: + edges: + type: array + description: >- + List of edges that form the Auxiliary Graph. Each item is a + reference to a single Knowledge Graph edge + items: + type: string + nullable: false + minItems: 1 + attributes: + type: array + description: >- + Attributes of the Auxiliary Graph + items: + $ref: '#/components/schemas/Attribute' + minItems: 0 + nullable: false + additionalProperties: true + required: + - edges + - attributes + KnowledgeGraph: + type: object + description: >- + The knowledge graph associated with a set of results. The instances + of Node and Edge defining this graph represent instances of + biolink:NamedThing (concept nodes) and biolink:Association + (relationship edges) representing (Attribute) annotated knowledge + returned from the knowledge sources and inference agents wrapped by + the given TRAPI implementation. + properties: + nodes: + type: object + description: >- + Dictionary of Node instances used in the KnowledgeGraph, + referenced elsewhere in the TRAPI output by the dictionary key. + additionalProperties: + $ref: '#/components/schemas/Node' + edges: + type: object + description: >- + Dictionary of Edge instances used in the KnowledgeGraph, + referenced elsewhere in the TRAPI output by the dictionary key. + additionalProperties: + $ref: '#/components/schemas/Edge' + additionalProperties: true + required: + - nodes + - edges + QueryGraph: + type: object + description: >- + A graph representing a biomedical question. It serves as a template for + each result (answer), where each bound knowledge graph node/edge is + expected to obey the constraints of the associated query graph element. + properties: + nodes: + type: object + description: >- + The node specifications. The keys of this map are unique node + identifiers and the corresponding values include the constraints + on bound nodes. + additionalProperties: + $ref: '#/components/schemas/QNode' + edges: + type: object + description: >- + The edge specifications. The keys of this map are unique edge + identifiers and the corresponding values include the constraints + on bound edges, in addition to specifying the subject and object + QNodes. + additionalProperties: + $ref: '#/components/schemas/QEdge' + additionalProperties: true + required: + - nodes + - edges + QNode: + type: object + description: A node in the QueryGraph used to represent an entity in a + query. If a CURIE is not specified, any nodes matching the category + of the QNode will be returned in the Results. + properties: + ids: + type: array + items: + $ref: '#/components/schemas/CURIE' + minItems: 1 + example: ["OMIM:603903"] + description: >- + A CURIE identifier (or list of identifiers) for this node. + The 'ids' field will hold a list of CURIEs only in the case of a + BATCH set_interpretation, where each CURIE is queried + separately. If a list of queried CURIEs is to be considered as a + set (as under a MANY or ALL set_interpretation), the 'ids' field + will hold a single id representing this set, and the individual members + of this set will be captured in a separate 'member_ids' field. + Note that the set id MUST be created as a UUID by the system that + defines the queried set, using a centralized nodenorm service. + Note also that downstream systems MUST re-use the original set UUID + in the messages they create/send, which will facilitate merging or + caching operations. + nullable: true + categories: + type: array + description: >- + These should be Biolink Model categories and are allowed + to be of type 'abstract' or 'mixin' (only in QGraphs!). + Use of 'deprecated' categories should be avoided. + items: + $ref: '#/components/schemas/BiolinkEntity' + minItems: 1 + nullable: true + set_interpretation: + type: string + description: >- + Indicates how multiple CURIEs in the ids property MUST be + interpreted. BATCH indicates that the query is intended to be + a batch query and each CURIE is treated independently. ALL means + that all specified CURIES MUST appear in each Result. + MANY means that member CURIEs MUST form one or more + sets in the Results, and sets with more members are generally + considered more desirable that sets with fewer members. + If this property is missing or null, the default is BATCH. + enum: + - BATCH + - ALL + - MANY + nullable: true + member_ids: + type: array + description: >- + A list of CURIE identifiers for members of a queried set. This + field MUST be populated under a set_interpretation of MANY + or ALL, when the 'ids' field holds a UUID representing the set + itself. This field MUST NOT be used under a set_interpretation + of BATCH. + nullable: true + items: + $ref: '#/components/schemas/CURIE' + constraints: + type: array + description: >- + A list of constraints applied to a query node. + If there are multiple items, they must all be true (equivalent + to AND) + items: + $ref: '#/components/schemas/AttributeConstraint' + default: [] + additionalProperties: true + QEdge: + type: object + description: >- + An edge in the QueryGraph used as a filter pattern specification in a + query. If the optional predicate property is not specified, + it is assumed to be a wildcard match to the target knowledge space. + If specified, the ontological inheritance hierarchy associated with + the term provided is assumed, such that edge bindings returned may be + an exact match to the given QEdge predicate term, + or to a term that is a descendant of the QEdge predicate term. + properties: + knowledge_type: + description: >- + Indicates the type of knowledge that the client wants from the + server between the subject and object. If the value is + 'lookup', then the client wants direct lookup information from + knowledge sources. If the value is 'inferred', then the client + wants the server to get creative and connect the subject and + object in more speculative and non-direct-lookup ways. If this + property is absent or null, it MUST be assumed to mean + 'lookup'. This feature is currently experimental and may be + further extended in the future. + example: lookup + nullable: true + type: string + predicates: + type: array + description: >- + These should be Biolink Model predicates and are allowed to be of + type 'abstract' or 'mixin' (only in QGraphs!). Use of 'deprecated' + predicates should be avoided. + items: + $ref: '#/components/schemas/BiolinkPredicate' + minItems: 1 + nullable: true + subject: + type: string + example: https://omim.org/entry/603903 + description: >- + Corresponds to the map key identifier of the + subject concept node anchoring the query filter + pattern for the query relationship edge. + object: + type: string + example: https://www.uniprot.org/uniprot/P00738 + description: >- + Corresponds to the map key identifier of the + object concept node anchoring the query filter + pattern for the query relationship edge. + attribute_constraints: + type: array + description: >- + A list of attribute constraints applied to a query edge. + If there are multiple items, they must all be true (equivalent + to AND) + items: + $ref: '#/components/schemas/AttributeConstraint' + default: [] + qualifier_constraints: + type: array + description: >- + A list of QualifierConstraints that provide nuance to the QEdge. + If multiple QualifierConstraints are provided, there is an OR + relationship between them. If the QEdge has multiple + predicates or if the QNodes that correspond to the subject or + object of this QEdge have multiple categories or multiple + curies, then qualifier_constraints MUST NOT be specified + because these complex use cases are not supported at this time. + items: + $ref: '#/components/schemas/QualifierConstraint' + default: [] + additionalProperties: true + required: + - subject + - object + Node: + type: object + description: >- + A node in the KnowledgeGraph which represents some biomedical + concept. Nodes are identified by the keys in the KnowledgeGraph + Node mapping. + properties: + name: + type: string + example: Haptoglobin + description: Formal name of the entity + nullable: true + categories: + type: array + description: >- + These should be Biolink Model categories and are NOT allowed + to be of type 'abstract' or 'mixin'. Returning 'deprecated' + categories should also be avoided. + items: + $ref: '#/components/schemas/BiolinkEntity' + minItems: 1 + nullable: false + attributes: + type: array + description: A list of attributes describing the node + items: + $ref: '#/components/schemas/Attribute' + minItems: 0 + nullable: false + is_set: + type: boolean + description: >- + Indicates that the node represents a set of entities. + If this property is missing or null, it is assumed to be + false. + nullable: true + required: + - categories + - attributes + additionalProperties: false + Attribute: + type: object + description: >- + Generic attribute for a node or an edge that expands the key-value + pair concept by including fields for additional metadata. These fields + can be used to describe the source of the statement made in a key-value + pair of the attribute object, or describe the attribute's value itself + including its semantic type, or a url providing additional information + about it. An attribute may be further qualified with sub-attributes + (for example to provide confidence intervals on a value). + properties: + attribute_type_id: + $ref: '#/components/schemas/CURIE' + description: >- + The 'key' of the attribute object, holding a CURIE of an ontology + property defining the attribute (preferably the CURIE of a + Biolink association slot). This property captures the relationship + asserted to hold between the value of the attribute, and the node + or edge from which it hangs. For example, that a value of + '0.000153' represents a p-value supporting an edge, or that + a value of 'ChEMBL' represents the original source of the knowledge + expressed in the edge. + example: biolink:synonym + original_attribute_name: + type: string + description: >- + The term used by the original source of an attribute to describe + the meaning or significance of the value it captures. This may be + a column name in a source tsv file, or a key in a source json + document for the field in the data that held the attribute's + value. Capturing this information where possible lets us preserve + what the original source said. Note that the data type is string' + but the contents of the field could also be a CURIE of a third + party ontology term. + example: p-value + nullable: true + value: + description: >- + Value of the attribute. May be any data type, including a list. + example: 0.000153 + value_type_id: + oneOf: + - $ref: '#/components/schemas/CURIE' + description: >- + CURIE describing the semantic type of an attribute's value. Use + a Biolink class if possible, otherwise a term from an external + ontology. If a suitable CURIE/identifier does not exist, enter a + descriptive phrase here and submit the new type for consideration + by the appropriate authority. + example: EDAM:data_1187 + nullable: true + attribute_source: + type: string + description: >- + The source of the core assertion made by the key-value pair of an + attribute object. Use a CURIE or namespace designator for this + resource where possible. + example: UniProtKB + nullable: true + value_url: + type: string + description: >- + Human-consumable URL linking to a web document that provides + additional information about an attribute's value (not the node + or the edge fom which it hangs). + example: https://pubmed.ncbi.nlm.nih.gov/32529952 + nullable: true + description: + type: string + description: >- + Human-readable description for the attribute and its value. + example: Assertion Authored By Dr. Trans L. Ator + nullable: true + attributes: + type: array + description: >- + A list of attributes providing further information about the + parent attribute (for example to provide provenance + information about the parent attribute). + items: + $ref: '#/components/schemas/Attribute' + nullable: true + required: + - attribute_type_id + - value + additionalProperties: false + Edge: + type: object + description: >- + A specification of the semantic relationship linking two concepts + that are expressed as nodes in the knowledge "thought" graph + resulting from a query upon the underlying knowledge source. + properties: + predicate: + description: >- + The type of relationship between the subject and object + for the statement expressed in an Edge. These should be + Biolink Model predicate terms and are NOT allowed + to be of type 'abstract' or 'mixin'. Returning 'deprecated' + predicate terms should also be avoided. + example: biolink:gene_associated_with_condition + oneOf: + - $ref: '#/components/schemas/BiolinkPredicate' + nullable: false + subject: + description: >- + Corresponds to the map key CURIE of the + subject concept node of this relationship edge. + example: MONDO:0011382 + oneOf: + - $ref: '#/components/schemas/CURIE' + nullable: false + object: + description: >- + Corresponds to the map key CURIE of the + object concept node of this relationship edge. + example: UniProtKB:P00738 + oneOf: + - $ref: '#/components/schemas/CURIE' + nullable: false + attributes: + description: A list of additional attributes for this edge + items: + $ref: '#/components/schemas/Attribute' + nullable: true + type: array + qualifiers: + description: >- + A set of Qualifiers that act together to add nuance + or detail to the statement expressed in an Edge. + items: + $ref: '#/components/schemas/Qualifier' + nullable: true + type: array + sources: + type: array + description: >- + A list of RetrievalSource objects that provide information + about how a particular Information Resource served + as a source from which the knowledge expressed in an Edge, + or data used to generate this knowledge, was retrieved. + items: + $ref: '#/components/schemas/RetrievalSource' + minItems: 1 + nullable: false + additionalProperties: false + required: + - object + - predicate + - subject + - sources + Qualifier: + additionalProperties: false + description: >- + An additional nuance attached to an assertion + type: object + properties: + qualifier_type_id: + $ref: '#/components/schemas/CURIE' + description: >- + CURIE for a Biolink 'qualifier' association slot, generally taken + from Biolink association slots designated for this purpose + (that is, association slots with names ending in 'qualifier') + e.g. biolink:subject_aspect_qualifier, + biolink:subject_direction_qualifier, + biolink:object_aspect_qualifier, etc. Such qualifiers are used + to elaborate a second layer of meaning of a knowledge graph edge. + Available qualifiers are edge properties in the Biolink Model (see + https://biolink.github.io/biolink-model/docs/edge_properties.html) + which have slot names with the suffix string 'qualifier'. + pattern: ^biolink:[a-z][a-z_]*$ + example: biolink:subject_aspect_qualifier + nullable: false + qualifier_value: + type: string + description: >- + The value associated with the type of the qualifier, drawn from + a set of controlled values by the type as specified in + the Biolink model (e.g. 'expression' or 'abundance' for the + qualifier type 'biolink:subject_aspect_qualifier', etc). + The enumeration of qualifier values for a given qualifier + type is generally going to be constrained by the category + of edge (i.e. biolink:Association subtype) of the (Q)Edge. + example: expression + nullable: false + required: + - qualifier_type_id + - qualifier_value + QualifierConstraint: + additionalProperties: false + description: >- + Defines a query constraint based on the + qualifier_types and qualifier_values of a set of + Qualifiers attached to an edge. For example, it can constrain a + "ChemicalX - affects - ?Gene" query to return only edges where + ChemicalX specifically affects the 'expression' of the Gene, by + constraining on the qualifier_type "biolink:object_aspect_qualifier" + with a qualifier_value of "expression". + properties: + qualifier_set: + type: array + description: >- + A set of Qualifiers that serves to add nuance to a query, + by constraining allowed values held by Qualifiers + on queried Edges. + items: + $ref: '#/components/schemas/Qualifier' + nullable: false + required: + - qualifier_set + type: object + BiolinkEntity: + description: >- + Compact URI (CURIE) for a Biolink class, biolink:NamedThing + or a child thereof. The CURIE must use the prefix 'biolink:' + followed by the PascalCase class name. + type: string + pattern: ^biolink:[A-Z][a-zA-Z]*$ + externalDocs: + description: Biolink model entities + url: https://biolink.github.io/biolink-model/docs/NamedThing.html + example: biolink:PhenotypicFeature + BiolinkPredicate: + description: >- + CURIE for a Biolink 'predicate' slot, taken from the Biolink slot + ('is_a') hierarchy rooted in biolink:related_to (snake_case). This + predicate defines the Biolink relationship between the subject and + object nodes of a biolink:Association defining a knowledge graph edge. + type: string + pattern: ^biolink:[a-z][a-z_]*$ + externalDocs: + description: Biolink model predicates + url: https://biolink.github.io/biolink-model/docs/related_to.html + example: biolink:interacts_with + CURIE: + type: string + description: >- + A Compact URI, consisting of a prefix and a reference separated + by a colon, such as UniProtKB:P00738. Via an external context + definition, the CURIE prefix and colon may be replaced by a URI + prefix, such as http://identifiers.org/uniprot/, to form a full + URI. + externalDocs: + url: https://www.w3.org/TR/2010/NOTE-curie-20101216/ + MetaKnowledgeGraph: + type: object + description: >- + Knowledge-map representation of this TRAPI web service. The meta + knowledge graph is composed of the union of most specific categories + and predicates for each node and edge. + properties: + nodes: + type: object + description: >- + Collection of the most specific node categories provided by + this TRAPI web service, indexed by Biolink class CURIEs. + A node category is only exposed here if there is + node for which that is the most specific category available. + additionalProperties: + $ref: '#/components/schemas/MetaNode' + edges: + type: array + description: >- + List of the most specific edges/predicates provided by this TRAPI + web service. A predicate is only exposed here if there is an edge + for which the predicate is the most specific available. + items: + $ref: '#/components/schemas/MetaEdge' + required: + - nodes + - edges + MetaNode: + type: object + description: >- + Description of a node category provided by this TRAPI web service. + properties: + id_prefixes: + type: array + description: >- + List of CURIE prefixes for the node category that this TRAPI web + service understands and accepts on the input. + items: + type: string + minItems: 1 + example: [CHEMBL.COMPOUND, INCHIKEY] + attributes: + type: array + description: >- + Node attributes provided by this TRAPI web service. + items: + $ref: '#/components/schemas/MetaAttribute' + nullable: true + required: + - id_prefixes + additionalProperties: false + MetaEdge: + type: object + description: >- + Edge in a meta knowledge map describing relationship between a subject + Biolink class and an object Biolink class. + properties: + subject: + $ref: '#/components/schemas/BiolinkEntity' + description: >- + Subject node category of this relationship edge. + example: biolink:ChemicalEntity + predicate: + $ref: '#/components/schemas/BiolinkPredicate' + description: >- + Biolink relationship between the subject and object categories. + example: biolink:affects + object: + $ref: '#/components/schemas/BiolinkEntity' + description: >- + Object node category of this relationship edge. + example: biolink:Protein + knowledge_types: + description: >- + A list of knowledge_types that are supported by the service. + If the knowledge_types is null, this means that only 'lookup' + is supported. Currently allowed values are 'lookup' or 'inferred'. + items: + type: string + minItems: 1 + nullable: true + type: array + attributes: + type: array + description: >- + Edge attributes provided by this TRAPI web service. + items: + $ref: '#/components/schemas/MetaAttribute' + nullable: true + qualifiers: + description: >- + Qualifiers that are possible to be found on this edge type. + items: + $ref: '#/components/schemas/MetaQualifier' + nullable: true + type: array + association: + description: >- + The Biolink association type (entity) that this edge represents. + Associations are classes in Biolink + that represent a relationship between two entities. + For example, the association 'gene interacts with gene' + is represented by the Biolink class, + 'biolink:GeneToGeneAssociation'. If association + is filled out, then the testing harness can + help validate that the qualifiers are being used + correctly. + example: 'biolink:ChemicalToGeneAssociation' + $ref: '#/components/schemas/BiolinkEntity' + required: + - subject + - predicate + - object + additionalProperties: false + MetaQualifier: + type: object + properties: + qualifier_type_id: + $ref: '#/components/schemas/CURIE' + description: >- + The CURIE of the qualifier type. + example: biolink:subject_aspect_qualifier + nullable: false + applicable_values: + type: array + description: >- + The list of values that are possible for this qualifier. + items: + type: string + example: [expression, activity, abundance, degradation] + required: + - qualifier_type_id + MetaAttribute: + type: object + properties: + attribute_type_id: + $ref: '#/components/schemas/CURIE' + description: >- + Type of an attribute provided by this TRAPI web service + (preferably the CURIE of a Biolink association slot) + example: biolink:p_value + attribute_source: + type: string + description: >- + Source of an attribute provided by this TRAPI web service. + example: infores:chembl + nullable: true + original_attribute_names: + type: array + items: + type: string + description: >- + Names of an the attribute as provided by the source. + nullable: true + minItems: 1 + constraint_use: + type: boolean + description: >- + Indicates whether this attribute can be used as a query + constraint. + default: false + constraint_name: + type: string + description: >- + Human-readable name or label for the constraint concept. + Required whenever constraint_use is true. + example: p-value + nullable: true + required: + - attribute_type_id + AttributeConstraint: + type: object + description: >- + Generic query constraint for a query node or query edge + properties: + id: + oneOf: + - $ref: '#/components/schemas/CURIE' + description: >- + CURIE of the concept being constrained. For properties + defined by the Biolink model this SHOULD be a biolink CURIE. + otherwise, if possible, from the EDAM ontology. If a suitable + CURIE does not exist, enter a descriptive phrase here and + submit the new type for consideration by the appropriate + authority. + example: EDAM:data_0844 + name: + type: string + description: >- + Human-readable name or label for the constraint concept. + If appropriate, it SHOULD be the term name of the CURIE used + as the 'id'. This is redundant but required for human + readability. + example: molecular mass + not: + type: boolean + default: false + operator: + type: string + description: >- + Relationship between the database value and the constraint value + for the specified id. The operators ==, >, and < mean + is equal to, is greater than, and is less than, + respectively. The 'matches' operator indicates that the value + is a regular expression to be evaluated. If value is a list type, + then at least one evaluation must be true (equivalent to OR). + This means that the == operator with a list acts like a SQL 'IN' + clause. If the value of the compared attribute is a list, then + comparisons are performed between each of the constraint values + and each of the attribute values, and any one true evaluation + counts as an overall true (e.g., [1,2,3] == [6,7,2] is true). + The == operator is therefore a broad interpretation of inclusion. + The '===' operator requires that the constraint value and + the attribute value be the same data type, length, + content, and order (e.g. only [1,2,3] === [1,2,3]). + The 'not' property negates the operator such that not + and == means 'not equal to' (or 'not in' for a list), and not > + means <=, and not < means >=, not matches means does not + match, and not === means the match between the constraint + and attribute values are not exact. + The '==' operator SHOULD NOT be used in a manner that + describes an "is a" subclass relationship for the parent QNode. + enum: + - == + - '>' + - < + - matches + - === + value: + example: 57.0 + description: >- + Value of the attribute. May be any data type, including a list. + If the value is a list and there are multiple items, at least one + comparison must be true (equivalent to OR) unless the '===' + operator is used. If 'value' is of data + type 'object', the keys of the object MAY be treated as a list. + A 'list' data type paired with the '>' or '<' operators will + encode extraneous comparisons, but this is permitted as it is in + SQL and other languages. + unit_id: + example: UO:0000222 + description: >- + CURIE of the units of the value or list of values in the 'value' + property. The Units of Measurement Ontology (UO) should be used + if possible. The unit_id MUST be provided for (lists of) + numerical values that correspond to a quantity that has units. + nullable: true + unit_name: + example: kilodalton + description: >- + Term name that is associated with the CURIE of the units + of the value or list of values in the 'value' + property. The Units of Measurement Ontology (UO) SHOULD be used + if possible. This property SHOULD be provided if a unit_id is + provided. This is redundant but recommended for human readability. + nullable: true + required: + - name + - id + - operator + - value + additionalProperties: false + RetrievalSource: + type: object + description: >- + Provides information about how a particular InformationResource + served as a source from which knowledge expressed in an Edge, or + data used to generate this knowledge, was retrieved. + properties: + resource_id: + $ref: '#/components/schemas/CURIE' + description: >- + The CURIE for an Information Resource that served as a source + of knowledge expressed in an Edge, or a source of data used to + generate this knowledge. + example: infores:drugbank + nullable: false + resource_role: + $ref: '#/components/schemas/ResourceRoleEnum' + description: >- + The role played by the InformationResource in serving as a + source for an Edge. Note that a given Edge should have one + and only one 'primary' source, and may have any number of + 'aggregator' or 'supporting data' sources. + upstream_resource_ids: + type: array + nullable: true + items: + $ref: '#/components/schemas/CURIE' + description: >- + An upstream InformationResource from which the resource + being described directly retrieved a record of the knowledge + expressed in the Edge, or data used to generate this knowledge. + This is an array because there are cases where a merged Edge + holds knowledge that was retrieved from multiple sources. e.g. + an Edge provided by the ARAGORN ARA can expressing knowledge it + retrieved from both the automat-mychem-info and molepro KPs, + which both provided it with records of this single fact. + example: ["infores:automat-mychem-info", "infores:molepro"] + source_record_urls: + type: array + nullable: true + items: + type: string + description: >- + A URL linking to a specific web page or document provided by the + source, that contains a record of the knowledge expressed in the + Edge. If the knowledge is contained in more than one web page on + an Information Resource's site, urls MAY be provided for each. + For example, Therapeutic Targets Database (TTD) has separate web + pages for 'Imatinib' and its protein target KIT, both of which hold + the claim that 'the KIT protein is a therapeutic target for Imatinib'. + example: >- + [https://db.idrblab.net/ttd/data/drug/details/d0az3c, + https://db.idrblab.net/ttd/data/target/details/t57700] + required: + - resource_id + - resource_role + additionalProperties: true + ResourceRoleEnum: + type: string + description: >- + The role played by the InformationResource in serving as a + source for an Edge. Note that a given Edge should have one + and only one 'primary' source, and may have any number of + 'aggregator' or 'supporting data' sources. This enumeration + is found in Biolink Model, but is repeated here for convenience. + enum: + - primary_knowledge_source + - aggregator_knowledge_source + - supporting_data_source diff --git a/TCT/schemas/trapi-2.0.0-beta2.yaml b/TCT/schemas/trapi-2.0.0-beta2.yaml new file mode 100644 index 0000000..e03ddbb --- /dev/null +++ b/TCT/schemas/trapi-2.0.0-beta2.yaml @@ -0,0 +1,1814 @@ +--- +openapi: 3.1.2 +info: + description: INSERT-DESCRIPTION-OF-YOUR-SERVICE-HERE + version: INSERT-VERSION-OF-YOUR-SERVICE-HERE + title: INSERT-TITLE-OF-YOUR-SERVICE-HERE + contact: + email: INSERT@EMAIL-ADDRESS-OF-IMPLEMENTER-HERE.ORG + license: + name: Apache 2.0 + url: http://www.apache.org/licenses/LICENSE-2.0.html + termsOfService: INSERT-URL-HERE + x-translator: + component: INSERT-EITHER-ARA-OR-KP-OR-ARS-OR-Utility-HERE + team: + - INSERT-ONE-TRANSLATOR-TEAM-NAME-PER-ELEMENT-HERE + biolink-version: INSERT-BIOLINK-MODEL-VERSION-HERE + infores: INSERT-INFORES-CURIE-OF-THIS-RESOURCE-HERE + externalDocs: + description: >- + The values for component and team are restricted according to this + external JSON schema. See schema and examples at url + url: >- + https://github.com/NCATSTranslator/translator_extensions/blob/production/x-translator/ + x-trapi: + version: 2.0.0 + multicuriequery: >- + REPLACE-WITH-true-IF-multicurie-queries-ARE-IMPLEMENTED-ELSE-false + asyncquery: REPLACE-WITH-true-IF-/asyncquery-IS-IMPLEMENTED-ELSE-false + pathfinderquery: >- + REPLACE-WITH-true-IF-pathfinder-queries-ARE-IMPLEMENTED-ELSE-false + operations: + - LIST-ALL-SUPPORTED-OPERATION-IDS-HERE-OR-REMOVE-THIS-SECTION + batch_size_limit: ADD-BATCH-SIZE-LIMIT-OR-REMOVE-IF-NO-LIMIT + rate_limit: ADD-REQUEST-RATE-LIMIT-OR-REMOVE-IF-NO-LIMIT + test_data_location: ADD-URL-THAT-RESOLVES-TO-SRI-TESTING-HARNESS-DATA + externalDocs: + description: >- + The values for version are restricted according to the regex in + this external JSON schema. See schema and examples at url + url: >- + https://github.com/NCATSTranslator/translator_extensions/blob/production/x-trapi/ +servers: + - url: INSERT-URL-HERE + description: INSERT-SERVER-DESCRIPTION-HERE +externalDocs: + description: >- + Documentation for the NCATS Biomedical Translator Reasoners web services + url: https://github.com/NCATSTranslator/ReasonerAPI +tags: + - name: meta_knowledge_graph + description: >- + Retrieve the meta knowledge graph representation of this + TRAPI web service. KPs MUST provide all subject category - predicate - + object category triplets that are supported by the service, + NOT including all implied ancestor relationships. ARAs SHOULD provide + the union of all meta knowledge graphs of all the KPs that they can + consult. + externalDocs: + description: >- + Documentation for the reasoner meta_knowledge_graph function + url: INSERT-URL-HERE-OR-REMOVE-EXTERNALDOCS-IF-NA + - name: query + description: Initiate a query and wait to receive the response + externalDocs: + description: Documentation for the reasoner query function + url: INSERT-URL-HERE-OR-REMOVE-EXTERNALDOCS-IF-NA + - name: asyncquery + description: Initiate a query with a callback to receive the response + externalDocs: + description: Documentation for the reasoner asynchquery function + url: INSERT-URL-HERE-OR-REMOVE-EXTERNALDOCS-IF-NA + - name: asyncquery_status + description: >- + Retrieve the current status of a previously submitted + asyncquery given its job_id + - name: translator + description: Required for SmartAPI validation of x-translator + - name: trapi + description: Required for SmartAPI validation of x-trapi +paths: + /meta_knowledge_graph: + get: + tags: + - meta_knowledge_graph + summary: Meta knowledge graph representation of this TRAPI web service. + responses: + '200': + description: >- + Returns meta knowledge graph representation of this TRAPI web + service. + content: + application/json: + schema: + $ref: '#/components/schemas/MetaKnowledgeGraph' + /query: + post: + tags: + - query + summary: Initiate a query and wait to receive a Response + description: '' + requestBody: + description: Query information to be submitted + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/Query' + responses: + '200': + description: >- + OK. There may or may not be results. Note that some of the provided + identifiers may not have been recognized. + content: + application/json: + schema: + $ref: '#/components/schemas/Response' + '400': + description: >- + Bad request. The request is invalid according to this OpenAPI + schema OR a specific identifier is believed to be invalid somehow + (not just unrecognized). + content: + application/json: + schema: + type: string + '409': + description: >- + There is a conflict between client-given TRAPI parameters and server + capabilities. The response body may contain additional details about + the conflict. + content: + application/json: + schema: + type: string + '413': + description: >- + Payload too large. Indicates that batch size was over the limit + specified in x-trapi. + content: + application/json: + schema: + type: string + '429': + description: >- + Too many requests. Indicates that the client issued requests that + exceed the rate limit specified in x-trapi. + content: + application/json: + schema: + type: string + '500': + description: >- + Internal server error. + content: + application/json: + schema: + type: string + '501': + description: >- + Not implemented. + content: + application/json: + schema: + type: string + /asyncquery: + post: + tags: + - asyncquery + summary: Initiate a query with a callback to receive the response + description: '' + requestBody: + description: Query information to be submitted + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/AsyncQuery' + responses: + '200': + description: >- + The query is accepted for processing and the Response will be + sent to the callback url when complete. + content: + application/json: + schema: + $ref: '#/components/schemas/AsyncQueryResponse' + '400': + description: >- + Bad request. The request is invalid according to this OpenAPI + schema OR a specific identifier is believed to be invalid somehow + (not just unrecognized). + content: + application/json: + schema: + type: string + '409': + description: >- + There is a conflict between client-given TRAPI parameters and server + capabilities. The response body may contain additional details about + the conflict. + content: + application/json: + schema: + type: string + '413': + description: >- + Payload too large. Indicates that batch size was over the limit + specified in x-trapi. + content: + application/json: + schema: + type: string + '429': + description: >- + Too many requests. Indicates that the client issued requests that + exceed the rate limit specified in x-trapi. + content: + application/json: + schema: + type: string + '500': + description: >- + Internal server error. + content: + application/json: + schema: + type: string + '501': + description: >- + Not implemented. + content: + application/json: + schema: + type: string + /asyncquery_status/{job_id}: + get: + tags: + - asyncquery_status + summary: >- + Retrieve the current status of a previously submitted + asyncquery given its job_id + operationId: asyncquery_status + parameters: + - in: path + name: job_id + description: Identifier of the job for status request + required: true + schema: + type: string + examples: + - rXEOAosN3L + responses: + '200': + description: >- + Returns the status and current logs of a previously + submitted asyncquery. + content: + application/json: + schema: + $ref: '#/components/schemas/AsyncQueryStatusResponse' + '404': + description: job_id not found + '501': + description: >- + Return code 501 indicates that this endpoint has not been + implemented at this site. Sites that implement /asyncquery + MUST implement /asyncquery_status/{job_id}, but those that + do not implement /asyncquery SHOULD NOT implement + /asyncquery_status. + content: + application/json: + schema: + type: string +components: + schemas: + Query: + description: >- + The Query class is used to package a user request for information. A + Query object consists of a required Message object with optional + additional properties. Additional properties are intended to convey + implementation-specific or query-independent parameters. For example, + an additional property specifying a log level could allow a user to + override the default log level in order to receive more fine-grained + log information when debugging an issue. + x-body-name: request_body + type: object + properties: + submitter: + type: string + description: >- + Any string for self-identifying the submitter of a query. The + purpose of this optional property is to aid in the tracking of + the source of queries for development and issue resolution. + parameters: + $ref: '#/components/schemas/QueryParameters' + description: >- + Query-time parameters that don't affect the semantics + of the query or intended workflow, but may affect overall + behavior of the server in the execution of this query. + The server MUST repeat the parameters it is given in its Response. + message: + $ref: '#/components/schemas/Message' + description: >- + The query Message is a serialization of the user request. Content + of the Message object depends on the intended TRAPI operation. For + example, the fill operation requires a non-empty query_graph property + as part of the Message, whereas other operations, e.g. overlay, + require non-empty results and knowledge_graph properties. + workflow: + $ref: https://standards.ncats.io/workflow/1.3.5/schema + description: List of workflow steps to be executed. + additionalProperties: true + required: + - message + AsyncQuery: + description: >- + The AsyncQuery class is effectively the same as the Query class but + it requires a callback property. + allOf: + - $ref: '#/components/schemas/Query' + - type: object + properties: + callback: + type: string + format: uri + pattern: ^https?:// + description: >- + Upon completion, this server will send a POST request to the + callback URL with `Content-Type: application/json` header and + request body containing a JSON-encoded `Response` object. + The server MAY POST `Response` objects before work is fully + complete to provide interim results with a Response.status + value of 'Running'. If a POST operation to the callback URL + does not succeed, the server SHOULD retry the POST at least + once. + additionalProperties: true + required: + - callback + QueryParameters: + type: object + description: >- + Query-time parameters that don't affect the semantics + of a query or intended workflow, but may affect overall + behavior of the server in the execution of this query. + The server MUST repeat the parameters it is given in its Response. + properties: + timeout: + type: number + description: >- + Custom time in seconds that the client is willing to wait for a response. + After this time has elapsed, the service MAY consider the query + failed and respond with logs indicating as such. + If the service knows it cannot respond in the given time, it MAY + respond with an HTTP 409 and a response explaining its time capabilities. + Negative values SHOULD be interpreted as disabling any default + timeout the server implements. + log_level: + $ref: '#/components/schemas/LogLevel' + description: The least critical level of logs to return. + bypass_cache: + type: boolean + default: false + description: >- + Set to true in order to request that the agent obtain + fresh information from its sources in all cases where + it has a viable choice between requesting fresh information + in real time and using cached information. The agent + receiving this flag MUST also include it in TRAPI sent to + downstream sources (e.g., ARS -> ARAs -> KPs). + additionalProperties: true + AsyncQueryResponse: + type: object + description: >- + The AsyncQueryResponse object contains a payload that must be + returned from a submitted async_query. + properties: + status: + description: >- + One of a standardized set of short codes: + e.g. Accepted, QueryNotTraversable, KPsNotAvailable + type: string + examples: + - Accepted + description: + description: >- + A brief human-readable description of the result of the + async_query submission. + type: string + examples: + - Async_query has been queued + job_id: + description: >- + An identifier for the submitted job that can be used with + /async_query_status to receive an update on the status of + the job. + type: string + examples: + - rXEOAosN3L + additionalProperties: true + required: + - job_id + AsyncQueryStatusResponse: + type: object + description: >- + The AsyncQueryStatusResponse object contains a payload that describes + the current status of a previously submitted async_query. + properties: + status: + description: >- + One of a standardized set of short codes: + Queued, Running, Completed, Failed + type: string + examples: + - Running + description: + description: >- + A brief human-readable description of the current state + or summary of the problem if the status is Failed. + type: string + examples: + - Callback URL returned 500 + logs: + description: >- + A list of LogEntry items, containing errors, warnings, debugging + information, etc. List items MUST be in chronological order with + earliest first. The most recent entry should be last. Its timestamp + will be compared against the current time to see if there is + still activity. + type: array + items: + $ref: '#/components/schemas/LogEntry' + minItems: 1 + response_url: + description: >- + Optional URL that can be queried to restrieve the full TRAPI + Response. + type: string + examples: + - https://arax.ncats.io/api/arax/v2.0/response/116481 + additionalProperties: true + required: + - status + - description + - logs + Response: + type: object + description: >- + The Response object contains the main payload when a TRAPI query + endpoint interprets and responds to the submitted query successfully + (i.e., HTTP Status Code 200). The message property contains the + knowledge of the response (query graph, knowledge graph, and results). + The status, description, and logs properties provide additional details + about the response. + properties: + parameters: + $ref: '#/components/schemas/QueryParameters' + description: >- + Query-time parameters that the service received in the original query. + The server MUST repeat the parameters it is given in its Response. + message: + $ref: '#/components/schemas/Message' + description: >- + Contains the knowledge of the response (query graph, knowledge + graph, and results). + status: + description: >- + One of a standardized set of short codes, + e.g. Success, QueryNotTraversable, KPsNotAvailable + type: string + examples: + - Success + description: + description: A brief human-readable description of the outcome + type: string + examples: + - Success. 42 results found. + logs: + description: >- + A list of LogEntry items, containing errors, warnings, debugging + information, etc. List items MUST be in chronological order with + earliest first. + type: array + items: + $ref: '#/components/schemas/LogEntry' + minItems: 1 + workflow: + $ref: https://standards.ncats.io/workflow/1.3.5/schema + description: List of workflow steps that were executed. + schema_version: + type: string + examples: + - 2.0.0 + description: Version label of the TRAPI schema used in this document + biolink_version: + type: string + examples: + - 4.4.0 + description: Version label of the Biolink model used in this document + data_release_versions: + description: Versions of data sources used in this document. + type: object + minProperties: 1 + additionalProperties: + type: string + examples: + - translator_kg: 2026_06_21 + additionalProperties: true + required: + - message + Message: + description: >- + The message object holds the main content of a Query or a Response in + three properties: query_graph, results, and knowledge_graph. + The query_graph property contains the query configuration, the results + property contains any answers that are returned by the service, + and knowledge_graph property contains lists of edges and nodes in the + thought graph corresponding to this message. The content of these + properties is context-dependent to the encompassing object and + the TRAPI operation requested. + type: object + properties: + results: + description: >- + List of all returned Result objects for the query posed. + The list SHOULD NOT be assumed to be ordered. The 'score' property, + if present, MAY be used to infer result rankings. If Results are + not expected (such as for a query Message), this property SHOULD + be absent. If Results are expected (such as for a response + Message) and no Results are available, this property SHOULD be an + array with 0 Results in it. + type: array + items: + $ref: '#/components/schemas/Result' + query_graph: + description: >- + QueryGraph object that contains a serialization of a query in the + form of a graph + $ref: '#/components/schemas/QueryGraph' + knowledge_graph: + $ref: '#/components/schemas/KnowledgeGraph' + description: >- + KnowledgeGraph object that contains lists of nodes and edges + in the thought graph corresponding to the message + auxiliary_graphs: + type: object + description: >- + Dictionary of AuxiliaryGraph instances that are used by Knowledge + Graph Edges and Result Analyses. These are referenced elsewhere by + the dictionary key. + additionalProperties: + $ref: '#/components/schemas/AuxiliaryGraph' + minProperties: 1 + additionalProperties: false + LogEntry: + description: >- + The LogEntry object contains information useful for tracing + and debugging across Translator components. Although an + individual component (for example, an ARA or KP) may have its + own logging and debugging infrastructure, this internal + information is not, in general, available to other components. + In addition to a timestamp and logging level, LogEntry + includes a string intended to be read by a human, along with + one of a standardized set of codes describing the condition of + the component sending the message. + type: object + properties: + timestamp: + type: string + format: date-time + description: >- + Timestamp in ISO 8601 format, providing the LogEntry time + either in univeral coordinated time (UTC) using the 'Z' tag + (e.g 2020-09-03T18:13:49Z), or, if local time is provided, + the timezone offset must be provided + (e.g. 2020-09-03T18:13:49-04:00). + examples: + - '2020-09-03T18:13:49+00:00' + level: + $ref: '#/components/schemas/LogLevel' + code: + type: string + description: >- + One of a standardized set of short codes + e.g. QueryNotTraversable, KPNotAvailable, KPResponseMalformed + message: + type: string + description: A human-readable log message + additionalProperties: true + required: + - timestamp + - message + LogLevel: + type: string + description: Logging level + enum: + - ERROR + - WARNING + - INFO + - DEBUG + Result: + type: object + description: >- + A Result object specifies the nodes and edges in the knowledge graph + that satisfy the structure or conditions of a user-submitted query + graph. It must contain a NodeBindings object (list of query graph node + to knowledge graph node mappings) and a list of Analysis objects. + properties: + node_bindings: + type: object + description: >- + The dictionary of input QNodes to KnowledgeGraph Node + bindings where the dictionary keys are the key identifiers of the + QNodes and the associated values of those keys are + instances of NodeBinding schema type (see below). Because a given + QNode may have multiple KnowledgeGraph Nodes bound in the result, + the NodeBinding object may list multiple KnowledgeGraph Nodes. + additionalProperties: + $ref: '#/components/schemas/NodeBinding' + minProperties: 1 + analyses: + type: array + description: >- + The list of all Analysis components that contribute to the result. + See below for Analysis components. + items: + $ref: '#/components/schemas/Analysis' + minItems: 1 + additionalProperties: true + required: + - node_bindings + NodeBinding: + type: object + description: >- + A NodeBinding object defines all relevant KnowledgeGraph Node mappings, + identified by the corresponding object key identifier(s) of the + Node(s) within the Knowledge Graph. Instances of NodeBinding may + include extra annotation in the form of additional properties. + (such annotation is not yet fully standardized). Each Node + Binding must bind directly to node in the original Query Graph. + properties: + ids: + type: array + items: + $ref: '#/components/schemas/CURIE' + minItems: 1 + description: >- + The CURIEs of one or more Nodes within the Knowledge Graph. + additionalProperties: true + required: + - ids + Analysis: + type: object + description: >- + An Analysis is a dictionary that contains information about + the result tied to a particular service. Each Analysis is + generated by a single reasoning service, and describes the + outputs of analyses performed by the reasoner on a particular + Result (e.g. a result score), along with provenance information + supporting the analysis (e.g. method or data that supported + generation of the score). + properties: + resource_id: + $ref: '#/components/schemas/CURIE' + description: >- + The infores id of the resource generating this Analysis + edge_bindings: + type: object + minProperties: 1 + description: >- + The dictionary of input Query Graph to Knowledge Graph edge + bindings where the dictionary keys are the key identifiers of + the Query Graph edges and the associated values of those keys + are instances of EdgeBinding schema type (see below). Because + a given QEdge may have multiple knowledge Edges bound in the + result, the EdgeBinding object may list multiple knowledge Edges. + additionalProperties: + $ref: '#/components/schemas/EdgeBinding' + path_bindings: + type: object + minProperties: 1 + description: >- + The dictionary of input QueryGraph paths to Analysis paths, + specifically used only for pathfinder queries. + additionalProperties: + $ref: '#/components/schemas/PathBinding' + score: + type: number + format: float + examples: + - 163.233 + description: >- + A numerical score associated with this result indicating the + relevance or confidence of this result relative to others in the + returned set. Higher MUST be better. + support_graphs: + type: array + description: >- + This is a list of references to AuxiliaryGraph instances + that supported the analysis of a Result as performed by the + reasoning service. Each item in the list is the key of a + single AuxiliaryGraph. + minItems: 1 + items: + type: string + scoring_method: + type: string + description: >- + An identifier and link to an explanation for the method used + to generate the score. + attributes: + type: array + description: >- + The attributes of this particular Analysis. + items: + $ref: '#/components/schemas/Attribute' + additionalProperties: true + required: + - resource_id + anyOf: + - required: [edge_bindings] + - required: [path_bindings] + EdgeBinding: + type: object + description: >- + An EdgeBinding object defines all relevant KnowledgeGraph Edge mappings, + identified by the corresponding 'id' object key identifier of the + Edge within the knowledge graph. Instances of EdgeBinding may include + extra annotation (such annotation is not yet fully standardized). + EdgeBindings are captured within a specific reasoner's Analysis + object because the Edges in the KnowledgeGraph that get bound to + the input QueryGraph may differ between reasoners. + properties: + ids: + type: array + items: + type: string + minItems: 1 + description: The key identifiers of specific KnowledgeGraph Edges. + additionalProperties: true + required: + - ids + PathBinding: + type: object + description: >- + A PathBinding object binds a single QueryGraph path (the key to + this object) to one or more relevant AuxiliaryGraph ids containing + a list of edges in the path. The AuxiliaryGraph does not convey any + order of edges in the path. + properties: + ids: + type: array + items: + type: string + minItems: 1 + description: The key identifiers of specific auxiliary graphs. + additionalProperties: true + required: + - ids + AuxiliaryGraph: + type: object + description: >- + A single AuxiliaryGraph instance that is used by KnowledgeGraph + Edges, Result Analysis support graphs, and PathBindings. + Edges comprising an AuxiliaryGraph are a subset of the + KnowledgeGraph in the message. Data creators can + create an AuxiliaryGraph to assemble a specific collection + of edges from the KnowledgeGraph into a named graph that can be + referenced from an Edge as evidence/explanation supporting that Edge, + from a Result Analysis as information used to generate a score, or + from a PathBinding as the path for that Analysis. + properties: + edges: + type: array + description: >- + List of edges that form the AuxiliaryGraph. Each item is a + reference to a single KnowledgeGraph Edge. This list is not + ordered, nor is the order intended to convey any relationship + between the edges that form this AuxiliaryGraph. + items: + type: string + minItems: 1 + additionalProperties: true + required: + - edges + KnowledgeGraph: + type: object + description: >- + The knowledge graph associated with a set of results. The instances + of Node and Edge defining this graph represent instances of + biolink:NamedThing (concept nodes) and biolink:Association + (relationship edges) representing (Attribute) annotated knowledge + returned from the knowledge sources and inference agents wrapped by + the given TRAPI implementation. + properties: + nodes: + type: object + description: >- + Dictionary of Node instances used in the KnowledgeGraph, + referenced elsewhere in the TRAPI output by the dictionary key. + additionalProperties: + $ref: '#/components/schemas/Node' + edges: + type: object + description: >- + Dictionary of Edge instances used in the KnowledgeGraph, + referenced elsewhere in the TRAPI output by the dictionary key. + additionalProperties: + $ref: '#/components/schemas/Edge' + additionalProperties: true + required: + - nodes + QueryGraph: + type: object + description: >- + A graph representing a biomedical question. It serves as a template for + each Result (answer), where each bound knowledge graph node/edge is + expected to obey the constraints of the associated QueryGraph element. + properties: + nodes: + type: object + description: >- + The node specifications. The keys of this map are unique node + identifiers and the corresponding values include the constraints + on bound nodes. + additionalProperties: + $ref: '#/components/schemas/QNode' + minProperties: 1 + edges: + type: object + description: >- + The edge specifications. The keys of this map are unique edge + identifiers and the corresponding values include the constraints + on bound edges, in addition to specifying the subject and object + QNodes. + additionalProperties: + $ref: '#/components/schemas/QEdge' + minProperties: 1 + paths: + type: object + description: >- + The QueryGraph path specification, used only for pathfinder type + queries. The keys of this map are unique path identifiers and + the corresponding values include the constraints on bound paths, + in addition to specifying the subject, object, and intermediate + QNodes. + additionalProperties: + $ref: '#/components/schemas/QPath' + minProperties: 1 + maxProperties: 1 + additionalProperties: true + required: + - nodes + QNode: + type: object + description: >- + A node in the QueryGraph used to represent an entity in a + query. If no CURIEs are not specified, any nodes matching the category + of the QNode will be returned in the Results. + properties: + ids: + type: array + items: + $ref: '#/components/schemas/CURIE' + minItems: 1 + examples: + - ['OMIM:603903'] + description: >- + A list of one or more CURIE identifiers for this node. + The 'ids' property will hold a list of CURIEs only in the case of + a BATCH set_interpretation, where each CURIE is queried + separately. If a list of queried CURIEs is to be considered as a + set (as under a MANY or ALL set_interpretation), the 'ids' property + will hold a single id representing this set, and the individual members + of this set will be captured in a separate 'member_ids' property. + Note that the set id MUST be created as a UUID by the system that + defines the queried set, using a centralized nodenorm service. + Note also that downstream systems MUST re-use the original set UUID + in the messages that they create/send to facilitate merging or + caching operations. + categories: + type: array + description: >- + Biolink Model categories, which are allowed + to be of type 'abstract' or 'mixin' (only in QGraphs!). + Use of deprecated categories SHOULD be avoided. + items: + $ref: '#/components/schemas/BiolinkEntity' + minItems: 1 + set_interpretation: + type: string + description: >- + Indicates how multiple CURIEs in the ids property MUST be + interpreted. BATCH indicates that the query is intended to be + a batch query and each CURIE is treated independently. ALL means + that all specified CURIES MUST appear in each Result. + MANY means that member CURIEs MUST form one or more + sets in the Results, and sets with more members are generally + considered more desirable that sets with fewer members. + Only when there are no ids provided, set_interpretation MAY be + set to COLLATE to indicate that multiple matching nodes MUST be + collated into a single Result, rather than separated into + separate Results. If this property is absent, the default is + BATCH. + enum: + - BATCH + - ALL + - MANY + - COLLATE + member_ids: + type: array + description: >- + A list of CURIE identifiers for members of a queried set. This + property MUST be populated under a set_interpretation of MANY + or ALL, when the 'ids' property holds a UUID representing the set + itself. This property MUST NOT be used under a set_interpretation + of BATCH or COLLATE or when set_interpretation is absent. + minItems: 1 + items: + $ref: '#/components/schemas/CURIE' + constraints: + type: array + description: >- + A list of constraints applied to a query node. + If there are multiple items, they must all be true (equivalent + to AND) + items: + $ref: '#/components/schemas/AttributeConstraint' + minItems: 1 + additionalProperties: true + QEdge: + type: object + description: >- + An edge in the QueryGraph used as a filter pattern specification in a + query. If the optional predicate property is not specified, + it is assumed to be a wildcard match to the target knowledge space. + If specified, the ontological inheritance hierarchy associated with + the term provided is assumed, such that edge bindings returned may be + an exact match to the given QEdge predicate term, + or to a term that is a descendant of the QEdge predicate term. + properties: + knowledge_type: + description: >- + Indicates the type of knowledge that the client wants from the + server between the subject and object. If the value is + 'lookup', then the client wants direct lookup information from + knowledge sources. If the value is 'inferred', then the client + wants the server to get creative and connect the subject and + object in more speculative and non-direct-lookup ways. If this + property is absent, it MUST be assumed to mean + 'lookup'. + examples: + - lookup + type: string + predicates: + type: array + description: >- + These should be Biolink Model predicates and are allowed to be of + type 'abstract' or 'mixin' (only in QGraphs!). Use of 'deprecated' + predicates SHOULD be avoided. + items: + $ref: '#/components/schemas/BiolinkPredicate' + minItems: 1 + subject: + type: string + examples: + - https://omim.org/entry/603903 + description: >- + Corresponds to the map key identifier of the + subject concept node anchoring the query filter + pattern for the query relationship edge. + object: + type: string + examples: + - https://www.uniprot.org/uniprot/P00738 + description: >- + Corresponds to the map key identifier of the + object concept node anchoring the query filter + pattern for the query relationship edge. + constraints: + $ref: '#/components/schemas/QEdgeConstraints' + description: >- + An object containing all constraints placed on the QEdge. + ALL edges bound to this QEdge MUST conform to ALL given constraints; + underlying edges (such as those appearing in supporting graphs) + are not required to conform to the given constraints. + additionalProperties: true + required: + - subject + - object + QEdgeConstraints: + type: object + description: >- + A subschema for constraints that may be placed on a given QEdge. + ALL edges bound to the given QEdge MUST conform to ALL given constraints; + underlying edges (such as those appearing in supporting graphs) + are not required to conform to the given constraints. + minProperties: 1 + additionalProperties: true + properties: + knowledge_level: + $ref: '#/components/schemas/AllowDenyConstraint' + description: >- + A constraint defining knowledge_level values which are either allowed + or denied on bound edges. + Provided string(s) MUST be a valid biolink knowledge_level value. + (See https://biolink.github.io/biolink-model/KnowledgeLevelEnum/) + agent_type: + $ref: '#/components/schemas/AllowDenyConstraint' + description: >- + A constraint defining agent_type values which are either allowed + or denied on bound edges. + Provided string(s) MUST be a valid biolink agent_type value. + (See https://biolink.github.io/biolink-model/AgentTypeEnum/) + attributes: + type: array + description: >- + A list of attribute constraints applied to a query edge. + If there are multiple items, they must all be true (equivalent + to AND) + minItems: 1 + items: + $ref: '#/components/schemas/AttributeConstraint' + qualifiers: + type: array + description: >- + A list of QualifierSetConstraints applied to a QEdge. + If multiple QualifierSetConstraints are provided, there is an OR + relationship between them. If the QEdge has multiple + predicates or if the QNodes that correspond to the subject or + object of this QEdge have multiple categories or multiple + curies, then constraints.qualifiers MUST NOT be specified + because these complex use cases are not supported at this time. + minItems: 1 + items: + $ref: '#/components/schemas/QualifierSetConstraint' + sources: + description: >- + A list of infores CURIEs which are either allowed or denied in the + sources (resource_id) of the bound Edge. + If `behavior` is set to "ALLOW", ANY (at least 1) of the given infores + CURIEs MUST be present. + If `behavior` is set to "DENY", then ALL given infores CURIEs MUST + NOT be present. + allOf: + - $ref: '#/components/schemas/AllowDenyConstraint' + - type: object + properties: + values: + type: array + description: >- + These SHOULD be infores CURIEs, so this is a subschema that + is more stringent than the one in AllowDenyConstraint. + minItems: 1 + items: + $ref: '#/components/schemas/CURIE' + primary_only: + type: boolean + description: >- + When set to `false` (the default), the ALLOW/DENY constraint + of `values` applies to ALL RetrievalSources in + the sources of the bound Edge. + When set to `true`, the constraint ONLY applies to the + RetrievalSource with the resource_role primary_knowledge_source. + default: false + AllowDenyConstraint: + type: object + description: >- + A list of values which are to either be allowed or denied. + If `behavior` is set to "ALLOW", then ANY (at least 1) of the given values + MUST appear in the constrained property in order for it to meet the + constraint (OR relationship). + If `behavior` is set to "DENY", then ALL of the given values MUST NOT + appear in the constrained property in order for it to meet the constraint + (NOT (x OR y) relationship). + properties: + behavior: + type: string + enum: + - ALLOW + - DENY + values: + type: array + minItems: 1 + items: + type: string + required: + - behavior + - values + QPath: + type: object + description: >- + A path in the QueryGraph used for pathfinder queries. Both subject and + object MUST reference QNodes that have a CURIE in their ids property. + Paths returned that bind to this QPath MUST represent some + relationship between the subject and object. + properties: + subject: + type: string + description: >- + Corresponds to the map key identifier of the subject concept node + for the start of the queried path. + examples: + - n0 + object: + type: string + description: >- + Corresponds to the map key identifier of the object concept node + for the end of the queried path. + examples: + - n1 + predicates: + type: array + description: >- + QPath predicates are intended to convey what type of paths are desired, + NOT a constraint on the types of predicates that may be in result + paths. + If no predicate is listed, the ARA SHOULD find paths such that the + relationship represented by the path is a "related_to" relationship. + These should be Biolink Model predicates and are allowed to be of + type 'abstract' or 'mixin'. Use of 'deprecated' + predicates SHOULD be avoided. + items: + $ref: '#/components/schemas/BiolinkPredicate' + minItems: 1 + constraints: + type: array + description: >- + A list of constraints for the QPath. If multiple constraints are + listed, it should be interpreted as an OR relationship. Each path + returned MUST comply with at least one constraint. + items: + $ref: '#/components/schemas/PathConstraint' + minItems: 1 + additionalProperties: true + required: + - object + - subject + PathConstraint: + type: object + description: >- + A constraint for paths. ARAs must comply with constraints when finding + paths. + properties: + required_intermediate_categories: + type: array + description: >- + A list of Biolink Model categories by which to constrain paths returned. + If multiple categories are listed, it should be interpreted as an + AND relationship. Each path returned by ARAs MUST contain at least + one node of each category listed. + items: + $ref: '#/components/schemas/BiolinkEntity' + minItems: 1 + additionalProperties: true + Node: + type: object + description: >- + A node in the KnowledgeGraph which represents some biomedical + concept. Nodes are identified by the keys in the KnowledgeGraph + Node mapping. + properties: + name: + type: string + examples: + - Haptoglobin + description: Formal name of the entity + categories: + type: array + description: >- + These should be Biolink Model categories and are NOT allowed + to be of type 'abstract' or 'mixin'. Returning 'deprecated' + categories SHOULD also be avoided. + items: + $ref: '#/components/schemas/BiolinkEntity' + minItems: 1 + attributes: + type: array + description: A list of attributes describing the node + items: + $ref: '#/components/schemas/Attribute' + is_set: + type: boolean + description: >- + Indicates that the node represents a set of entities. + If this property is absent, it is assumed to be + false. + required: + - categories + additionalProperties: false + Attribute: + type: object + description: >- + Generic attribute for a node or an edge that expands the key-value + pair concept by including properties for additional metadata. These properties + MAY be used to describe the source of the statement made in a key-value + pair of the attribute object, or describe the attribute's value itself + including its semantic type, or a url providing additional information + about it. An attribute may be further qualified with sub-attributes + (for example to provide confidence intervals on a value). + properties: + attribute_type_id: + $ref: '#/components/schemas/CURIE' + description: >- + The main 'key' of the attribute object, holding a CURIE of an ontology + property defining the specific attribute (preferably the CURIE of + a Biolink association slot). This property captures the relationship + asserted to hold between the value of the attribute, and the node + or edge from which it hangs. For example, that a value of + '0.000153' represents a p-value supporting an edge. + examples: + - biolink:synonym + original_attribute_name: + type: string + description: >- + The term used by the original source of an attribute to describe + the meaning or significance of the value it captures. This may be + a column name in a source tsv file, or a key in a source json + document for the field in the data that held the attribute's + value. Capturing this information where possible preserves + original source meaning. Note that the data type is string + but the contents of the property MAY be a CURIE of a third + party ontology term. + examples: + - p-value + value: + description: >- + Value of the attribute. May be any data type, including a list. + examples: + - 0.000153 + value_type_id: + $ref: '#/components/schemas/CURIE' + description: >- + CURIE describing the semantic type of an attribute's value. Use + a Biolink class if possible, otherwise a term from an external + ontology. If a suitable CURIE/identifier does not exist, enter a + descriptive phrase here and submit the new type for consideration + by the appropriate authority. + examples: + - EDAM:data_1187 + attribute_source: + type: string + description: >- + The source of the core assertion made by the key-value pair of an + attribute object. Use a CURIE or namespace designator for this + resource where possible. + examples: + - UniProtKB + value_url: + type: string + description: >- + Human-consumable URL linking to a web document that provides + additional information about this attribute's value (not the node + or the edge from which it hangs). + examples: + - https://pubmed.ncbi.nlm.nih.gov/32529952 + description: + type: string + description: >- + Human-readable description for the attribute and its value. + examples: + - Assertion Authored By Dr. Trans L. Ator + attributes: + type: array + description: >- + A list of attributes providing further information about the + parent attribute (for example to provide provenance + information about the parent attribute). + items: + $ref: '#/components/schemas/Attribute' + required: + - attribute_type_id + - value + additionalProperties: false + Edge: + type: object + description: >- + A specification of the semantic relationship linking two concepts + that are expressed as nodes in the knowledge graph + resulting from a query to the service. + properties: + predicate: + $ref: '#/components/schemas/BiolinkPredicate' + description: >- + The type of relationship between the subject and object + for the statement expressed in an Edge. The predicate SHOULD + be from the Biolink Model and MUST NOT be + of type 'abstract' or 'mixin'. Returning 'deprecated' + predicate terms SHOULD be avoided. + examples: + - biolink:gene_associated_with_condition + subject: + $ref: '#/components/schemas/CURIE' + description: >- + Corresponds to the map key CURIE of the + subject concept node of this relationship edge. + examples: + - MONDO:0011382 + object: + $ref: '#/components/schemas/CURIE' + description: >- + Corresponds to the map key CURIE of the + object concept node of this relationship edge. + examples: + - UniProtKB:P00738 + attributes: + description: A list of additional attributes for this edge + items: + $ref: '#/components/schemas/Attribute' + type: array + qualifiers: + description: >- + A set of Qualifiers that act together to add nuance + or detail to the statement expressed in an Edge. + items: + $ref: '#/components/schemas/Qualifier' + minItems: 1 + type: array + sources: + type: array + description: >- + A list of RetrievalSource objects that provide information + about how a particular information resource served + as a source from which the knowledge expressed in an Edge, + or data used to generate this knowledge, was retrieved. + items: + $ref: '#/components/schemas/RetrievalSource' + minItems: 1 + knowledge_level: + type: string + description: >- + One of the biolink-enumerated permissible values for `knowledge level` + that provides the level of knowledge the Edge represents. + (See https://biolink.github.io/biolink-model/KnowledgeLevelEnum/) + examples: + - knowledge_assertion + - logical_entailment + - prediction + agent_type: + type: string + description: >- + One of the biolink-enumerated permissible values for `agent type` + that provides the kind of agent that originated the knowledge + presented by the Edge. + (See https://biolink.github.io/biolink-model/AgentTypeEnum/) + examples: + - manual_agent + - automated_agent + - text_mining_agent + additionalProperties: false + required: + - object + - predicate + - subject + - sources + - knowledge_level + - agent_type + Qualifier: + additionalProperties: false + description: >- + An additional nuance attached to an assertion + type: object + properties: + qualifier_type_id: + $ref: '#/components/schemas/CURIE' + description: >- + CURIE for a Biolink 'qualifier' association slot, generally taken + from Biolink association slots designated for this purpose + e.g. biolink:subject_aspect_qualifier, + biolink:subject_direction_qualifier, + biolink:object_aspect_qualifier, etc. Such qualifiers are used + to elaborate a second layer of meaning of a knowledge graph edge. + Available qualifiers can be found at + https://biolink.github.io/biolink-model/qualifiers.html, + which mostly have slot names with the suffix string 'qualifier'. + pattern: ^biolink:[a-z][a-z_]*$ + examples: + - biolink:subject_aspect_qualifier + qualifier_value: + type: string + description: >- + The value associated with the type of the qualifier, drawn from + a set of controlled values by the type as specified in + the Biolink model (e.g. 'expression' or 'abundance' for the + qualifier type 'biolink:subject_aspect_qualifier', etc). + The enumeration of qualifier values for a given qualifier + type is generally constrained by the category + of the edge (i.e. biolink:Association subtype). + examples: + - expression + required: + - qualifier_type_id + - qualifier_value + QualifierSetConstraint: + type: object + description: >- + A constraint on the qualifiers of a bound Edge (types and values). + A given key-value pair defines the required qualifier_type_id + and qualifier_value of one Qualifier, respectively. + For example, a QualifierSetConstraint can constrain a + "ChemicalX - affects - ?Gene" query to return only edges where + ChemicalX specifically affects the 'expression' of the Gene, by + constraining on the qualifier_type "biolink:object_aspect_qualifier" + with a qualifier_value of "expression". + Multiple type-value pairs have an AND relationship. + minProperties: 1 + patternProperties: + '^biolink:': + type: string + BiolinkEntity: + description: >- + Compact URI (CURIE) for a Biolink class, biolink:NamedThing + or a child thereof. The CURIE must use the prefix 'biolink:' + followed by the PascalCase class name. + type: string + pattern: ^biolink:[A-Z][a-zA-Z]*$ + externalDocs: + description: Biolink model entities + url: https://biolink.github.io/biolink-model/docs/NamedThing.html + examples: + - biolink:PhenotypicFeature + BiolinkPredicate: + description: >- + CURIE for a Biolink 'predicate' slot, taken from the Biolink slot + ('is_a') hierarchy rooted in biolink:related_to (snake_case). This + predicate defines the Biolink relationship between the subject and + object nodes of a biolink:Association defining a knowledge graph edge. + type: string + pattern: ^biolink:[a-z][a-z_]*$ + externalDocs: + description: Biolink model predicates + url: https://biolink.github.io/biolink-model/docs/related_to.html + examples: + - biolink:interacts_with + CURIE: + type: string + description: >- + A Compact URI, consisting of a prefix and a reference separated + by a colon, such as UniProtKB:P00738. Via an external context + definition, the CURIE prefix and colon may be replaced by a URI + prefix, such as https://identifiers.org/uniprot/, to form a full + URI. + externalDocs: + url: https://www.w3.org/TR/2010/NOTE-curie-20101216/ + MetaKnowledgeGraph: + type: object + description: >- + Knowledge-map representation of this TRAPI web service. The meta + knowledge graph is composed of the union of most specific categories + and predicates for each node and edge. + properties: + nodes: + type: object + minProperties: 1 + description: >- + Collection of the most specific node categories provided by + this TRAPI web service, indexed by Biolink class CURIEs. + A node category is only exposed here if there is + node for which that is the most specific category available. + additionalProperties: + $ref: '#/components/schemas/MetaNode' + edges: + type: array + description: >- + List of the most specific edges/predicates provided by this TRAPI + web service. A predicate is only exposed here if there is an edge + for which the predicate is the most specific available. + items: + $ref: '#/components/schemas/MetaEdge' + required: + - nodes + - edges + MetaNode: + type: object + description: >- + Description of a node category provided by this TRAPI web service. + properties: + id_prefixes: + type: array + description: >- + List of CURIE prefixes for the node category that this TRAPI web + service understands and accepts on the input. + items: + type: string + minItems: 1 + examples: + - [CHEMBL.COMPOUND, INCHIKEY] + attributes: + type: array + description: >- + Node attributes provided by this TRAPI web service. + items: + $ref: '#/components/schemas/MetaAttribute' + required: + - id_prefixes + additionalProperties: false + MetaEdge: + type: object + description: >- + Edge in a meta knowledge map describing relationship between a subject + Biolink class and an object Biolink class. + properties: + subject: + $ref: '#/components/schemas/BiolinkEntity' + description: >- + Subject node category of this relationship edge. + examples: + - biolink:ChemicalEntity + predicate: + $ref: '#/components/schemas/BiolinkPredicate' + description: >- + Biolink relationship between the subject and object categories. + examples: + - biolink:affects + object: + $ref: '#/components/schemas/BiolinkEntity' + description: >- + Object node category of this relationship edge. + examples: + - biolink:Protein + knowledge_types: + description: >- + A list of knowledge_types that are supported by the service. + If this property is absent, this means that only 'lookup' + is supported. Currently allowed values are 'lookup' or 'inferred'. + items: + type: string + minItems: 1 + type: array + attributes: + type: array + description: >- + Edge attributes provided by this TRAPI web service. + items: + $ref: '#/components/schemas/MetaAttribute' + qualifiers: + description: >- + Qualifiers that are possible to be found on this edge type. + items: + $ref: '#/components/schemas/MetaQualifier' + minItems: 1 + type: array + association: + description: >- + The Biolink association type (entity) that this edge represents. + Associations are classes in Biolink + that represent a relationship between two entities. + For example, the association 'gene interacts with gene' + is represented by the Biolink class, + 'biolink:GeneToGeneAssociation'. If association + is filled out, then the testing harness can + help validate that the qualifiers are being used + correctly. + examples: + - biolink:ChemicalToGeneAssociation + $ref: '#/components/schemas/BiolinkEntity' + knowledge_levels: + description: >- + The knowledge levels contributing to this meta edge. + If provided, this property SHOULD contain all possible + knowledge_levels relevant edges might return. + (See https://biolink.github.io/biolink-model/KnowledgeLevelEnum/) + type: array + items: + type: string + minItems: 1 + agent_types: + description: >- + The agent types contributing to this meta edge. + If provided, this property SHOULD contain all possible + agent_types relevant edges might return. + (See https://biolink.github.io/biolink-model/AgentTypeEnum/) + type: array + items: + type: string + minItems: 1 + sources: + description: >- + The infores CURIEs for sources contributing to this meta edge. + If provided, this property SHOULD contain resource_ids for + all relevant resource_roles. Services MAY choose to omit + their own aggregator resource_ids. + type: array + items: + $ref: '#/components/schemas/CURIE' + minItems: 1 + required: + - subject + - predicate + - object + additionalProperties: false + MetaQualifier: + type: object + properties: + qualifier_type_id: + $ref: '#/components/schemas/CURIE' + description: >- + The CURIE of the qualifier type. + examples: + - biolink:subject_aspect_qualifier + applicable_values: + type: array + minItems: 1 + description: >- + The list of values that are possible for this qualifier. + items: + type: string + examples: + - [expression, activity, abundance, degradation] + required: + - qualifier_type_id + MetaAttribute: + type: object + properties: + attribute_type_id: + $ref: '#/components/schemas/CURIE' + description: >- + Type of an attribute provided by this TRAPI web service + (preferably the CURIE of a Biolink association slot) + examples: + - biolink:p_value + attribute_source: + type: string + description: >- + Source of an attribute provided by this TRAPI web service. + examples: + - infores:chembl + original_attribute_names: + type: array + items: + type: string + description: >- + Names of an the attribute as provided by the source. + minItems: 1 + constraint_use: + type: boolean + description: >- + Indicates whether this attribute can be used as a query + constraint. + default: false + constraint_name: + type: string + description: >- + Human-readable name or label for the constraint concept. + Required whenever constraint_use is true. + examples: + - p-value + required: + - attribute_type_id + AttributeConstraint: + type: object + description: >- + Generic query constraint for a query node or query edge + properties: + id: + $ref: '#/components/schemas/CURIE' + description: >- + CURIE of the concept being constrained. For properties + defined by the Biolink Model this SHOULD be a biolink CURIE, + or else, if possible, from the EDAM ontology. If a suitable + CURIE does not exist, enter a descriptive phrase here and + submit the new type for consideration by the appropriate + ontology. + examples: + - EDAM:data_0844 + name: + type: string + description: >- + Human-readable name or label for the constraint concept. + It SHOULD be the term name of the CURIE used + as the 'id' and SHOULD NOT supersede the use of AttributeConstraint.id + (which is required). This is redundant but recommended for human + readability. + examples: + - molecular mass + not: + type: boolean + default: false + operator: + type: string + description: >- + Relationship between the database value and the constraint value + for the specified id. The operators ==, >, and < mean + is equal to, is greater than, and is less than, + respectively. The 'matches' operator indicates that the value + is a regular expression to be evaluated. If value is a list type, + then at least one evaluation must be true (equivalent to OR). + This means that the == operator with a list acts like a SQL 'IN' + clause. If the value of the compared attribute is a list, then + comparisons are performed between each of the constraint values + and each of the attribute values, and any one true evaluation + counts as an overall true (e.g., [1,2,3] == [6,7,2] is true). + The == operator is therefore a broad interpretation of inclusion. + The '===' operator requires that the constraint value and + the attribute value be the same data type, length, + content, and order (e.g. only [1,2,3] === [1,2,3]). + The 'not' property negates the operator such that not + and == means 'not equal to' (or 'not in' for a list), and not > + means <=, and not < means >=, not matches means does not + match, and not === means the match between the constraint + and attribute values are not exact. + The '==' operator SHOULD NOT be used in a manner that + describes an "is a" subclass relationship for the parent QNode. + enum: + - == + - '>' + - < + - matches + - === + value: + examples: + - 57.0 + description: >- + Value of the attribute. May be any data type, including a list. + If the value is a list and there are multiple items, at least one + comparison must be true (equivalent to OR) unless the '===' + operator is used. If 'value' is of data + type 'object', the keys of the object MAY be treated as a list. + A 'list' data type paired with the '>' or '<' operators will + encode extraneous comparisons, but this is permitted as it is in + SQL and other languages. + unit_id: + examples: + - UO:0000222 + description: >- + CURIE of the units of the value or list of values in the 'value' + property. The Units of Measurement Ontology (UO) should be used + if possible. The unit_id MUST be provided for (lists of) + numerical values that correspond to a quantity that has units. + unit_name: + examples: + - kilodalton + description: >- + Term name that is associated with the CURIE of the units + of the value or list of values in the 'value' + property. The Units of Measurement Ontology (UO) SHOULD be used + if possible. This property SHOULD be provided if a unit_id is + provided. This is redundant but recommended for human readability. + required: + - id + - operator + - value + additionalProperties: false + RetrievalSource: + type: object + description: >- + Provides information about how a particular InformationResource + served as a source from which knowledge expressed in an Edge, or + data used to generate this knowledge, was retrieved. + properties: + resource_id: + $ref: '#/components/schemas/CURIE' + description: >- + The CURIE for an Information Resource that served as a source + of knowledge expressed in an Edge, or a source of data used to + generate this knowledge. + examples: + - infores:drugbank + resource_role: + $ref: '#/components/schemas/ResourceRoleEnum' + description: >- + The role played by the InformationResource in serving as a + source for an Edge. Note that a given Edge should have one + and only one 'primary' source, and may have any number of + 'aggregator' or 'supporting data' sources. + upstream_resource_ids: + type: array + minItems: 1 + items: + $ref: '#/components/schemas/CURIE' + description: >- + An upstream InformationResource from which the resource + being described directly retrieved a record of the knowledge + expressed in the Edge, or data used to generate this knowledge. + This is an array because there are cases where a merged Edge + holds knowledge that was retrieved from multiple sources. e.g. + an Edge provided by the ARAGORN ARA can express knowledge it + retrieved from both the automat-mychem-info and molepro KPs, + which both provided it with records of this single fact. + examples: + - ['infores:automat-mychem-info', 'infores:molepro'] + source_record_urls: + type: array + minItems: 1 + items: + type: string + examples: + - https://db.idrblab.net/ttd/data/drug/details/d0az3c + - https://db.idrblab.net/ttd/data/target/details/t57700 + description: >- + A URL linking to a specific web page or document provided by the + source that contains a record of the knowledge expressed in the + Edge. If the knowledge is contained in more than one web page on + an information resource's site, urls MAY be provided for each. + For example, Therapeutic Targets Database (TTD) has separate web + pages for 'Imatinib' and its protein target KIT, both of which hold + the claim that 'the KIT protein is a therapeutic target for Imatinib'. + required: + - resource_id + - resource_role + additionalProperties: true + ResourceRoleEnum: + type: string + description: >- + The role played by the information resource in serving as a + source for an Edge. Note that a given Edge should have one + and only one 'primary_knowledge_source' source, + and may have any number of 'aggregator_knowledge_source' + or 'supporting_data_source' sources. This enumeration + is found in Biolink Model, but is repeated here for convenience. + enum: + - primary_knowledge_source + - aggregator_knowledge_source + - supporting_data_source diff --git a/TCT/trapi_validation.py b/TCT/trapi_validation.py new file mode 100644 index 0000000..8f9bae4 --- /dev/null +++ b/TCT/trapi_validation.py @@ -0,0 +1,82 @@ +"""Offline validation against checksum-pinned upstream TRAPI schemas.""" + +from copy import deepcopy +from functools import lru_cache +import hashlib +import json +from pathlib import Path + +ROOT = Path(__file__).with_name("schemas") + + +@lru_cache(maxsize=2) +def schema(version): + import yaml + from jsonschema import Draft202012Validator + from referencing import Registry + + manifest = json.loads((ROOT / "manifest.json").read_text()) + if version not in manifest: + raise ValueError( + f"Unsupported TRAPI version {version!r}; available: {sorted(manifest)}" + ) + raw = (ROOT / f"trapi-{version}.yaml").read_bytes() + if hashlib.sha256(raw).hexdigest() != manifest[version]["sha256"]: + raise ValueError("Pinned TRAPI schema checksum mismatch") + document = yaml.safe_load(raw) + + def convert(value): + # OpenAPI 3.0 nullable is not a JSON Schema keyword. Preserve its meaning. + if isinstance(value, list): + return [convert(x) for x in value] + if not isinstance(value, dict): + return value + result = {k: convert(v) for k, v in value.items() if k != "nullable"} + return ( + {"anyOf": [result, {"type": "null"}]} if value.get("nullable") else result + ) + + root = { + "$ref": "#/components/schemas/Response", + "components": convert(document["components"]), + } + + def no_network(uri): + raise ValueError(f"External schema reference is not vendored: {uri}") + + return Draft202012Validator(root, registry=Registry(retrieve=no_network)), manifest[ + version + ] + + +def validate(response, version="1.5.0"): + validator, pin = schema(version) + + def leaves(error): + if error.context: + for child in error.context: + yield from leaves(child) + elif error.schema != {"type": "null"}: + yield error + + try: + errors = sorted( + [ + { + "path": "/" + "/".join(str(x) for x in e.absolute_path), + "message": e.message[:1000], + } + for parent in validator.iter_errors(response) + for e in leaves(parent) + ], + key=lambda e: (e["path"], e["message"]), + ) + except Exception as error: + errors = [{"path": "/", "message": f"Validation unavailable: {error}"}] + return { + "version": version, + "schema": deepcopy(pin), + "valid": not errors, + "errors": errors, + "scope": "TRAPI structural schema only; not Biolink semantic validation", + } diff --git a/pyproject.toml b/pyproject.toml index 9da61b0..6b1dbfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,6 +27,8 @@ dependencies = [ "igraph", "pyvis", "zstandard", + "jsonschema>=4.18", + "PyYAML>=6", ] [project.optional-dependencies] diff --git a/tests/fixtures/rollup/README.md b/tests/fixtures/rollup/README.md new file mode 100644 index 0000000..e6f6c0f --- /dev/null +++ b/tests/fixtures/rollup/README.md @@ -0,0 +1,8 @@ +These JSON service responses and native-output goldens are copied unchanged from +Phase 1's single-NLQ benchmark on codex/trapi-benchmarks (0b24011). They are +explicitly synthetic. case.json retains the original corpus attachment hash and +baseline identity. No biomedical assertion, historic count or live coverage is a +golden. composition.json adds explicit query mappings and a closed replay recipe +for that same question. It uses the existing filters/join, replacing input literals +with recorded-source references. Its source IDs hash the original requests and +responses. Structural empty-attribute additions are reported by the reducer. diff --git a/tests/fixtures/rollup/case.json b/tests/fixtures/rollup/case.json new file mode 100644 index 0000000..ce7bc42 --- /dev/null +++ b/tests/fixtures/rollup/case.json @@ -0,0 +1,90 @@ +{ + "attachment_sha256": "1862b4b01ef8d49e72fd96ec6ac17cddfd878cb8b8bb9a8bd762a91dcff07324", + "baseline_commit": "c8a2200", + "expected_paths": [ + { + "chemical": "EXAMPLE:chemical1", + "disease": "MONDO:0004975", + "evidence": [ + "a1", + "b1" + ], + "gene": "EXAMPLE:gene1" + } + ], + "id": "alzheimer-two-hop-lookup-v1", + "limitations": [ + "Responses intentionally contain nonmatching distractor edges to exercise explicit filtering.", + "No medical validity claim or live completeness claim.", + "No TRAPI rollup validator exists at this phase." + ], + "origin": "synthetic; no biological truth asserted", + "queries": { + "chemicals": { + "message": { + "query_graph": { + "edges": { + "e00": { + "knowledge_type": "lookup", + "object": "n1", + "predicates": [ + "biolink:affects" + ], + "subject": "n0" + } + }, + "nodes": { + "n0": { + "categories": [ + "biolink:ChemicalEntity" + ] + }, + "n1": { + "categories": [ + "biolink:Gene" + ], + "ids": [ + "EXAMPLE:gene1", + "EXAMPLE:gene2" + ] + } + } + } + } + }, + "genes": { + "message": { + "query_graph": { + "edges": { + "e00": { + "knowledge_type": "lookup", + "object": "n1", + "predicates": [ + "biolink:associated_with" + ], + "subject": "n0" + } + }, + "nodes": { + "n0": { + "categories": [ + "biolink:Gene" + ] + }, + "n1": { + "categories": [ + "biolink:Disease" + ], + "ids": [ + "MONDO:0004975" + ] + } + } + } + } + } + }, + "question": "Which chemicals are recorded as affecting genes associated with Alzheimer\u2019s disease? Show the intermediate genes and the supporting evidence for each connection.", + "schema_validation": "not_performed", + "source": "User-provided HelmsDeep NLQ corpus, A4" +} diff --git a/tests/fixtures/rollup/chemical.json b/tests/fixtures/rollup/chemical.json new file mode 100644 index 0000000..fd83b1a --- /dev/null +++ b/tests/fixtures/rollup/chemical.json @@ -0,0 +1,154 @@ +{ + "message": { + "auxiliary_graphs": {}, + "knowledge_graph": { + "edges": { + "a1": { + "object": "EXAMPLE:gene1", + "predicate": "biolink:affects", + "sources": [ + { + "resource_id": "infores:synthetic-a", + "resource_role": "primary_knowledge_source" + } + ], + "subject": "EXAMPLE:chemical1" + }, + "a2": { + "object": "EXAMPLE:gene1", + "predicate": "biolink:related_to", + "sources": [ + { + "resource_id": "infores:synthetic-a", + "resource_role": "primary_knowledge_source" + } + ], + "subject": "EXAMPLE:chemical2" + }, + "a3": { + "object": "EXAMPLE:gene2", + "predicate": "biolink:affects", + "sources": [ + { + "resource_id": "infores:synthetic-a", + "resource_role": "primary_knowledge_source" + } + ], + "subject": "EXAMPLE:chemical3" + } + }, + "nodes": { + "EXAMPLE:chemical1": { + "categories": [ + "biolink:ChemicalEntity" + ], + "name": "EXAMPLE:chemical1" + }, + "EXAMPLE:chemical2": { + "categories": [ + "biolink:ChemicalEntity" + ], + "name": "EXAMPLE:chemical2" + }, + "EXAMPLE:chemical3": { + "categories": [ + "biolink:ChemicalEntity" + ], + "name": "EXAMPLE:chemical3" + }, + "EXAMPLE:gene1": { + "categories": [ + "biolink:Gene" + ], + "name": "EXAMPLE:gene1" + }, + "EXAMPLE:gene2": { + "categories": [ + "biolink:Gene" + ], + "name": "EXAMPLE:gene2" + } + } + }, + "results": [ + { + "analyses": [ + { + "edge_bindings": { + "e00": [ + { + "id": "a1" + } + ] + }, + "resource_id": "infores:synthetic-provider" + } + ], + "node_bindings": { + "n0": [ + { + "id": "EXAMPLE:chemical1" + } + ], + "n1": [ + { + "id": "EXAMPLE:gene1" + } + ] + } + }, + { + "analyses": [ + { + "edge_bindings": { + "e00": [ + { + "id": "a2" + } + ] + }, + "resource_id": "infores:synthetic-provider" + } + ], + "node_bindings": { + "n0": [ + { + "id": "EXAMPLE:chemical2" + } + ], + "n1": [ + { + "id": "EXAMPLE:gene1" + } + ] + } + }, + { + "analyses": [ + { + "edge_bindings": { + "e00": [ + { + "id": "a3" + } + ] + }, + "resource_id": "infores:synthetic-provider" + } + ], + "node_bindings": { + "n0": [ + { + "id": "EXAMPLE:chemical3" + } + ], + "n1": [ + { + "id": "EXAMPLE:gene2" + } + ] + } + } + ] + } +} diff --git a/tests/fixtures/rollup/composition.json b/tests/fixtures/rollup/composition.json new file mode 100644 index 0000000..ca4b6f9 --- /dev/null +++ b/tests/fixtures/rollup/composition.json @@ -0,0 +1,180 @@ +{ + "closed": true, + "query_graph": { + "nodes": { + "chemical": { + "categories": [ + "biolink:ChemicalEntity" + ] + }, + "gene": { + "categories": [ + "biolink:Gene" + ] + }, + "disease": { + "categories": [ + "biolink:Disease" + ], + "ids": [ + "MONDO:0004975" + ] + } + }, + "edges": { + "affects": { + "subject": "chemical", + "object": "gene", + "predicates": [ + "biolink:affects" + ], + "knowledge_type": "lookup" + }, + "associated": { + "subject": "gene", + "object": "disease", + "predicates": [ + "biolink:associated_with" + ], + "knowledge_type": "lookup" + } + } + }, + "sources": { + "source:eac1467c7b45817aeb8e47d133d9d0e19dbf69fb0eb7948e938feea18f23f4f4": { + "nodes": { + "n0": "gene", + "n1": "disease" + }, + "edges": { + "e00": "associated" + } + }, + "source:a62cfd67b86ab9ed05a4baa1f70ff162d900943a4bf7a2bd3f0f7a2b192acaac": { + "nodes": { + "n0": "chemical", + "n1": "gene" + }, + "edges": { + "e00": "affects" + } + } + }, + "workflow": { + "schema_version": 1, + "steps": [ + { + "id": "genes", + "op": "source", + "source_id": "source:eac1467c7b45817aeb8e47d133d9d0e19dbf69fb0eb7948e938feea18f23f4f4" + }, + { + "id": "chemicals", + "op": "source", + "source_id": "source:a62cfd67b86ab9ed05a4baa1f70ff162d900943a4bf7a2bd3f0f7a2b192acaac" + }, + { + "id": "gene_rows", + "op": "rows", + "input": { + "$ref": "genes", + "path": "/message/knowledge_graph/edges" + } + }, + { + "id": "chemical_rows", + "op": "rows", + "input": { + "$ref": "chemicals", + "path": "/message/knowledge_graph/edges" + } + }, + { + "id": "gene_matches", + "op": "filter", + "input": { + "$ref": "gene_rows" + }, + "conditions": [ + { + "path": "/value/predicate", + "operator": "eq", + "value": "biolink:associated_with" + }, + { + "path": "/value/object", + "operator": "eq", + "value": "MONDO:0004975" + } + ] + }, + { + "id": "chemical_matches", + "op": "filter", + "input": { + "$ref": "chemical_rows" + }, + "conditions": [ + { + "path": "/value/predicate", + "operator": "eq", + "value": "biolink:affects" + } + ] + }, + { + "id": "paths", + "op": "join", + "left": { + "$ref": "chemical_matches" + }, + "right": { + "$ref": "gene_matches" + }, + "left_on": "/value/object", + "right_on": "/value/subject" + }, + { + "id": "summary", + "op": "project", + "input": { + "$ref": "paths" + }, + "fields": { + "chemical": "/left/value/subject", + "gene": "/left/value/object", + "disease": "/right/value/object", + "chemical_edge": "/left/key", + "gene_edge": "/right/key" + } + } + ], + "outputs": { + "paths": { + "$ref": "paths" + }, + "summary": { + "$ref": "summary" + } + } + }, + "selection": { + "output": "summary", + "resource_id": "infores:tct", + "nodes": { + "chemical": "/chemical", + "gene": "/gene", + "disease": "/disease" + }, + "edges": { + "affects": { + "source_id": "source:a62cfd67b86ab9ed05a4baa1f70ff162d900943a4bf7a2bd3f0f7a2b192acaac", + "path": "/chemical_edge" + }, + "associated": { + "source_id": "source:eac1467c7b45817aeb8e47d133d9d0e19dbf69fb0eb7948e938feea18f23f4f4", + "path": "/gene_edge" + } + } + } +} \ No newline at end of file diff --git a/tests/fixtures/rollup/gene.json b/tests/fixtures/rollup/gene.json new file mode 100644 index 0000000..cd8dc77 --- /dev/null +++ b/tests/fixtures/rollup/gene.json @@ -0,0 +1,111 @@ +{ + "message": { + "auxiliary_graphs": {}, + "knowledge_graph": { + "edges": { + "b1": { + "object": "MONDO:0004975", + "predicate": "biolink:associated_with", + "sources": [ + { + "resource_id": "infores:synthetic-b", + "resource_role": "primary_knowledge_source" + } + ], + "subject": "EXAMPLE:gene1" + }, + "b2": { + "object": "EXAMPLE:other-disease", + "predicate": "biolink:associated_with", + "sources": [ + { + "resource_id": "infores:synthetic-b", + "resource_role": "primary_knowledge_source" + } + ], + "subject": "EXAMPLE:gene2" + } + }, + "nodes": { + "EXAMPLE:gene1": { + "categories": [ + "biolink:Gene" + ], + "name": "EXAMPLE:gene1" + }, + "EXAMPLE:gene2": { + "categories": [ + "biolink:Gene" + ], + "name": "EXAMPLE:gene2" + }, + "EXAMPLE:other-disease": { + "categories": [ + "biolink:Disease" + ], + "name": "EXAMPLE:other-disease" + }, + "MONDO:0004975": { + "categories": [ + "biolink:Disease" + ], + "name": "MONDO:0004975" + } + } + }, + "results": [ + { + "analyses": [ + { + "edge_bindings": { + "e00": [ + { + "id": "b1" + } + ] + }, + "resource_id": "infores:synthetic-provider" + } + ], + "node_bindings": { + "n0": [ + { + "id": "EXAMPLE:gene1" + } + ], + "n1": [ + { + "id": "MONDO:0004975" + } + ] + } + }, + { + "analyses": [ + { + "edge_bindings": { + "e00": [ + { + "id": "b2" + } + ] + }, + "resource_id": "infores:synthetic-provider" + } + ], + "node_bindings": { + "n0": [ + { + "id": "EXAMPLE:gene2" + } + ], + "n1": [ + { + "id": "EXAMPLE:other-disease" + } + ] + } + } + ] + } +} diff --git a/tests/fixtures/rollup/golden.json b/tests/fixtures/rollup/golden.json new file mode 100644 index 0000000..acfc595 --- /dev/null +++ b/tests/fixtures/rollup/golden.json @@ -0,0 +1,400 @@ +{ + "final": { + "answer": "In this synthetic snapshot, EXAMPLE:chemical1 affects EXAMPLE:gene1, which is associated with MONDO:0004975. This is not evidence of treatment efficacy.", + "coverage": "Complete for this frozen synthetic snapshot only; not a live or exhaustive biomedical answer.", + "paths": [ + { + "chemical": "EXAMPLE:chemical1", + "disease": "MONDO:0004975", + "evidence": [ + "a1", + "b1" + ], + "gene": "EXAMPLE:gene1" + } + ], + "sources": [ + "infores:synthetic-a", + "infores:synthetic-b" + ] + }, + "history": [ + { + "native_type": "TranslatorNode", + "output": { + "attributes": null, + "curie": "MONDO:0004975", + "curie_synonyms": [ + "MONDO:0004975" + ], + "label": "Alzheimer disease", + "synonyms": [ + "Alzheimer disease" + ], + "taxa": null, + "types": [ + "biolink:Disease" + ] + }, + "tool": "normalize_nodes" + }, + { + "native_type": "tuple", + "output": [ + [ + { + "ci_url": null, + "id": "https://smart-api.info/ui/chemical-provider", + "prod_url": "https://fixture.invalid/chemical/query/", + "test_url": null, + "title": "chemical-provider" + }, + { + "ci_url": null, + "id": "https://smart-api.info/ui/gene-provider", + "prod_url": "https://fixture.invalid/gene/query/", + "test_url": null, + "title": "gene-provider" + } + ], + { + "chemical-provider": "https://fixture.invalid/chemical/query/", + "gene-provider": "https://fixture.invalid/gene/query/" + } + ], + "tool": "get_kp_info" + }, + { + "native_type": "dict", + "output": { + "auxiliary_graphs": {}, + "knowledge_graph": { + "edges": { + "b1": { + "object": "MONDO:0004975", + "predicate": "biolink:associated_with", + "sources": [ + { + "resource_id": "infores:synthetic-b", + "resource_role": "primary_knowledge_source" + } + ], + "subject": "EXAMPLE:gene1" + }, + "b2": { + "object": "EXAMPLE:other-disease", + "predicate": "biolink:associated_with", + "sources": [ + { + "resource_id": "infores:synthetic-b", + "resource_role": "primary_knowledge_source" + } + ], + "subject": "EXAMPLE:gene2" + } + }, + "nodes": { + "EXAMPLE:gene1": { + "categories": [ + "biolink:Gene" + ], + "name": "EXAMPLE:gene1" + }, + "EXAMPLE:gene2": { + "categories": [ + "biolink:Gene" + ], + "name": "EXAMPLE:gene2" + }, + "EXAMPLE:other-disease": { + "categories": [ + "biolink:Disease" + ], + "name": "EXAMPLE:other-disease" + }, + "MONDO:0004975": { + "categories": [ + "biolink:Disease" + ], + "name": "MONDO:0004975" + } + } + }, + "results": [ + { + "analyses": [ + { + "edge_bindings": { + "e00": [ + { + "id": "b1" + } + ] + }, + "resource_id": "infores:synthetic-provider" + } + ], + "node_bindings": { + "n0": [ + { + "id": "EXAMPLE:gene1" + } + ], + "n1": [ + { + "id": "MONDO:0004975" + } + ] + } + }, + { + "analyses": [ + { + "edge_bindings": { + "e00": [ + { + "id": "b2" + } + ] + }, + "resource_id": "infores:synthetic-provider" + } + ], + "node_bindings": { + "n0": [ + { + "id": "EXAMPLE:gene2" + } + ], + "n1": [ + { + "id": "EXAMPLE:other-disease" + } + ] + } + } + ] + }, + "tool": "query_knowledge_provider" + }, + { + "native_type": "dict", + "output": { + "auxiliary_graphs": {}, + "knowledge_graph": { + "edges": { + "a1": { + "object": "EXAMPLE:gene1", + "predicate": "biolink:affects", + "sources": [ + { + "resource_id": "infores:synthetic-a", + "resource_role": "primary_knowledge_source" + } + ], + "subject": "EXAMPLE:chemical1" + }, + "a2": { + "object": "EXAMPLE:gene1", + "predicate": "biolink:related_to", + "sources": [ + { + "resource_id": "infores:synthetic-a", + "resource_role": "primary_knowledge_source" + } + ], + "subject": "EXAMPLE:chemical2" + }, + "a3": { + "object": "EXAMPLE:gene2", + "predicate": "biolink:affects", + "sources": [ + { + "resource_id": "infores:synthetic-a", + "resource_role": "primary_knowledge_source" + } + ], + "subject": "EXAMPLE:chemical3" + } + }, + "nodes": { + "EXAMPLE:chemical1": { + "categories": [ + "biolink:ChemicalEntity" + ], + "name": "EXAMPLE:chemical1" + }, + "EXAMPLE:chemical2": { + "categories": [ + "biolink:ChemicalEntity" + ], + "name": "EXAMPLE:chemical2" + }, + "EXAMPLE:chemical3": { + "categories": [ + "biolink:ChemicalEntity" + ], + "name": "EXAMPLE:chemical3" + }, + "EXAMPLE:gene1": { + "categories": [ + "biolink:Gene" + ], + "name": "EXAMPLE:gene1" + }, + "EXAMPLE:gene2": { + "categories": [ + "biolink:Gene" + ], + "name": "EXAMPLE:gene2" + } + } + }, + "results": [ + { + "analyses": [ + { + "edge_bindings": { + "e00": [ + { + "id": "a1" + } + ] + }, + "resource_id": "infores:synthetic-provider" + } + ], + "node_bindings": { + "n0": [ + { + "id": "EXAMPLE:chemical1" + } + ], + "n1": [ + { + "id": "EXAMPLE:gene1" + } + ] + } + }, + { + "analyses": [ + { + "edge_bindings": { + "e00": [ + { + "id": "a2" + } + ] + }, + "resource_id": "infores:synthetic-provider" + } + ], + "node_bindings": { + "n0": [ + { + "id": "EXAMPLE:chemical2" + } + ], + "n1": [ + { + "id": "EXAMPLE:gene1" + } + ] + } + }, + { + "analyses": [ + { + "edge_bindings": { + "e00": [ + { + "id": "a3" + } + ] + }, + "resource_id": "infores:synthetic-provider" + } + ], + "node_bindings": { + "n0": [ + { + "id": "EXAMPLE:chemical3" + } + ], + "n1": [ + { + "id": "EXAMPLE:gene2" + } + ] + } + } + ] + }, + "tool": "query_knowledge_provider" + }, + { + "native_type": "dict", + "output": { + "outputs": { + "paths": [ + { + "left": { + "key": "a1", + "value": { + "object": "EXAMPLE:gene1", + "predicate": "biolink:affects", + "sources": [ + { + "resource_id": "infores:synthetic-a", + "resource_role": "primary_knowledge_source" + } + ], + "subject": "EXAMPLE:chemical1" + } + }, + "right": { + "key": "b1", + "value": { + "object": "MONDO:0004975", + "predicate": "biolink:associated_with", + "sources": [ + { + "resource_id": "infores:synthetic-b", + "resource_role": "primary_knowledge_source" + } + ], + "subject": "EXAMPLE:gene1" + } + } + } + ], + "summary": [ + { + "chemical": "EXAMPLE:chemical1", + "chemical_edge": "a1", + "disease": "MONDO:0004975", + "gene": "EXAMPLE:gene1", + "gene_edge": "b1" + } + ] + }, + "status": "succeeded" + }, + "tool": "run_query_workflow" + } + ], + "limitations": [ + "MCP currently renders the discovery DataFrame as text rather than structured rows.", + "query_KP prints a success line on stdout, so CLI output is not a single JSON document." + ], + "mcp_registry_output": [ + " id ... test_url\n0 https://smart-api.info/ui/chemical-provider ... None\n1 https://smart-api.info/ui/gene-provider ... None\n\n[2 rows x 5 columns]", + { + "chemical-provider": "https://fixture.invalid/chemical/query/", + "gene-provider": "https://fixture.invalid/gene/query/" + } + ], + "origin": "Reviewed projection of baseline c8a2200 fixed trajectory; synthetic records only", + "service_call_count": 4 +} diff --git a/tests/fixtures/rollup/normalization.json b/tests/fixtures/rollup/normalization.json new file mode 100644 index 0000000..9f8fe61 --- /dev/null +++ b/tests/fixtures/rollup/normalization.json @@ -0,0 +1,17 @@ +{ + "MONDO:0004975": { + "equivalent_identifiers": [ + { + "identifier": "MONDO:0004975", + "label": "Alzheimer disease" + } + ], + "id": { + "identifier": "MONDO:0004975", + "label": "Alzheimer disease" + }, + "type": [ + "biolink:Disease" + ] + } +} diff --git a/tests/fixtures/rollup/predicates.json b/tests/fixtures/rollup/predicates.json new file mode 100644 index 0000000..7676e94 --- /dev/null +++ b/tests/fixtures/rollup/predicates.json @@ -0,0 +1,8 @@ +{ + "chemical-provider": [ + "biolink:affects" + ], + "gene-provider": [ + "biolink:associated_with" + ] +} diff --git a/tests/fixtures/rollup/providers.json b/tests/fixtures/rollup/providers.json new file mode 100644 index 0000000..ea90456 --- /dev/null +++ b/tests/fixtures/rollup/providers.json @@ -0,0 +1,4 @@ +{ + "chemical-provider": "https://fixture.invalid/chemical/query", + "gene-provider": "https://fixture.invalid/gene/query" +} diff --git a/tests/fixtures/rollup/registry.json b/tests/fixtures/rollup/registry.json new file mode 100644 index 0000000..e70ef05 --- /dev/null +++ b/tests/fixtures/rollup/registry.json @@ -0,0 +1,28 @@ +{ + "hits": [ + { + "_id": "chemical-provider", + "info": { + "title": "chemical-provider" + }, + "servers": [ + { + "url": "https://fixture.invalid/chemical", + "x-maturity": "production" + } + ] + }, + { + "_id": "gene-provider", + "info": { + "title": "gene-provider" + }, + "servers": [ + { + "url": "https://fixture.invalid/gene", + "x-maturity": "production" + } + ] + } + ] +} diff --git a/tests/fixtures/rollup/rollup-golden.json b/tests/fixtures/rollup/rollup-golden.json new file mode 100644 index 0000000..6405c3d --- /dev/null +++ b/tests/fixtures/rollup/rollup-golden.json @@ -0,0 +1,15 @@ +{ + "reducer_version": 1, + "trapi_version": "1.5.0", + "rollup_hash": "9c2acd024df2eec85dda1a22bd911a7b2921e8fc4d156f6efdd505d94dc4383e", + "selected_result_count": 1, + "accumulated_edge_count": 5, + "source_ids": [ + "source:a62cfd67b86ab9ed05a4baa1f70ff162d900943a4bf7a2bd3f0f7a2b192acaac", + "source:eac1467c7b45817aeb8e47d133d9d0e19dbf69fb0eb7948e938feea18f23f4f4" + ], + "selected_original_edges": [ + "a1", + "b1" + ] +} diff --git a/tests/test_evidence.py b/tests/test_evidence.py new file mode 100644 index 0000000..0846d15 --- /dev/null +++ b/tests/test_evidence.py @@ -0,0 +1,649 @@ +"""Offline Phase 4 acceptance, including the single original NLQ fixture.""" + +import asyncio +from copy import deepcopy +import hashlib +import json +from pathlib import Path +import random +import sqlite3 + +import pytest +import requests + +from TCT.evidence import ( + accumulate, + canonical, + digest, + evidence, + EvidenceError, + inventory, + read_run, + reduce_evidence, +) +from TCT.execution import execution +from TCT.interfaces import tools +from TCT.interfaces.serialization import to_jsonable +from TCT.trapi_validation import validate + +FIXTURES = Path(__file__).parent / "fixtures" / "rollup" + + +def load(name): + return json.loads((FIXTURES / (name + ".json")).read_text()) + + +@pytest.fixture +def captured(monkeypatch, tmp_path): + calls = [] + from TCT.config import service_url + + providers = load("providers") + + def frozen(session, method, url, **kw): + calls.append((method, url, deepcopy(kw))) + if method.lower() == "get" and url == service_url("smartapi_registry"): + body = load("registry") + elif method.lower() == "get" and url.endswith("/get_normalized_nodes"): + assert kw["params"]["curie"] == "MONDO:0004975" + body = load("normalization") + elif method.lower() == "post" and url in providers.values(): + which = "genes" if url == providers["gene-provider"] else "chemicals" + assert kw["json"] == load("case")["queries"][which] + body = load("gene" if which == "genes" else "chemical") + else: + raise AssertionError(f"Unexpected service request {method} {url}") + response = requests.Response() + response.status_code = 200 + response._content = json.dumps(body).encode() + return response + + monkeypatch.setattr(requests.sessions.Session, "request", frozen) + with execution( + {"record": True, "trapi": True, "store": str(tmp_path), "langfuse": False} + ) as run: + normalization = tools.normalize_nodes( + "MONDO:0004975", return_equivalent_identifiers=True + ) + registry = tools.get_kp_info() + genes = tools.query_knowledge_provider( + "gene-provider", + load("case")["queries"]["genes"], + registry[1], + load("predicates"), + ) + chemicals = tools.query_knowledge_provider( + "chemical-provider", + load("case")["queries"]["chemicals"], + registry[1], + load("predicates"), + ) + assert len(calls) == 4 + for actual, golden in zip( + [normalization, registry, genes, chemicals], + load("golden")["history"], + strict=False, + ): + assert to_jsonable(actual) == golden["output"] + records = read_run(tmp_path, run.run_id) + assert accumulate(run) == records + # No replay/export operation may contact any service. + monkeypatch.setattr( + requests.sessions.Session, + "request", + lambda *a, **kw: pytest.fail("replay performed network I/O"), + ) + return records, load("composition"), tmp_path, run.run_id + + +def test_single_nlq_valid_rollup_and_lineage(captured): + events, composition, _, _ = captured + report = reduce_evidence(events, composition) + assert report["status"] == "valid", report + response = report["draft"] + assert validate(response)["valid"] + assert response["message"]["query_graph"] == composition["query_graph"] + (result,) = response["message"]["results"] + assert {k: v[0]["id"] for k, v in result["node_bindings"].items()} == { + "chemical": "EXAMPLE:chemical1", + "gene": "EXAMPLE:gene1", + "disease": "MONDO:0004975", + } + assert { + x for origin in result["x-tct-evidence"] for x in origin["original_ids"] + } == {"a1", "b1"} + assert result["x-tct-selection-rows"] == [0] + assert ( + len(response["message"]["knowledge_graph"]["edges"]) == 5 + ) # Includes all three distractors. + trace = response["x-tct"]["transformations"] + assert any(step["op"] == "join" and step["row_lineage"] for step in trace.values()) + assert any(step["op"] == "filter" and step["decisions"] for step in trace.values()) + assert all(response["x-tct"]["structural_additions"].values()) + assert report["rollup_hash"] == digest(response) + assert report["rollup_hash"] == load("rollup-golden")["rollup_hash"] + assert sorted(report["references"]) == load("rollup-golden")["source_ids"] + assert not any( + k in canonical(response) + for k in ["started_at", "finished_at", "duration_seconds"] + ) + assert ( + evidence({"action": "export", "events": events, "composition": composition}) + == response + ) + + +def test_accumulation_algebra_and_shuffled_completion(captured): + events, composition, _, _ = captured + a, b, c = events[:3], events[3:6], events[6:] + assert accumulate(accumulate(a, b), c) == accumulate(a, accumulate(b, c)) + assert accumulate(a, b) == accumulate(b, a) + assert accumulate(events, events) == accumulate(events) + expected = reduce_evidence(events, composition) + for seed in range(8): + shuffled = deepcopy(events + events) + random.Random(seed).shuffle(shuffled) + assert reduce_evidence(shuffled, composition) == expected + # Dictionary insertion order is representation only, even for duplicate events. + backwards = json.loads( + json.dumps(events, sort_keys=True), + object_pairs_hook=lambda pairs: dict(reversed(pairs)), + ) + assert reduce_evidence(events + backwards, composition) == expected + + +def test_conflicting_event_identity_is_rejected(captured): + events, _, _, _ = captured + changed = deepcopy(events[0]) + changed["status"] = "failed" + with pytest.raises(EvidenceError, match="Conflicting"): + accumulate(events, [changed]) + + +@pytest.mark.parametrize( + "mutation,phrase", + [ + (lambda c: c.update(closed=False), "closed=true"), + (lambda c: c["sources"].update({"source:missing": {}}), "Missing closed input"), + ( + lambda c: c["query_graph"]["edges"]["affects"].update( + subject="gene", object="chemical" + ), + "direction", + ), + ( + lambda c: c["query_graph"]["edges"]["affects"].update( + knowledge_type="inferred" + ), + "inference", + ), + ( + lambda c: c["query_graph"]["edges"]["affects"].update( + attribute_constraints=[{}] + ), + "constraint", + ), + ( + lambda c: c["workflow"]["steps"].append( + {"id": "bad", "op": "query", "url": "https://invalid", "query": {}} + ), + "network", + ), + (lambda c: c["selection"]["edges"]["affects"].update(path="/gene_edge"), "b1"), + (lambda c: c["selection"]["nodes"].update(disease="/chemical"), "identifier"), + ], +) +def test_invalid_compositions_cannot_be_exported(captured, mutation, phrase): + events, composition, _, _ = captured + mutation(composition) + report = evidence( + {"action": "replay", "events": events, "composition": composition} + ) + assert report["status"] == "invalid" + assert phrase in str(report), report + with pytest.raises(EvidenceError, match="export rejected"): + evidence({"action": "export", "events": events, "composition": composition}) + + +def test_pending_closed_inputs_and_unsupported_final_version(captured): + events, composition, _, _ = captured + events.append( + { + "format_version": 1, + "run_id": "pending", + "call_id": "pending", + "status": "running", + } + ) + assert "still running" in reduce_evidence(events, composition)["error"] + assert ( + "final TRAPI 2.0.0" + in reduce_evidence(events[:-1], composition, "2.0.0")["error"] + ) + + +def test_journal_read_checks_hashes_and_does_not_create(tmp_path, captured): + events, composition, store, run_id = captured + assert evidence({"store": str(store), "run_id": run_id})["event_count"] == len( + events + ) + assert ( + evidence( + { + "action": "export", + "store": str(store), + "run_id": run_id, + "composition": composition, + } + ) + == reduce_evidence(events, composition)["draft"] + ) + nonexistent = tmp_path / "absent" + with pytest.raises(EvidenceError): + read_run(nonexistent, "missing") + assert not nonexistent.exists() + with sqlite3.connect(store / "journal.sqlite3") as db: + checksum = db.execute("SELECT event_hash FROM calls LIMIT 1").fetchone()[0] + (store / "artifacts" / checksum).write_text("{}") + with pytest.raises(EvidenceError, match="checksum"): + read_run(store, run_id) + + +def event(body, query, id="one"): + return { + "format_version": 1, + "run_id": "fixture", + "call_id": id, + "status": "succeeded", + "observations": [ + { + "kind": "service", + "status": "received", + "status_code": 200, + "request": { + "url": "https://fixture.invalid/" + id, + "arguments": {"json": {"message": {"query_graph": query}}}, + }, + "body_json": {"message": body}, + } + ], + } + + +def collision_case(): + query = { + "nodes": { + "a": {"categories": ["biolink:Gene"]}, + "b": {"categories": ["biolink:Gene"]}, + }, + "edges": { + "e": {"subject": "a", "object": "b", "predicates": ["biolink:related_to"]} + }, + } + body = { + "knowledge_graph": { + "nodes": { + "X:1": {"categories": ["biolink:Gene"], "name": "one"}, + "X:2": {"categories": ["biolink:Gene"]}, + }, + "edges": { + "same": { + "subject": "X:1", + "object": "X:2", + "predicate": "biolink:related_to", + "sources": [ + { + "resource_id": "infores:fixture", + "resource_role": "primary_knowledge_source", + } + ], + "qualifiers": [ + { + "qualifier_type_id": "biolink:object_aspect_qualifier", + "qualifier_value": "activity", + } + ], + "attributes": [ + { + "attribute_type_id": "biolink:support_graphs", + "value": ["same"], + } + ], + } + }, + }, + "auxiliary_graphs": {"same": {"edges": ["same"]}}, + "results": [ + { + "node_bindings": { + "a": [{"id": "X:1", "query_id": "X:1"}], + "b": [{"id": "X:2"}], + }, + "analyses": [ + { + "resource_id": "infores:fixture", + "score": 0.7, + "scoring_method": "fixture", + "edge_bindings": {"e": [{"id": "same"}]}, + "support_graphs": ["same"], + } + ], + } + ], + } + first = event(body, query) + other = deepcopy(body) + other["knowledge_graph"]["nodes"]["X:1"]["name"] = "alternate" + other["knowledge_graph"]["edges"]["same"]["sources"][0]["resource_id"] = ( + "infores:other" + ) + second = event(other, query, "two") + events = [first, second] + sources, _, _ = inventory(events) + ids = sorted(sources) + composition = { + "closed": True, + "query_graph": query, + "sources": { + key: {"nodes": {"a": "a", "b": "b"}, "edges": {"e": "e"}} for key in ids + }, + "workflow": { + "schema_version": 1, + "steps": [ + { + "id": "answers", + "op": "literal", + "value": [{"a": "X:1", "b": "X:2", "edge": "same"}], + } + ], + "outputs": {"answers": {"$ref": "answers"}}, + }, + "selection": { + "output": "answers", + "resource_id": "infores:tct", + "nodes": {"a": "/a", "b": "/b"}, + "edges": {"e": {"source_id": ids[0], "path": "/edge"}}, + }, + } + return events, composition + + +def test_identifier_collisions_preserve_qualifiers_analyses_and_support(): + events, composition = collision_case() + report = reduce_evidence(events, composition) + assert report["status"] == "valid", report + response = report["draft"] + message = response["message"] + assert ( + len(message["knowledge_graph"]["edges"]) + == len(message["auxiliary_graphs"]) + == 2 + ) + assert len(response["x-tct"]["node_variants"]["X:1"]) == 2 + for source_id, refs in report["references"].items(): + edge = message["knowledge_graph"]["edges"][refs["edges"]["same"]] + assert edge["qualifiers"][0]["qualifier_value"] == "activity" + assert edge["attributes"][0]["value"] == [refs["auxiliary_graphs"]["same"]] + analysis = response["x-tct"]["source_results"][source_id][0]["analyses"][0] + assert analysis["score"] == 0.7 and analysis["scoring_method"] == "fixture" + assert analysis["support_graphs"] == [refs["auxiliary_graphs"]["same"]] + assert analysis["edge_bindings"]["e"][0]["id"] == refs["edges"]["same"] + assert message["auxiliary_graphs"][refs["auxiliary_graphs"]["same"]][ + "edges" + ] == [refs["edges"]["same"]] + assert reduce_evidence(list(reversed(events)), composition) == report + + +def test_invalid_reference_and_schema_errors_are_explicit(): + events, composition = collision_case() + events[0]["observations"][0]["body_json"]["message"]["knowledge_graph"]["edges"][ + "same" + ]["sources"] = [] + # Changed response identity must be remapped deliberately, never matched heuristically. + assert reduce_evidence(events, composition)["status"] == "invalid" + assert not validate( + { + "message": { + "knowledge_graph": {"nodes": {"X:1": {}}, "edges": {}}, + "results": [], + } + } + )["valid"] + + +def test_beta_path_native_schema_and_unsupported_conversion(): + query = { + "nodes": { + "a": {"categories": ["biolink:Gene"]}, + "b": {"categories": ["biolink:Gene"]}, + }, + "paths": {"p": {"subject": "a", "object": "b"}}, + } + body = { + "knowledge_graph": { + "nodes": { + "X:1": {"categories": ["biolink:Gene"]}, + "X:2": {"categories": ["biolink:Gene"]}, + }, + "edges": { + "e": { + "subject": "X:1", + "object": "X:2", + "predicate": "biolink:related_to", + "sources": [ + { + "resource_id": "infores:fixture", + "resource_role": "primary_knowledge_source", + } + ], + "knowledge_level": "knowledge_assertion", + "agent_type": "manual_agent", + } + }, + }, + "auxiliary_graphs": {"p": {"edges": ["e"]}}, + "results": [ + { + "node_bindings": {"a": {"ids": ["X:1"]}, "b": {"ids": ["X:2"]}}, + "analyses": [ + { + "resource_id": "infores:fixture", + "path_bindings": {"p": {"ids": ["p"]}}, + } + ], + } + ], + } + events = [event(body, query)] + source_id = next(iter(inventory(events)[0])) + composition = { + "closed": True, + "query_graph": query, + "sources": { + source_id: {"nodes": {"a": "a", "b": "b"}, "edges": {}, "paths": {"p": "p"}} + }, + "workflow": { + "schema_version": 1, + "steps": [ + { + "id": "answer", + "op": "literal", + "value": [{"a": "X:1", "b": "X:2", "path": "p"}], + } + ], + "outputs": {"answer": {"$ref": "answer"}}, + }, + "selection": { + "output": "answer", + "resource_id": "infores:tct", + "nodes": {"a": "/a", "b": "/b"}, + "edges": {}, + "paths": {"p": {"source_id": source_id, "path": "/path"}}, + }, + } + report = reduce_evidence(events, composition, "2.0.0-beta2") + assert report["status"] == "valid", report + assert report["validation"]["schema"]["prerelease"] is True + assert report["draft"]["schema_version"] == "2.0.0-beta2" + assert reduce_evidence(events, composition, "1.5.0")["status"] == "invalid" + assert reduce_evidence(events, composition, "2.0.0")["status"] == "invalid" + + +def test_cli_mcp_and_native_evidence_are_equivalent(captured, capsys): + from TCT.interfaces.cli import main + from TCT.interfaces.mcp import mcp + + events, composition, store, run_id = captured + capsys.readouterr() + request = { + "action": "export", + "store": str(store), + "run_id": run_id, + "composition": composition, + } + native = tools.evidence(request) + assert main(["evidence", "--request", json.dumps(request)]) == 0 + assert json.loads(capsys.readouterr().out) == native + + async def call(): + registered = await mcp.get_tool("evidence") + result = await registered.run({"request": request}) + return result.structured_content + + assert asyncio.run(call()) == native + + +def test_pins_have_matching_digests(): + root = Path(__file__).parents[1] / "TCT" / "schemas" + for version, pin in json.loads((root / "manifest.json").read_text()).items(): + assert ( + hashlib.sha256((root / f"trapi-{version}.yaml").read_bytes()).hexdigest() + == pin["sha256"] + ) + + +def test_export_can_be_recorded_in_the_same_run(captured): + events, composition, store, run_id = captured + request = { + "action": "export", + "store": str(store), + "run_id": run_id, + "composition": composition, + } + baseline = tools.evidence(request) + with execution( + {"record": True, "store": str(store), "run_id": run_id, "langfuse": False} + ) as run: + assert tools.evidence(request) == baseline + assert run[0]["status"] == "succeeded" + assert read_run(store, run_id) + + +def test_dependency_order_is_explicit_and_unknown_rank_is_rejected(captured): + events, composition, _, _ = captured + baseline = reduce_evidence(events, composition) + reordered = deepcopy(composition) + reordered["workflow"]["steps"].reverse() + result = reduce_evidence(events, reordered) + assert result["status"] == "valid" + assert result["draft"]["message"] == baseline["draft"]["message"] + assert ( + result["draft"]["x-tct"]["transformations"] + == baseline["draft"]["x-tct"]["transformations"] + ) + composition["rank"] = "best" + assert "ranking" in reduce_evidence(events, composition)["error"] + + +def test_missing_recorded_input_and_invalid_source_report(captured): + events, composition, _, _ = captured + source_event = next(e for e in events if e.get("operation") == "query_KP") + assert ( + "Missing closed input" + in reduce_evidence([e for e in events if e is not source_event], composition)[ + "error" + ] + ) + source_event["status"] = "capture_failed" + assert "did not complete" in reduce_evidence(events, composition)["error"] + + +def test_offline_external_schema_refs_are_rejected_without_fetch(monkeypatch): + monkeypatch.setattr( + requests.sessions.Session, + "request", + lambda *a, **kw: pytest.fail("network validation"), + ) + report = validate({"message": {}, "workflow": [{"id": "lookup"}]}) + assert not report["valid"] + assert ( + "not vendored" in report["errors"][0]["message"] + or "Unresolvable" in report["errors"][0]["message"] + ) + + +def test_sources_from_existing_workflow_evidence_are_reused(captured): + events, _, _, _ = captured + source = next(iter(inventory(events)[0].values())) + record = { + "format_version": 1, + "run_id": "workflow", + "call_id": "workflow", + "status": "succeeded", + "output": { + "schema_version": 1, + "steps": { + "q": { + "op": "query", + "status": "succeeded", + "inputs": {"url": source["url"], "query": source["query"]}, + "transport": {"status_code": 200}, + "response": source["response"], + } + }, + }, + } + extracted, issues, workflows = inventory([record]) + assert set(extracted) <= set(inventory(events)[0]) + assert not issues and len(workflows) == 1 + + +def test_rollup_is_detached_and_failed_workflow_inputs_are_not_selected(captured): + events, composition, _, _ = captured + original_events, original_composition = deepcopy(events), deepcopy(composition) + report = reduce_evidence(events, composition) + report["draft"]["x-tct"]["source_mappings"].clear() + report["draft"]["x-tct"]["recipe"]["steps"].clear() + assert events == original_events and composition == original_composition + source = next(iter(inventory(events)[0].values())) + record = { + "format_version": 1, + "run_id": "failed", + "call_id": "failed", + "status": "succeeded", + "output": { + "schema_version": 1, + "steps": { + "query": { + "op": "query", + "status": "failed", + "inputs": {"url": source["url"], "query": source["query"]}, + "transport": {"status_code": 200}, + "response": source["response"], + } + }, + }, + } + sources, issues, _ = inventory([record]) + assert not next(iter(sources.values()))["eligible"] + assert any(issue["status"] == "workflow_step_failed" for issue in issues) + + +def test_malformed_and_null_provider_body_is_inspectable(): + record = event({"knowledge_graph": None}, {"nodes": {}, "edges": {}}) + assert ( + next(iter(evidence({"events": [record]})["sources"].values()))["node_count"] + == 0 + ) + record["observations"][0]["body_json"] = {"error": "not a TRAPI response"} + assert evidence({"events": [record]})["issues"][0]["status"] == "invalid_trapi_body" diff --git a/tests/test_mcp_contract.py b/tests/test_mcp_contract.py index 4e1843f..8400692 100644 --- a/tests/test_mcp_contract.py +++ b/tests/test_mcp_contract.py @@ -63,6 +63,7 @@ "validate_query_workflow": "(workflow: 'dict[str, Any]') -> 'dict[str, Any]'", "run_query_workflow": "(workflow: 'dict[str, Any]') -> 'dict[str, Any]'", "replay_query_workflow": "(bundle: 'dict[str, Any]', mode: 'str' = 'snapshot') -> 'dict[str, Any]'", + "evidence": "(request: 'dict[str, Any]') -> 'dict[str, Any]'", } @@ -226,6 +227,10 @@ }, ["start", "end"], ), + "evidence": ( + {"request": {"type": "object", "additionalProperties": True}}, + ["request"], + ), } @@ -292,6 +297,7 @@ def test_tools_keep_unstructured_output_contract(): "validate_query_workflow", "run_query_workflow", "replay_query_workflow", + "evidence", }: assert tool.output_schema is not None else: diff --git a/tests/test_toolkit_capture.py b/tests/test_toolkit_capture.py index 4e6113d..3f68429 100644 --- a/tests/test_toolkit_capture.py +++ b/tests/test_toolkit_capture.py @@ -328,7 +328,8 @@ def test_incomplete_requested_capture_surfaces_after_native_operation( def test_working_surface_coverage_and_unsupported_endpoint(): assert {c[0] for c in CASES} == {f.__name__ for f in tools.TOOLS} - { - "trapi_query_endpoint" + "trapi_query_endpoint", + "evidence", } assert len(CASES) == 18 with pytest.raises(TypeError): diff --git a/tests/test_tools.py b/tests/test_tools.py index 252a476..985d6f0 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -28,6 +28,7 @@ "validate_query_workflow", "run_query_workflow", "replay_query_workflow", + "evidence", ] @@ -57,7 +58,7 @@ def find_spec(self, fullname, path=None, target=None): sys.meta_path.insert(0, RejectMcpImports()) from TCT.interfaces.cli import build_parser from TCT.interfaces.tools import TOOLS -assert len(TOOLS) == 19 +assert len(TOOLS) == 20 assert build_parser().prog == "tct" """ diff --git a/uv.lock b/uv.lock index d1cc34f..e07f3fe 100644 --- a/uv.lock +++ b/uv.lock @@ -3427,10 +3427,12 @@ version = "0.2.0" source = { editable = "." } dependencies = [ { name = "igraph" }, + { name = "jsonschema" }, { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "numpy", version = "2.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "pandas" }, { name = "pyvis" }, + { name = "pyyaml" }, { name = "requests" }, { name = "zstandard" }, ] @@ -3467,6 +3469,7 @@ requires-dist = [ { name = "fastmcp", marker = "extra == 'mcp'", specifier = ">=2.12.2" }, { name = "igraph" }, { name = "ipycytoscape", marker = "extra == 'vision'" }, + { name = "jsonschema", specifier = ">=4.18" }, { name = "langfuse", marker = "extra == 'langfuse'", specifier = ">=3.63.0" }, { name = "matplotlib", marker = "extra == 'vision'" }, { name = "networkx", marker = "extra == 'vision'" }, @@ -3474,6 +3477,7 @@ requires-dist = [ { name = "pandas" }, { name = "pyvis" }, { name = "pyvis", marker = "extra == 'vision'" }, + { name = "pyyaml", specifier = ">=6" }, { name = "requests" }, { name = "seaborn", marker = "extra == 'vision'" }, { name = "zstandard" }, From b165ec6d236d1d857926b1ba4ba6bc81ef92addc Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Thu, 1 Oct 2026 10:47:23 -0700 Subject: [PATCH 7/8] Add opt-in response caching and deterministic snapshot replay --- TCT/TCT_pathfinder.py | 8 +- TCT/execution.py | 62 +++- TCT/interfaces/WORKFLOWS.md | 123 ++++++- TCT/journal.py | 49 +-- TCT/name_resolver.py | 10 +- TCT/node_normalizer.py | 8 +- TCT/response_cache.py | 503 +++++++++++++++++++++++++++++ TCT/service_capture.py | 39 ++- TCT/translator_kpinfo.py | 2 +- TCT/translator_metakg.py | 32 +- TCT/translator_query.py | 4 +- TCT/workflow.py | 60 +++- tests/test_execution.py | 2 +- tests/test_response_cache.py | 606 +++++++++++++++++++++++++++++++++++ 14 files changed, 1419 insertions(+), 89 deletions(-) create mode 100644 TCT/response_cache.py create mode 100644 tests/test_response_cache.py diff --git a/TCT/TCT_pathfinder.py b/TCT/TCT_pathfinder.py index 862914c..ad3364b 100644 --- a/TCT/TCT_pathfinder.py +++ b/TCT/TCT_pathfinder.py @@ -388,7 +388,7 @@ def query_aragorn_pathfinder(node1_id:str, node1_category:str, node2_id:str, nod """ aragorn_endpoint = service_url("aragorn") query_current = format_pathfinder_query(node1_id, node1_category, node2_id, node2_category) - response = capture_request(requests.post, aragorn_endpoint, json=query_current) + response = capture_request(requests.post, aragorn_endpoint, json=query_current, cache_kind="trapi", method="post") return response @@ -417,7 +417,7 @@ def query_aragorn_pathfinder_with_constraints(node1_id:str, node1_category:str, object_categories=node2_category, constraints=constraints ) - response = capture_request(requests.post, aragorn_endpoint, json=query_current) + response = capture_request(requests.post, aragorn_endpoint, json=query_current, cache_kind="trapi", method="post") return response @capture_operation() @@ -438,7 +438,7 @@ def query_arax_pathfinder(node1_id:str, node1_category:str, node2_id:str, node2_ """ ARAX_endpoint = service_url("arax") query_current = format_pathfinder_query(node1_id, node1_category, node2_id, node2_category) - response = capture_request(requests.post, ARAX_endpoint, json=query_current) + response = capture_request(requests.post, ARAX_endpoint, json=query_current, cache_kind="trapi", method="post") return response @capture_operation() @@ -466,7 +466,7 @@ def query_arax_pathfinder_with_constraints(node1_id:str, node1_category:str, nod object_categories=node2_category, constraints=constraints ) - response = capture_request(requests.post, ARAX_endpoint, json=query_current) + response = capture_request(requests.post, ARAX_endpoint, json=query_current, cache_kind="trapi", method="post") return response @capture_operation() diff --git a/TCT/execution.py b/TCT/execution.py index f1e3bb3..6816ae5 100644 --- a/TCT/execution.py +++ b/TCT/execution.py @@ -14,6 +14,7 @@ _settings = ContextVar("tct_execution_settings", default=None) _run = ContextVar("tct_execution_run", default=None) _active = ContextVar("tct_execution_call", default=None) +_cache = ContextVar("tct_execution_cache", default=None) logger = logging.getLogger(__name__) @@ -36,6 +37,7 @@ def __init__(self, run_id, journal=None): super().__init__() self.run_id = run_id self.journal = journal + self.cache_events = [] def langfuse_override(): @@ -67,11 +69,13 @@ def validate_execution(config): } if not isinstance(config, dict) or set(config) - allowed: raise ValueError("Unknown execution settings") - for key in ("trapi", "record", "langfuse", "cache"): + for key in ("trapi", "record", "langfuse"): if key in config and type(config[key]) is not bool: raise ValueError(f"{key} must be a boolean") - if config.get("cache"): - raise ValueError("Execution caching is not implemented yet") + if "cache" in config and config["cache"] is not False: + from .response_cache import validate_policy + + validate_policy(config["cache"], partial=True) for key in ("store", "run_id", "step_id"): if key in config and ( not isinstance(config[key], str) or not config[key].strip() @@ -83,24 +87,38 @@ def validate_execution(config): or len(set(config["depends_on"])) != len(config["depends_on"]) ): raise ValueError("depends_on must contain distinct nonempty step IDs") - return dict(config) + return dict(config) | ( + {"cache": deepcopy(config["cache"])} + if isinstance(config.get("cache"), dict) + else {} + ) + + +def _merge_settings(previous, supplied): + settings = previous | supplied + if isinstance(previous.get("cache"), dict) and isinstance( + supplied.get("cache"), dict + ): + settings["cache"] = previous["cache"] | supplied["cache"] + return settings @contextmanager def execution(config=None): - """Enable independent tracing, TRAPI capture, or durable recording. + """Enable independent tracing, TRAPI capture, recording, or response caching. Unspecified settings inherit. False explicitly disables a feature. Durable recording requires a store; Python can generate a run_id on first activation. """ supplied = validate_execution({} if config is None else config) previous = _settings.get() or {} - settings = previous | supplied + settings = _merge_settings(previous, supplied) if settings.get("record") and not settings.get("store"): raise ValueError("record=true requires an explicit store") if settings.get("depends_on") and not settings.get("record"): raise ValueError("depends_on requires durable recording") - active = bool(settings.get("trapi") or settings.get("record")) + cache_enabled = settings.get("cache", False) is not False + active = bool(settings.get("trapi") or settings.get("record") or cache_enabled) run = _run.get() if active else None changed_run = any( key in supplied and supplied[key] != previous.get(key) @@ -122,11 +140,19 @@ def execution(config=None): ) from error run = ExecutionRun(run_id, journal) settings["run_id"] = run_id + cache = None + if cache_enabled: + from .response_cache import CacheSession, validate_policy + + policy = validate_policy(settings["cache"]) + cache = CacheSession(settings, policy, run, previous=_cache.get()) setting_token = _settings.set(settings) run_token = _run.set(run) + cache_token = _cache.set(cache) try: yield run finally: + _cache.reset(cache_token) _run.reset(run_token) _settings.reset(setting_token) @@ -134,9 +160,12 @@ def execution(config=None): @contextmanager def interface_execution(config=None): supplied = validate_execution({} if config is None else config) - merged = (_settings.get() or {}) | supplied - if merged.get("record") and not merged.get("run_id"): - raise ValueError("CLI/MCP durable recording requires an explicit run_id") + merged = _merge_settings(_settings.get() or {}, supplied) + persistent = merged.get("record") or ( + merged.get("cache", False) is not False and merged.get("store") + ) + if persistent and not merged.get("run_id"): + raise ValueError("CLI/MCP persistent execution requires an explicit run_id") with execution(supplied) as run: yield run @@ -163,7 +192,8 @@ def execute(function, args, kwargs, *, interface=None, kind=None): """One execution boundary shared by interface wrappers and decorators.""" run = _run.get() settings = _settings.get() or {} - if interface is None and run is None: + capturing = run is not None and (settings.get("trapi") or settings.get("record")) + if interface is None and not capturing: enabled = settings.get("langfuse") if enabled is False or ( enabled is None @@ -173,7 +203,7 @@ def execute(function, args, kwargs, *, interface=None, kind=None): return function(*args, **kwargs) record = None token = None - if run is not None: + if capturing: parent = _active.get() same_run = parent and parent["run_id"] == run.run_id call_id = uuid4().hex @@ -284,7 +314,13 @@ def captured(*args, **kwargs): def observation_target(): """Internal capture sink; disabled execution must not inspect responses.""" - return _active.get() if _run.get() is not None else None + settings = _settings.get() or {} + return _active.get() if settings.get("trapi") or settings.get("record") else None + + +def cache_session(): + """Internal HTTP boundary; reading it does not import hashing or persistence.""" + return _cache.get() def observe_evidence(kind, **data): diff --git a/TCT/interfaces/WORKFLOWS.md b/TCT/interfaces/WORKFLOWS.md index eaacdba..c63269d 100644 --- a/TCT/interfaces/WORKFLOWS.md +++ b/TCT/interfaces/WORKFLOWS.md @@ -210,8 +210,8 @@ within the same run. Parent-call links describe nesting, not data dependencies. inherit from the enclosing Python context, and Langfuse otherwise uses the existing `TCT_LANGFUSE_ENABLED` setting. Explicit false disables the corresponding feature. A store path alone does not activate recording. Recording requires an explicit -store. Unknown settings and invalid types fail before execution. `cache: false` -is accepted; true is rejected until the caching phase is implemented. +store. Unknown settings and invalid types fail before execution. `cache` is a +fourth independent feature, disabled by default; see Phase 5 below. With all features disabled, no new persistence, evidence hashing, copying, or SDK loading occurs. Existing toolkit resource caching is unchanged. Langfuse failures @@ -361,7 +361,7 @@ Conflicting content for the same run/call identity is rejected. The semantic has excludes capture IDs and workflow timestamps. It includes the explicit recipe, source mappings, selected answers, retained evidence, filter decisions and row lineage. This hash identifies a Phase 4 artifact; it is not the execution/cache -identity planned for Phase 5. Reordering equivalent recipe definitions can still +identity implemented in Phase 5. Reordering equivalent recipe definitions can still change this hash; no semantic query equivalences are assumed. Provider-local edge and auxiliary-graph IDs are deterministically namespaced. @@ -393,7 +393,7 @@ validation never fetches them. Full response bodies remain in the execution journal even if no supported rollup can be produced. The endpoint remains opt-in: capture alone does not automatically compose answers, -select mappings or enable caching. Phase 5 remains a separate trigger. +select mappings or enable caching. Phase 4 acceptance: **260 offline tests passed**. The original single-NLQ service and native-output fixtures produce one validated, traceable TRAPI 1.5.0 answer. @@ -403,3 +403,118 @@ retained provider analyses/support, dependency order, failed/missing inputs, journal checksums, same-run recording, schema failures, beta2 native paths and CLI/MCP/Python output parity. The built wheel includes both pinned schemas, manifest and license. No live provider or paid model calls were made. + +### Phase 5: response caching and execution epochs + +The same optional execution setting controls response caching. No additional CLI +flags, MCP tools, or native result envelopes are introduced. `{"cache": true}` +enables **within-run** reuse in memory. `cache: false` explicitly disables it. +Nothing is persisted unless `store` is supplied. For reuse across separate CLI/MCP +calls, supply both `store` and the same `run_id`; these interfaces reject persistent +execution without a run ID. Python can generate an ID in its enclosing context. +Existing resource caching is unchanged, and a resource-cache hit does not imply +that fresh service evidence was observed. Clear that cache explicitly when an +experiment requires a new service trajectory. + +Cross-run reuse is opt-in and requires an explicit authorization partition: + +```json +{ + "store": "./evidence", + "run_id": "experiment-002", + "cache": { + "scope": "cross_run", + "mode": "use", + "authorization_partition": "public-translator-ci", + "dataset_versions": {"provider-a": "release-2026-09"}, + "epoch": "query-policy-1" + } +} +``` + +Only include dataset versions you actually know. These fields declare dependencies; +TCT does not make extra calls to discover versions. All declared dependency versions +participate in identity, so changing an upstream normalizer/provider version, epoch, +authorization partition, runtime environment, endpoint, or effective request causes +a miss. The partition must change when access scope or implicit credentials change. +Explicit credentials also participate in hashing; request bodies and credentials +are not copied into the cache index. Execution recording remains a separate opt-in +feature and retains its existing request observations. + +Cross-run lifetimes default to **24 hours for normalization/name resolution, one +hour for capabilities, and 15 minutes for TRAPI**. Override individual durations +with `cache.ttl`, in seconds (for example `{"trapi": 60}`). These are freshness +policies, not guarantees that a live service is deterministic. Unknown service kinds +are never reused across runs. Entries already used by a run stay pinned for that +run; expiry is checked when a new run first looks up a shared entry. Warm hits do +not extend shared expiry. There is no automatic disk eviction in this phase. + +Modes are `use`, `refresh`, `bypass`, and `replay`. Refresh issues a new service +request and records a new observation; prior immutable snapshots remain available. +Bypass issues a request without reading or populating reusable entries, but still +records its snapshot when storage is requested. HTTP errors, invalid/non-JSON +responses, reported application errors, and transport failures never populate a +successful cache entry. A failed refresh leaves a previously successful entry intact. +Legacy functions retain their existing return behavior (including `None`); cache +receipts distinguish failure from a successful empty response. + +Three hashes serve distinct purposes: + +- `original_request_hash` identifies the request before toolkit reformulation. +- `effective_execution_hash` includes the actual service request, its settings, + dependency versions, auth partition, epoch, and cache/canonicalizer/adapter versions. +- `response_content_hash` identifies the exact response bytes, independent of the request. + +Canonicalization sorts JSON object keys only. It preserves arrays, query node IDs, +predicate order, URLs and literal values. Semantic rewrites are not assumed to be +equivalent. The existing provider predicate-pruning operation records +`tct.optimize_query_json.v1` when it changes the original request. Mode, TTL and +telemetry settings do not alter service-response identity. Unsupported non-JSON +request options fail explicitly when caching is requested. Adapter behavior changes +must update the corresponding identity version. + +To replay a saved run, use the original cache identity settings with +`"mode": "replay", "snapshot_run": "experiment-002"` and a **different** `run_id`. +Replay performs no service requests. Missing, exhausted or damaged snapshots fail +locally; there is no fallback to live HTTP. Occurrences of the same effective request +are consumed in their recorded order; independent requests can be interleaved. +`replay.sqlite3` stores transactional cursors so continuation works across separate +CLI processes and MCP calls. Use a fresh replay run ID to start again. The source +cache index and artifacts remain unchanged. Transport exception classes can be +replayed; original exception objects, sockets, elapsed timings and prepared-request +objects are not reconstructed. Streaming workflow responses keep their 8 MiB limit; +an incomplete response cannot be replayed as a successful snapshot. + +`cache.sqlite3` indexes immutable snapshots in the same content-addressed artifact +store as optional recording. Cache-only execution creates no call journal and +does not enable Langfuse or TRAPI capture. Python exposes receipts as +`run.cache_events`; when capture is enabled, service observations and workflow +steps carry the corresponding receipts. The `evidence` tool continues to export +the validated TRAPI rollup from captured evidence without adding cache envelopes +to ordinary tool results. Requested storage failures surface completed-call status. + +Nested cache policy objects inherit their outer fields; supplied fields replace +those fields (including dependency and TTL maps). Explicit false disables caching +until the nested context exits. Workers receive the caller's execution context. +Simultaneous cold misses may issue multiple requests; this is not an exactly-once +HTTP protocol. Replayed calls reserve an occurrence before returning it; a process +crash after reservation requires inspection or a new replay run ID. + +This boundary covers the instrumented normalization, discovery, provider and +pathfinder helpers plus workflow queries. Uninstrumented legacy functions, bulk +graph downloads and the unsupported endpoint placeholder are outside its scope. +The separate benchmark worktree labels the same NLQ as cold, warm, refreshed or +replayed. Its warmup is a separate fixed trajectory; it never supplies an agent's +answer or conversation history. Phase 6 comparative evaluation remains a separate +user trigger. + +Phase 5 acceptance (2026-10-01): **306 offline regression tests and 33 benchmark +tests passed**. The 46 cache tests cover default isolation, nested policies, +independent capture, explicit CLI/MCP settings, worker propagation, expiration, +dependency/authorization invalidation, canonicalization, refresh/bypass, failures, +checksums, persistent replay cursors and bounded workflow responses. The same +single NLQ retains its native goldens and Phase 4 rollup hash in all four cache +scenarios. The benchmark's fixed trajectory uses 4/0/4/0 evaluation service calls +for cold/warm/refreshed/replayed samples across Python, CLI and registered MCP. +The sampled adapter test allows valid wording variation and retains its budget +and resume controls. No live services, real models or paid providers were used. diff --git a/TCT/journal.py b/TCT/journal.py index 6fdc07d..95c369c 100644 --- a/TCT/journal.py +++ b/TCT/journal.py @@ -9,31 +9,13 @@ import tempfile -class Journal: +class ArtifactStore: + """Immutable bytes shared by recording and caching, enabled independently.""" + def __init__(self, store): self.root = Path(store).expanduser().resolve() self.artifacts = self.root / "artifacts" self.artifacts.mkdir(parents=True, exist_ok=True) - with self.connect() as db: - db.executescript(""" - CREATE TABLE IF NOT EXISTS runs ( - run_id TEXT PRIMARY KEY, format_version INTEGER NOT NULL); - CREATE TABLE IF NOT EXISTS calls ( - run_id TEXT NOT NULL, step_id TEXT NOT NULL, - call_id TEXT NOT NULL, parent_call_id TEXT, - operation TEXT NOT NULL, status TEXT NOT NULL, - input_hash TEXT NOT NULL, event_hash TEXT, - PRIMARY KEY(run_id, step_id)); - """) - - @contextmanager - def connect(self): - connection = sqlite3.connect(self.root / "journal.sqlite3", timeout=30) - try: - with connection: - yield connection - finally: - connection.close() def artifact(self, value): from .interfaces.serialization import to_jsonable @@ -68,6 +50,31 @@ def artifact(self, value): os.unlink(temporary) return digest + +class Journal(ArtifactStore): + def __init__(self, store): + super().__init__(store) + with self.connect() as db: + db.executescript(""" + CREATE TABLE IF NOT EXISTS runs ( + run_id TEXT PRIMARY KEY, format_version INTEGER NOT NULL); + CREATE TABLE IF NOT EXISTS calls ( + run_id TEXT NOT NULL, step_id TEXT NOT NULL, + call_id TEXT NOT NULL, parent_call_id TEXT, + operation TEXT NOT NULL, status TEXT NOT NULL, + input_hash TEXT NOT NULL, event_hash TEXT, + PRIMARY KEY(run_id, step_id)); + """) + + @contextmanager + def connect(self): + connection = sqlite3.connect(self.root / "journal.sqlite3", timeout=30) + try: + with connection: + yield connection + finally: + connection.close() + def begin(self, record): input_hash = self.artifact(record["inputs"]) with self.connect() as db: diff --git a/TCT/name_resolver.py b/TCT/name_resolver.py index 07bdabd..c1ec1b6 100644 --- a/TCT/name_resolver.py +++ b/TCT/name_resolver.py @@ -18,7 +18,7 @@ def status() -> str: """ Returns the status of the Name Resolver API. """ - response = capture_request(requests.get, urllib.parse.urljoin(service_url("name_resolver"), "status")) + response = capture_request(requests.get, urllib.parse.urljoin(service_url("name_resolver"), "status"), cache_kind="normalization", method="get") response.raise_for_status() return response.json() @@ -64,7 +64,7 @@ def lookup(query: str, return_top_response:bool=True, return_synonyms:bool=False # set autocomplete to be false by default if 'autocomplete' not in kwargs: kwargs['autocomplete'] = False - response = capture_request(requests.get, path, params={'string': query, 'limit': limit, **kwargs}) + response = capture_request(requests.get, path, params={'string': query, 'limit': limit, **kwargs}, cache_kind="normalization", method="get") if response.status_code == 200: result = response.json() if len(result) == 0: @@ -100,7 +100,7 @@ def synonyms(query: str|list, **kwargs) -> dict[str, TranslatorNode]: """ path = urllib.parse.urljoin(service_url("name_resolver"), 'synonyms') # set autocomplete to be false by default - response = capture_request(requests.get, path, params={'preferred_curies': query, **kwargs}) + response = capture_request(requests.get, path, params={'preferred_curies': query, **kwargs}, cache_kind="normalization", method="get") if response.status_code == 200: result = response.json() if len(result) == 0: @@ -162,7 +162,7 @@ def batch_lookup(strings:list[str], size: int=25, return_top_response:bool=True, "strings": chunk, **kwargs } - response = capture_request(requests.post, path, json = payload) + response = capture_request(requests.post, path, json = payload, cache_kind="normalization", method="post") if response.status_code == 200: result = response.json() if(len(result) == 0): @@ -207,7 +207,7 @@ def batch_synonyms(strings:list[str], size:int=50, **kwargs) -> dict[str, Transl curies = {} for chunk in chunks: # set autocomplete to be false by default - response = capture_request(requests.post, path, json={'preferred_curies': chunk, **kwargs}) + response = capture_request(requests.post, path, json={'preferred_curies': chunk, **kwargs}, cache_kind="normalization", method="post") if response.status_code == 200: result = response.json() if len(result) == 0: diff --git a/TCT/node_normalizer.py b/TCT/node_normalizer.py index 04e83f2..6b13d2b 100644 --- a/TCT/node_normalizer.py +++ b/TCT/node_normalizer.py @@ -18,7 +18,7 @@ def status(): """ Returns the status of the Node Normalizer API. """ - response = capture_request(requests.get, urllib.parse.urljoin(service_url("node_normalizer"), "status")) + response = capture_request(requests.get, urllib.parse.urljoin(service_url("node_normalizer"), "status"), cache_kind="normalization", method="get") response.raise_for_status() return response.json() @@ -64,9 +64,9 @@ def get_normalized_nodes(query: str | list[str], json_query = query if len(json_query) == 0: return {} - response = capture_request(requests.post, path, json={'curies': json_query, **kwargs}) + response = capture_request(requests.post, path, json={'curies': json_query, **kwargs}, cache_kind="normalization", method="post") else: - response = capture_request(requests.get, path, params={'curie': query, **kwargs}) + response = capture_request(requests.get, path, params={'curie': query, **kwargs}, cache_kind="normalization", method="get") if response.status_code == 200: result = response.json() normalized_dict = {} @@ -173,7 +173,7 @@ def ID_convert_to_preferred_name_nodeNormalizer(id_list: list[str]) -> dict[str, "description": False, # Change to True if you want descriptions from any identifiers we know about. "conflate": NODENORM_GENE_PROTEIN_CONFLATION, "drug_chemical_conflate": NODENORM_DRUG_CHEMICAL_CONFLATION, - }) + }, cache_kind="normalization", method="post") if not response.ok: raise RuntimeError("Error: NodeNorm request failed with status code " + str(response.status_code)) diff --git a/TCT/response_cache.py b/TCT/response_cache.py new file mode 100644 index 0000000..7aca028 --- /dev/null +++ b/TCT/response_cache.py @@ -0,0 +1,503 @@ +"""Opt-in HTTP response reuse. Imported only when execution caching is enabled. + +Only JSON object key order is canonicalized. Arrays, query IDs, predicate order, +URLs, and literal values retain their meaning. Snapshots are not answer proofs. +""" + +from base64 import b64decode, b64encode +from collections import defaultdict +from contextlib import contextmanager +import hashlib +import json +import math +from pathlib import Path +import sqlite3 +from threading import RLock +import time + +import requests + +from .config import get_runtime_config +from .execution import ExecutionRecordingError +from .journal import ArtifactStore + +POLICY_VERSION = "tct.http-cache.v1" +CANONICALIZER_VERSION = "json-object-order.v1" +DEFAULT_TTL = {"normalization": 86400, "capabilities": 3600, "trapi": 900} + + +class CacheError(ExecutionRecordingError): + """Requested caching failed; must not be mistaken for an empty answer.""" + + +class CacheReplayError(CacheError): + """A snapshot was absent, exhausted, or damaged. Never falls back to HTTP.""" + + +def canonical_bytes(value): + def require_json(item): + if isinstance(item, dict): + if any(not isinstance(key, str) for key in item): + raise TypeError("Cache identity requires string JSON object keys") + for child in item.values(): + require_json(child) + elif isinstance(item, list): + for child in item: + require_json(child) + elif item is not None and type(item) not in (str, bool, int, float): + raise TypeError("Cache identity requires JSON-compatible request options") + + require_json(value) + return json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + allow_nan=False, + ).encode("utf-8") + + +def digest(value): + return hashlib.sha256(canonical_bytes(value)).hexdigest() + + +def validate_policy(value, *, partial=False): + if value is True: + value = {} + allowed = { + "mode", + "scope", + "authorization_partition", + "dataset_versions", + "epoch", + "ttl", + "snapshot_run", + } + if not isinstance(value, dict) or set(value) - allowed: + raise ValueError("cache must be a boolean or a cache policy object") + policy = { + "mode": "use", + "scope": "run", + "dataset_versions": {}, + "epoch": "default", + "ttl": DEFAULT_TTL.copy(), + } | value + if policy["mode"] not in ("use", "refresh", "bypass", "replay"): + raise ValueError("cache.mode must be use, refresh, bypass, or replay") + if policy["scope"] not in ("run", "cross_run"): + raise ValueError("cache.scope must be run or cross_run") + for name in ("epoch", "authorization_partition", "snapshot_run"): + if name in policy and ( + not isinstance(policy[name], str) or not policy[name].strip() + ): + raise ValueError(f"cache.{name} must be a nonempty string") + versions = policy["dataset_versions"] + if not isinstance(versions, dict) or any( + not isinstance(k, str) or not k or not isinstance(v, str) or not v + for k, v in versions.items() + ): + raise ValueError("cache.dataset_versions must map dependency names to versions") + ttl = policy["ttl"] + if ( + not isinstance(ttl, dict) + or set(ttl) - set(DEFAULT_TTL) + or any( + type(v) not in (int, float) or not math.isfinite(v) or v < 0 + for v in ttl.values() + ) + ): + raise ValueError( + "cache.ttl must contain finite, nonnegative seconds by service kind" + ) + policy["ttl"] = DEFAULT_TTL | ttl + if not partial and policy["mode"] == "replay" and not policy.get("snapshot_run"): + raise ValueError("cache replay requires snapshot_run") + if not partial and policy["mode"] != "replay" and "snapshot_run" in policy: + raise ValueError("snapshot_run is only valid in replay mode") + if ( + not partial + and policy["scope"] == "cross_run" + and not policy.get("authorization_partition") + ): + raise ValueError( + "cross-run caching requires an explicit authorization_partition" + ) + return json.loads(canonical_bytes(policy)) + + +class CacheStore(ArtifactStore): + def __init__(self, root, *, readonly=False): + self.readonly = readonly + if readonly: + self.root = Path(root).expanduser().resolve() + self.artifacts = self.root / "artifacts" + else: + super().__init__(root) + with self.connect() as db: + db.executescript(""" + CREATE TABLE IF NOT EXISTS entries ( + namespace TEXT NOT NULL, key TEXT NOT NULL, + artifact TEXT NOT NULL, expires REAL, + PRIMARY KEY(namespace, key)); + CREATE TABLE IF NOT EXISTS history ( + sequence INTEGER PRIMARY KEY AUTOINCREMENT, + run_id TEXT NOT NULL, key TEXT NOT NULL, + artifact TEXT, receipt TEXT NOT NULL); + CREATE INDEX IF NOT EXISTS history_run ON history(run_id, sequence); + """) + + @contextmanager + def connect(self): + path = self.root / "cache.sqlite3" + db = ( + sqlite3.connect(path.as_uri() + "?mode=ro", uri=True, timeout=30) + if self.readonly + else sqlite3.connect(path, timeout=30) + ) + try: + with db: + yield db + finally: + db.close() + + def read(self, artifact): + if ( + not isinstance(artifact, str) + or len(artifact) != 64 + or any(c not in "0123456789abcdef" for c in artifact) + ): + raise ValueError("Invalid cache artifact digest") + raw = (self.artifacts / artifact).read_bytes() + if hashlib.sha256(raw).hexdigest() != artifact: + raise ValueError("Cache artifact checksum mismatch") + return json.loads(raw) + + def lookup(self, namespace, key, now): + with self.connect() as db: + row = db.execute( + "SELECT artifact, expires FROM entries WHERE namespace=? AND key=?", + (namespace, key), + ).fetchone() + if row and (row[1] is None or row[1] > now): + return self.read(row[0]) + return None + + def save(self, run_id, key, snapshot, receipt, *, pin, shared_expiry): + artifact = self.artifact(snapshot) if snapshot is not None else None + receipt["snapshot_hash"] = artifact + with self.connect() as db: + if pin: + db.execute( + "INSERT OR REPLACE INTO entries VALUES (?, ?, ?, NULL)", + ("run:" + run_id, key, artifact), + ) + if shared_expiry is not None: + db.execute( + "INSERT OR REPLACE INTO entries VALUES ('shared', ?, ?, ?)", + (key, artifact, shared_expiry), + ) + db.execute( + "INSERT INTO history(run_id, key, artifact, receipt) VALUES (?, ?, ?, ?)", + (run_id, key, artifact, canonical_bytes(receipt).decode()), + ) + + def history(self, run_id): + with self.connect() as db: + return db.execute( + "SELECT key, artifact FROM history WHERE run_id=? ORDER BY sequence", + (run_id,), + ).fetchall() + + def take_replay(self, run_id, source_run, key, artifacts): + # Keep the source index read-only. A separate transactional cursor makes + # separate CLI processes/MCP calls consume the same replay deterministically. + db = sqlite3.connect(self.root / "replay.sqlite3", timeout=30) + try: + with db: + db.execute( + "CREATE TABLE IF NOT EXISTS cursors (run_id TEXT, source_run TEXT, key TEXT, consumed INTEGER NOT NULL, PRIMARY KEY(run_id, source_run, key))" + ) + db.execute("BEGIN IMMEDIATE") + row = db.execute( + "SELECT consumed FROM cursors WHERE run_id=? AND source_run=? AND key=?", + (run_id, source_run, key), + ).fetchone() + index = row[0] if row else 0 + if index >= len(artifacts): + raise ValueError("No remaining snapshot for this effective request") + artifact = artifacts[index] + if artifact is None: + raise ValueError( + "Original attempt did not capture a complete response" + ) + snapshot = self.read(artifact) + db.execute( + "INSERT OR REPLACE INTO cursors VALUES (?, ?, ?, ?)", + (run_id, source_run, key, index + 1), + ) + return snapshot + finally: + db.close() + + +def _response(snapshot): + if "transport_error" in snapshot: + name = snapshot["transport_error"] + error_type = getattr(requests.exceptions, name, requests.RequestException) + if not isinstance(error_type, type) or not issubclass( + error_type, requests.RequestException + ): + error_type = requests.RequestException + raise error_type(f"Replayed transport failure: {name}") + response = requests.Response() + response.status_code = snapshot["status_code"] + response._content = b64decode(snapshot["body_base64"], validate=True) + response._content_consumed = True + response.headers.update(snapshot["headers"]) + response.url = snapshot["url"] + response.reason = snapshot["reason"] + response.encoding = snapshot["encoding"] + return response + + +def _cacheable(snapshot, kind): + if "transport_error" in snapshot or snapshot["status_code"] != 200: + return False + if any(k.lower() == "set-cookie" for k in snapshot["headers"]): + return False + try: + payload = json.loads(b64decode(snapshot["body_base64"])) + canonical_bytes(payload) # Non-finite values are not successful evidence. + except (ValueError, UnicodeError): + return False + if not isinstance(payload, (dict, list)): + return False + if isinstance(payload, dict) and ( + payload.get("error") + or payload.get("errors") + or payload.get("status") not in (None, "Success", "OK") + ): + return False + if kind == "trapi": + if not isinstance(payload, dict): + return False + message = payload.get("message") + kg = message.get("knowledge_graph") if isinstance(message, dict) else None + return ( + isinstance(kg, dict) + and isinstance(kg.get("nodes"), dict) + and isinstance(kg.get("edges"), dict) + ) + return True + + +class CacheSession: + def __init__(self, settings, policy, run, *, previous=None): + self.policy, self.run = policy, run + self.root = settings.get("store") + if ( + policy["scope"] == "cross_run" or policy["mode"] == "replay" + ) and not self.root: + raise ValueError( + "cross-run caching and snapshot replay require an explicit store" + ) + if policy["mode"] == "replay" and policy["snapshot_run"] == run.run_id: + raise ValueError("Replay requires a run_id distinct from snapshot_run") + same = ( + previous is not None and previous.run is run and previous.root == self.root + ) + self.memory = previous.memory if same else {} + self.lock = previous.lock if same else RLock() + self.replay = ( + previous.replay + if same + and previous.policy.get("snapshot_run") == policy.get("snapshot_run") + else None + ) + self.store = None + try: + if self.root: + self.store = CacheStore(self.root, readonly=policy["mode"] == "replay") + if policy["mode"] == "replay" and self.replay is None: + self.replay = defaultdict(list) + for key, artifact in self.store.history(policy["snapshot_run"]): + self.replay[key].append(artifact) + if not self.replay: + raise ValueError("Snapshot run has no service observations") + except Exception as error: + raise CacheError( + "Cannot initialize cache", operation_completed=False, run_id=run.run_id + ) from error + + def _identity(self, method, url, arguments, kind, original, reformulation): + # Credentials participate in the digest, never in the stored manifest. + # The explicit partition identifies the caller's auth/access epoch, + # including implicit credentials (netrc, proxy, etc.). + auth = ( + arguments.get("auth") + or arguments.get("cookies") + or any( + k.lower() in ("authorization", "cookie", "proxy-authorization") + for k in arguments.get("headers", {}) + ) + ) + if auth and not self.policy.get("authorization_partition"): + raise ValueError("Authenticated caching requires authorization_partition") + runtime = get_runtime_config() + request = {"method": method.upper(), "url": url, "arguments": arguments} + original_request = ( + request | {"arguments": original} if original is not None else request + ) + original_hash = digest(original_request) + key = digest( + { + "request": request, + "service_kind": kind, + "authorization_partition": self.policy.get( + "authorization_partition", "within-run" + ), + "dataset_versions": self.policy["dataset_versions"], + "epoch": self.policy["epoch"], + "runtime": { + "environment": runtime.environment, + "overrides": dict(runtime.overrides), + }, + "policy_version": POLICY_VERSION, + "canonicalizer_version": CANONICALIZER_VERSION, + "adapter_version": "tct.http-response.v1", + } + ) + changed = original_hash != digest(request) + if changed and not reformulation: + raise ValueError( + "A changed original request requires a reformulation identifier" + ) + return key, { + "original_request_hash": original_hash, + "effective_execution_hash": key, + "response_content_hash": None, + "mode": self.policy["mode"], + "reformulation": reformulation if changed else None, + "policy_version": POLICY_VERSION, + "canonicalizer_version": CANONICALIZER_VERSION, + } + + def _emit(self, key, snapshot, receipt, *, pin=False, shared=False): + receipt["response_content_hash"] = ( + snapshot.get("response_content_hash") if snapshot else None + ) + receipt["snapshot_hash"] = digest(snapshot) if snapshot is not None else None + try: + with self.lock: + if self.store and self.policy["mode"] != "replay": + expiry = ( + time.time() + self.policy["ttl"][receipt["service_kind"]] + if shared + else None + ) + self.store.save( + self.run.run_id, + key, + snapshot, + receipt, + pin=pin, + shared_expiry=expiry, + ) + if pin: + self.memory[key] = snapshot + self.run.cache_events.append(dict(receipt)) + except Exception as error: + raise CacheError( + "Cannot persist cache observation", + operation_completed=True, + run_id=self.run.run_id, + ) from error + + def request( + self, + request, + url, + arguments, + *, + method, + kind, + original=None, + reformulation=None, + consume=None, + receipt=None, + ): + receipt = receipt if receipt is not None else {} + try: + key, identity = self._identity( + method, url, arguments, kind, original, reformulation + ) + receipt.update(identity, service_kind=kind) + snapshot = None + with self.lock: + if self.policy["mode"] == "replay": + snapshot = self.store.take_replay( + self.run.run_id, + self.policy["snapshot_run"], + key, + self.replay[key], + ) + elif self.policy["mode"] == "use": + snapshot = self.memory.get(key) + if snapshot is None and self.store: + snapshot = self.store.lookup( + "run:" + self.run.run_id, key, time.time() + ) + if ( + snapshot is None + and self.policy["scope"] == "cross_run" + and kind in DEFAULT_TTL + ): + snapshot = self.store.lookup("shared", key, time.time()) + except Exception as error: + cls = CacheReplayError if self.policy["mode"] == "replay" else CacheError + raise cls( + str(error), operation_completed=False, run_id=self.run.run_id + ) from error + if snapshot is not None: + replay = self.policy["mode"] == "replay" + receipt["outcome"] = "replayed" if replay else "warm" + self._emit(key, snapshot, receipt, pin=not replay) + return _response(snapshot) + receipt["outcome"] = { + "use": "cold", + "refresh": "refreshed", + "bypass": "bypassed", + }[self.policy["mode"]] + try: + response = request(url, **arguments) + except requests.RequestException as error: + snapshot = {"transport_error": type(error).__name__} + receipt["status"] = "transport_error" + try: + self._emit(key, snapshot, receipt) + except CacheError as storage_error: + error.add_note(str(storage_error)) + raise + try: + body = consume(response) if consume is not None else response.content + if not isinstance(body, bytes): + raise TypeError("Caching requires a buffered byte response") + snapshot = { + "status_code": response.status_code, + "body_base64": b64encode(body).decode(), + "headers": dict(response.headers), + "url": response.url or url, + "encoding": response.encoding, + "reason": response.reason, + "response_content_hash": hashlib.sha256(body).hexdigest(), + } + except Exception: + receipt["status"] = "incomplete_snapshot" + self._emit(key, None, receipt) + raise + eligible = _cacheable(snapshot, kind) + receipt["status"] = "cacheable" if eligible else "not_cacheable" + pin = eligible and self.policy["mode"] != "bypass" + shared = pin and self.policy["scope"] == "cross_run" and kind in DEFAULT_TTL + self._emit(key, snapshot, receipt, pin=pin, shared=shared) + return response diff --git a/TCT/service_capture.py b/TCT/service_capture.py index 9d0459a..4e1cb5f 100644 --- a/TCT/service_capture.py +++ b/TCT/service_capture.py @@ -8,25 +8,54 @@ from copy import deepcopy import json -from .execution import observation_target +from .execution import cache_session, observation_target -def capture_request(request, /, url, **kwargs): +def capture_request( + request, + /, + url, + *, + cache_kind="other", + method=None, + original_arguments=None, + reformulation=None, + **kwargs, +): target = observation_target() + cache = cache_session() + receipt = {} + + def perform(): + if cache is None: + return request(url, **kwargs) + return cache.request( + request, + url, + kwargs, + method=method or request.__name__, + kind=cache_kind, + original=original_arguments, + reformulation=reformulation, + receipt=receipt, + ) + if target is None: - return request(url, **kwargs) + return perform() observation = {"kind": "service", "status": "pending"} + if cache is not None: + observation["cache"] = receipt target.setdefault("observations", []).append(observation) try: observation["request"] = { - "method": getattr(request, "__name__", "unknown"), + "method": method or getattr(request, "__name__", "unknown"), "url": url, "arguments": deepcopy(kwargs), } except Exception as error: target.setdefault("observation_errors", []).append(type(error).__name__) try: - response = request(url, **kwargs) + response = perform() except BaseException as error: observation.update(status="transport_error", error_type=type(error).__name__) raise diff --git a/TCT/translator_kpinfo.py b/TCT/translator_kpinfo.py index f9b2b49..31e752d 100644 --- a/TCT/translator_kpinfo.py +++ b/TCT/translator_kpinfo.py @@ -48,7 +48,7 @@ def get_translator_kp_info() -> tuple[pd.DataFrame, dict[str, str]]: """ # Get x-bte smartapi specs url = service_url("smartapi_registry") - response = capture_request(requests.get, url) + response = capture_request(requests.get, url, cache_kind="capabilities", method="get") try: response.raise_for_status() except Exception: diff --git a/TCT/translator_metakg.py b/TCT/translator_metakg.py index 1c638aa..dcf6142 100644 --- a/TCT/translator_metakg.py +++ b/TCT/translator_metakg.py @@ -75,15 +75,15 @@ def get_KP_metadata(APInames:dict[str, str], use_new_url=True) -> pd.DataFrame: for KP in APInames.keys(): json_text ={} if KP == "RTX KG2 - TRAPI 1.5.0": - text =capture_request(requests.get, "https://smart-api.info/api/metakg/consolidated?size=20&q=%28api.x-translator.component%3AKP+AND+api.name%3ARTX+KG2+%5C-+TRAPI+1%5C.4%5C.0%29").text # This works for the previous version + text =capture_request(requests.get, "https://smart-api.info/api/metakg/consolidated?size=20&q=%28api.x-translator.component%3AKP+AND+api.name%3ARTX+KG2+%5C-+TRAPI+1%5C.4%5C.0%29", cache_kind="capabilities", method="get").text # This works for the previous version json_text = json.loads(text) else: - text = capture_request(requests.get, find_link(KP, use_new_url=use_new_url)).text + text = capture_request(requests.get, find_link(KP, use_new_url=use_new_url), cache_kind="capabilities", method="get").text json_text = json.loads(text) if 'hits' not in json_text: if use_new_url: #print(KP, '- no hits found in new metakg URL, trying old URL pattern') - text = capture_request(requests.get, find_link(KP, use_new_url=False)).text + text = capture_request(requests.get, find_link(KP, use_new_url=False), cache_kind="capabilities", method="get").text json_text = json.loads(text) else: print(KP, '- no hits found') @@ -176,7 +176,7 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s url = CI_URL_PREFIX + 'BigGIM_DrugResponse_PerformancePhase/meta_knowledge_graph' url_dev = DEV_URL_PREFIX + 'BigGIM_DrugResponse_PerformancePhase/meta_knowledge_graph' try: - response = capture_request(requests.get, url, timeout=5) + response = capture_request(requests.get, url, timeout=5, cache_kind="capabilities", method="get") if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): @@ -187,7 +187,7 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s data["edges"][i]['object']) elif response.status_code == 404: # Try the dev URL if the CI URL returns a 404 - response_dev = capture_request(requests.get, url_dev, timeout=5) + response_dev = capture_request(requests.get, url_dev, timeout=5, cache_kind="capabilities", method="get") if response_dev.status_code == 200: data = response_dev.json() for i in range(len(data["edges"])): @@ -207,14 +207,14 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s url = CI_URL_PREFIX + 'PharmacogenomicsKG/meta_knowledge_graph' url_dev = DEV_URL_PREFIX + 'PharmacogenomicsKG/meta_knowledge_graph' try: - response = capture_request(requests.get, url) + response = capture_request(requests.get, url, cache_kind="capabilities", method="get") if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): APInames, metaKG = add_new_API_for_query(APInames, metaKG, "CATRAX Pharmacogenomics KP - TRAPI 1.5.0", CI_URL_PREFIX + "PharmacogenomicsKG/query", data["edges"][i]['predicate'], data["edges"][i]['subject'], data["edges"][i]['object']) elif response.status_code == 404: # Try the dev URL if the CI URL returns a 404 - response_dev = capture_request(requests.get, url_dev, timeout=5) + response_dev = capture_request(requests.get, url_dev, timeout=5, cache_kind="capabilities", method="get") if response_dev.status_code == 200: data = response_dev.json() for i in range(len(data["edges"])): @@ -231,14 +231,14 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s url_dev = DEV_URL_PREFIX + 'ctkp/meta_knowledge_graph' try: - response = capture_request(requests.get, url) + response = capture_request(requests.get, url, cache_kind="capabilities", method="get") if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): APInames, metaKG = add_new_API_for_query(APInames, metaKG, "Clinical Trials KP - TRAPI 1.5.0", CI_URL_PREFIX + "ctkp/query", data["edges"][i]['predicate'], data["edges"][i]['subject'], data["edges"][i]['object']) elif response.status_code == 404: # Try the dev URL if the CI URL returns a 404 - response_dev = capture_request(requests.get, url_dev, timeout=5) + response_dev = capture_request(requests.get, url_dev, timeout=5, cache_kind="capabilities", method="get") if response_dev.status_code == 200: data = response_dev.json() for i in range(len(data["edges"])): @@ -255,14 +255,14 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s #url = 'https://multiomics.rtx.ai:9990/dakp/meta_knowledge_graph' try: - response = capture_request(requests.get, url) + response = capture_request(requests.get, url, cache_kind="capabilities", method="get") if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): APInames, metaKG = add_new_API_for_query(APInames, metaKG, "Drug Approvals KP - TRAPI 1.5.0", CI_URL_PREFIX + "dakp/query", data["edges"][i]['predicate'], data["edges"][i]['subject'], data["edges"][i]['object']) elif response.status_code == 404: # Try the dev URL if the CI URL returns a 404 - response_dev = capture_request(requests.get, url_dev, timeout=5) + response_dev = capture_request(requests.get, url_dev, timeout=5, cache_kind="capabilities", method="get") if response_dev.status_code == 200: data = response_dev.json() for i in range(len(data["edges"])): @@ -279,14 +279,14 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s url_dev = DEV_URL_PREFIX + 'mokp/meta_knowledge_graph' #url = 'https://multiomics.rtx.ai:9990/mokp/meta_knowledge_graph' try: - response = capture_request(requests.get, url) + response = capture_request(requests.get, url, cache_kind="capabilities", method="get") if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): APInames, metaKG = add_new_API_for_query(APInames, metaKG, "Multiomics KP - TRAPI 1.5.0", CI_URL_PREFIX + "mokp/query", data["edges"][i]['predicate'], data["edges"][i]['subject'], data["edges"][i]['object']) elif response.status_code == 404: # Try the dev URL if the CI URL returns a 404 - response_dev = capture_request(requests.get, url_dev, timeout=5) + response_dev = capture_request(requests.get, url_dev, timeout=5, cache_kind="capabilities", method="get") if response_dev.status_code == 200: data = response_dev.json() for i in range(len(data["edges"])): @@ -301,14 +301,14 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s url = CI_URL_PREFIX + 'mbkp/meta_knowledge_graph' url_dev = DEV_URL_PREFIX + 'mbkp/meta_knowledge_graph' try: - response = capture_request(requests.get, url) + response = capture_request(requests.get, url, cache_kind="capabilities", method="get") if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): APInames, metaKG = add_new_API_for_query(APInames, metaKG, "Microbiome KP - TRAPI 1.5.0", CI_URL_PREFIX + "mbkp/query", data["edges"][i]['predicate'], data["edges"][i]['subject'], data["edges"][i]['object']) elif response.status_code == 404: # Try the dev URL if the CI URL returns a 404 - response_dev = capture_request(requests.get, url_dev, timeout=5) + response_dev = capture_request(requests.get, url_dev, timeout=5, cache_kind="capabilities", method="get") if response_dev.status_code == 200: data = response_dev.json() for i in range(len(data["edges"])): @@ -323,7 +323,7 @@ def add_plover_API(APInames:dict[str, str], metaKG:pd.DataFrame) -> tuple[dict[s url = 'https://kg2cploverdb.ci.transltr.io/meta_knowledge_graph' try: - response = capture_request(requests.get, url) + response = capture_request(requests.get, url, cache_kind="capabilities", method="get") if response.status_code == 200: data = response.json() for i in range(len(data["edges"])): diff --git a/TCT/translator_query.py b/TCT/translator_query.py index d74984b..8d2b8d3 100644 --- a/TCT/translator_query.py +++ b/TCT/translator_query.py @@ -184,7 +184,9 @@ def query_KP(API_name_cur, query_json, APInames, API_predicates): query_copy = deepcopy(query_json) # optimize on our private copy query_json_cur = optimize_query_json(query_copy, API_name_cur, API_predicates) - response = capture_request(requests.post, API_url_cur, json=query_json_cur) + response = capture_request(requests.post, API_url_cur, json=query_json_cur, cache_kind="trapi", method="post", + original_arguments={"json": query_json}, + reformulation="tct.optimize_query_json.v1") if response.status_code == 200: result = response.json().get("message", {}) kg = result.get("knowledge_graph", {}) diff --git a/TCT/workflow.py b/TCT/workflow.py index b60f996..1ee1044 100644 --- a/TCT/workflow.py +++ b/TCT/workflow.py @@ -446,16 +446,55 @@ def _check_query(query: Any) -> None: raise ValueError("query edge endpoints must reference query nodes") -def _query(args: dict[str, Any], record: dict[str, Any]) -> Any: - _check_query(args["query"]) - # No predicate pruning, retry, redirect, or implicit provider discovery. - with requests.post( - args["url"], +def _read_response_bytes(response): + chunks = bytearray() + for chunk in response.iter_content(chunk_size=65536): + chunks.extend(chunk) + if len(chunks) > MAX_RESPONSE_BYTES: + raise ValueError( + f"response exceeds {MAX_RESPONSE_BYTES} bytes; no complete snapshot captured" + ) + return bytes(chunks) + + +def _workflow_request(args, record): + from .execution import cache_session + + kwargs = dict( json=args["query"], timeout=args["timeout_seconds"], allow_redirects=False, stream=True, - ) as response: + ) + cache = cache_session() + if cache is None: + return requests.post(args["url"], **kwargs) + + def consume(response): + try: + body = _read_response_bytes(response) + response._content = body + response._content_consumed = True + return body + finally: + response.close() + + receipt = record.setdefault("cache", {}) + return cache.request( + requests.post, + args["url"], + kwargs, + method="post", + kind="trapi", + consume=consume, + receipt=receipt, + ) + + +def _query(args: dict[str, Any], record: dict[str, Any]) -> Any: + _check_query(args["query"]) + # No predicate pruning, retry, redirect, or implicit provider discovery. + with _workflow_request(args, record) as response: record["evidence_origin"] = "provider_response" record["transport"] = { "status_code": response.status_code, @@ -466,14 +505,7 @@ def _query(args: dict[str, Any], record: dict[str, Any]) -> Any: if key in response.headers }, } - chunks = bytearray() - for chunk in response.iter_content(chunk_size=65536): - chunks.extend(chunk) - if len(chunks) > MAX_RESPONSE_BYTES: - raise ValueError( - f"response exceeds {MAX_RESPONSE_BYTES} bytes; no complete snapshot captured" - ) - body = bytes(chunks).decode("utf-8") + body = _read_response_bytes(response).decode("utf-8") record["response_text"] = body payload = json.loads(body) _json(payload) # Reject non-finite numbers before treating this as evidence. diff --git a/tests/test_execution.py b/tests/test_execution.py index ee9f7c1..ce2f52d 100644 --- a/tests/test_execution.py +++ b/tests/test_execution.py @@ -218,7 +218,7 @@ def fail(*args): "settings", [ {"record": True}, - {"cache": True}, + {"cache": {"mode": "unknown"}}, {"trapi": "yes"}, {"unknown": True}, {"depends_on": ["x"]}, diff --git a/tests/test_response_cache.py b/tests/test_response_cache.py new file mode 100644 index 0000000..5adc56a --- /dev/null +++ b/tests/test_response_cache.py @@ -0,0 +1,606 @@ +"""Phase 5: cache identity, policy epochs, failures, and offline replay.""" + +from concurrent.futures import ThreadPoolExecutor +import asyncio +from copy import deepcopy +import hashlib +from io import BytesIO +import json +from pathlib import Path + +import pytest +import requests + +from TCT.execution import execution, submit_with_context, capture_operation +from TCT.response_cache import CacheError, CacheReplayError, CacheStore +from TCT.service_capture import capture_request + + +def response(payload=None, status=200): + r = requests.Response() + r.status_code = status + r._content = json.dumps({"value": 1} if payload is None else payload).encode() + r._content_consumed = True + r.headers["Content-Type"] = "application/json" + r.url = "https://synthetic.invalid/query" + r.raw = BytesIO(r.content) + return r + + +@pytest.fixture +def transport(): + calls = [] + + def send(url, **kwargs): + calls.append(deepcopy(kwargs)) + return response() + + return send, calls + + +@capture_operation() +def fetch( + send, *, kind="normalization", url="https://synthetic.invalid/query", **kwargs +): + return capture_request(send, url, method="post", cache_kind=kind, **kwargs) + + +def cross(tmp_path, **policy): + return { + "store": str(tmp_path), + "cache": { + "scope": "cross_run", + "authorization_partition": "synthetic-public", + **policy, + }, + } + + +def test_default_cache_is_memory_only_and_independent(transport, tmp_path): + send, calls = transport + with execution({"cache": True, "langfuse": False}) as run: + first = fetch(send, json={"x": 1}) + first._content = b'{"mutated": true}' + assert fetch(send, json={"x": 1}).json() == {"value": 1} + assert list(run) == [] + assert [e["outcome"] for e in run.cache_events] == ["cold", "warm"] + with execution({"cache": True}): + fetch(send, json={"x": 1}) + assert len(calls) == 2 + assert not list(tmp_path.iterdir()) + + +def test_nested_disable_and_policy_override(transport): + send, calls = transport + with execution({"cache": {"epoch": "a"}}) as outer: + fetch(send) + with execution({"cache": False}) as disabled: + assert disabled is None + fetch(send) + with execution({"cache": {"mode": "bypass"}}) as inner: + assert inner is outer + fetch(send) + fetch(send) + assert len(calls) == 3 + assert [e["outcome"] for e in outer.cache_events] == ["cold", "bypassed", "warm"] + + +@pytest.mark.parametrize( + "kind,ttl", [("normalization", 86400), ("capabilities", 3600), ("trapi", 900)] +) +def test_cross_run_expiry_and_run_pinning(monkeypatch, tmp_path, kind, ttl): + now = [1000] + monkeypatch.setattr("TCT.response_cache.time.time", lambda: now[0]) + calls = [] + + def send(*a, **kw): + calls.append(1) + return response( + {"message": {"knowledge_graph": {"nodes": {}, "edges": {}}, "results": []}} + ) + + with execution(cross(tmp_path)): + fetch(send, kind=kind) + now[0] += ttl - 1 + with execution(cross(tmp_path)) as pinned: + fetch(send, kind=kind) + now[0] += 2 + fetch(send, kind=kind) + assert [e["outcome"] for e in pinned.cache_events] == ["warm", "warm"] + with execution(cross(tmp_path)) as expired: + fetch(send, kind=kind) + assert expired.cache_events[0]["outcome"] == "cold" + assert len(calls) == 2 + + +def test_persistent_within_run_never_leaks_to_another_run(tmp_path, transport): + send, calls = transport + config = {"cache": True, "store": str(tmp_path), "run_id": "one"} + for settings in [config, config, config | {"run_id": "two"}]: + with execution(settings): + fetch(send) + assert len(calls) == 2 + assert not (tmp_path / "journal.sqlite3").exists() + + +@pytest.mark.parametrize( + "change", + [ + {"epoch": "new"}, + {"dataset_versions": {"provider": "2"}}, + {"dataset_versions": {"normalizer": "2"}}, + {"authorization_partition": "another-user"}, + ], +) +def test_identity_invalidates_when_any_declared_dependency_changes( + tmp_path, transport, change +): + send, calls = transport + with execution(cross(tmp_path)) as before: + fetch(send, json={"q": [1, 2]}) + with execution(cross(tmp_path, **change)) as after: + fetch(send, json={"q": [1, 2]}) + assert len(calls) == 2 + assert ( + before.cache_events[0]["original_request_hash"] + == after.cache_events[0]["original_request_hash"] + ) + assert ( + before.cache_events[0]["effective_execution_hash"] + != after.cache_events[0]["effective_execution_hash"] + ) + + +def test_three_hashes_and_only_declared_equivalences(transport): + send, calls = transport + with execution({"cache": True}) as run: + fetch(send, json={"a": 1, "b": [1, 2]}) + fetch(send, json={"b": [1, 2], "a": 1}) + fetch(send, json={"a": 1, "b": [2, 1]}) + fetch(send, url="https://other.invalid/query", json={"a": 1, "b": [1, 2]}) + first, equal, ordered, endpoint = run.cache_events + assert first["original_request_hash"] == equal["original_request_hash"] + assert len({e["effective_execution_hash"] for e in [first, ordered, endpoint]}) == 3 + assert ( + first["response_content_hash"] == hashlib.sha256(response().content).hexdigest() + ) + assert len(calls) == 3 + + +@pytest.mark.parametrize("arguments", [{"json": {1: "value"}}, {"timeout": (1, 2)}]) +def test_undeclared_json_coercions_are_rejected(transport, arguments): + send, calls = transport + with execution({"cache": True}), pytest.raises(CacheError, match="JSON"): + fetch(send, **arguments) + assert calls == [] + + +def test_reformulation_records_original_and_effective_requests(transport): + send, calls = transport + with execution({"cache": True}) as run: + fetch( + send, + json={"predicates": ["a"]}, + original_arguments={"json": {"predicates": ["a", "b"]}}, + reformulation="predicate-pruning.v1", + ) + fetch(send, json={"predicates": ["a"]}) + one, two = run.cache_events + assert one["original_request_hash"] != two["original_request_hash"] + assert one["effective_execution_hash"] == two["effective_execution_hash"] + assert one["reformulation"] == "predicate-pruning.v1" + assert len(calls) == 1 + + +def test_refresh_preserves_snapshots_and_replay_order(tmp_path): + values = iter([1, 2, 3]) + + def send(*args, **kwargs): + return response({"value": next(values)}) + + config = {"cache": True, "store": str(tmp_path), "run_id": "original"} + with execution(config) as run: + assert fetch(send).json() == {"value": 1} + with execution({"cache": {"mode": "refresh"}}): + assert fetch(send).json() == {"value": 2} + with execution({"cache": {"mode": "bypass"}}): + assert fetch(send).json() == {"value": 3} + assert fetch(send).json() == {"value": 2} + originals = {p.name: p.read_bytes() for p in tmp_path.iterdir() if p.is_file()} + for event in run.cache_events: + assert (tmp_path / "artifacts" / event["snapshot_hash"]).exists() + with execution( + config + | {"run_id": "replay", "cache": {"mode": "replay", "snapshot_run": "original"}} + ) as replay: + for value in [1, 2, 3, 2]: + assert fetch( + lambda *a, **kw: pytest.fail("replay performed HTTP") + ).json() == {"value": value} + with pytest.raises(CacheReplayError, match="No remaining snapshot"): + fetch(send) + assert all(e["outcome"] == "replayed" for e in replay.cache_events) + assert originals == {name: (tmp_path / name).read_bytes() for name in originals} + + +def test_replay_cursor_survives_separate_invocations(tmp_path): + values = iter([1, 2]) + + def send(*a, **kw): + return response({"value": next(values)}) + + with execution( + {"cache": {"mode": "refresh"}, "store": str(tmp_path), "run_id": "source"} + ): + fetch(send) + fetch(send) + replay = { + "cache": {"mode": "replay", "snapshot_run": "source"}, + "store": str(tmp_path), + "run_id": "target", + } + for value in (1, 2): + with execution(replay): + assert fetch(lambda *a, **kw: pytest.fail("network")).json() == { + "value": value + } + with execution(replay), pytest.raises(CacheReplayError): + fetch(send) + with execution(replay | {"run_id": "intentional-resample"}): + assert fetch(send).json() == {"value": 1} + + +@pytest.mark.parametrize( + "record,trapi", [(False, False), (True, False), (False, True), (True, True)] +) +def test_cache_capture_combinations_remain_independent( + record, trapi, tmp_path, transport +): + send, calls = transport + with execution( + { + "cache": True, + "record": record, + "trapi": trapi, + "langfuse": False, + "store": str(tmp_path), + } + ) as run: + fetch(send) + fetch(send) + assert len(calls) == 1 + assert len(run) == (2 if record or trapi else 0) + assert (tmp_path / "journal.sqlite3").exists() is record + if record or trapi: + assert run[1]["observations"][0]["cache"]["outcome"] == "warm" + + +@pytest.mark.parametrize( + "payload,status", + [ + ({}, 503), + ({"error": "failed"}, 200), + ({"status": "ERROR"}, 200), + ("bad shape", 200), + ], +) +def test_unsuccessful_responses_are_observed_but_never_hits(tmp_path, payload, status): + calls = [] + + def send(*a, **kw): + calls.append(1) + return response(payload, status) + + with execution({"cache": True, "store": str(tmp_path)}) as run: + fetch(send) + fetch(send) + assert len(calls) == 2 + assert all(e["status"] == "not_cacheable" for e in run.cache_events) + + +def test_transport_failure_is_not_an_empty_result_and_can_replay(tmp_path): + def send(*a, **kw): + raise requests.Timeout("synthetic timeout") + + with execution({"cache": True, "store": str(tmp_path), "run_id": "failed"}): + with pytest.raises(requests.Timeout): + fetch(send) + with execution( + {"store": str(tmp_path), "cache": {"mode": "replay", "snapshot_run": "failed"}} + ) as run: + with pytest.raises(requests.Timeout, match="Replayed transport failure"): + fetch(lambda *a, **kw: pytest.fail("network")) + assert run.cache_events[0]["response_content_hash"] is None + + +def test_failed_refresh_does_not_replace_previous_success(transport): + send, _ = transport + with execution({"cache": True}): + fetch(send) + with execution({"cache": {"mode": "refresh"}}): + assert fetch(lambda *a, **kw: response({}, 500)).status_code == 500 + assert fetch(lambda *a, **kw: pytest.fail("cached success lost")).json() == { + "value": 1 + } + + +def test_tampered_snapshot_fails_closed(tmp_path, transport): + send, _ = transport + with execution({"cache": True, "store": str(tmp_path), "run_id": "source"}) as run: + fetch(send) + (tmp_path / "artifacts" / run.cache_events[0]["snapshot_hash"]).write_text("{}") + with execution( + {"store": str(tmp_path), "cache": {"mode": "replay", "snapshot_run": "source"}} + ): + with pytest.raises(CacheReplayError, match="checksum"): + fetch(lambda *a, **kw: pytest.fail("network")) + + +def test_storage_failure_reports_completed_call(monkeypatch, tmp_path, transport): + send, calls = transport + + def broken(*a, **kw): + raise OSError("synthetic full disk") + + monkeypatch.setattr(CacheStore, "save", broken) + with execution({"cache": True, "store": str(tmp_path)}): + with pytest.raises(CacheError) as error: + fetch(send) + assert error.value.operation_completed is True + assert len(calls) == 1 + + +def test_workers_share_context_but_runs_do_not(transport): + send, calls = transport + with execution({"cache": True}) as run, ThreadPoolExecutor(4) as pool: + fetch(send) + outputs = [submit_with_context(pool, fetch, send) for _ in range(8)] + assert all(f.result().json() == {"value": 1} for f in outputs) + assert len(calls) == 1 + assert len(run.cache_events) == 9 + + +@pytest.mark.parametrize( + "policy", + [ + None, + 1, + "use", + {"scope": "global"}, + {"scope": "cross_run"}, + {"mode": "replay"}, + {"ttl": {"trapi": -1}}, + {"ttl": {"unknown": 1}}, + {"dataset_versions": {"provider": None}}, + ], +) +def test_invalid_policies_are_rejected_before_calls(policy): + with pytest.raises(ValueError): + with execution({"cache": policy}): + pytest.fail("invalid configuration") + + +def test_authenticated_cache_requires_explicit_partition(transport): + send, calls = transport + with execution({"cache": True}): + with pytest.raises(CacheError, match="authorization_partition"): + fetch(send, headers={"Authorization": "synthetic-secret"}) + assert calls == [] + + +@pytest.mark.parametrize("interface", ["cli", "mcp"]) +def test_persistent_cache_through_explicit_interface_settings( + interface, monkeypatch, tmp_path, capsys +): + from TCT.interfaces.cli import main + from TCT.interfaces.mcp import mcp + + payload = json.loads( + (Path(__file__).parent / "fixtures/rollup/normalization.json").read_text() + ) + calls = [] + + def frozen(*a, **kw): + calls.append(1) + return response(payload) + + monkeypatch.setattr(requests.sessions.Session, "request", frozen) + config = {"cache": True, "store": str(tmp_path), "run_id": "interface-run"} + + def call(config): + if interface == "cli": + path = tmp_path / "execution.json" + path.write_text(json.dumps(config)) + assert ( + main( + [ + "normalize-nodes", + "--query", + "MONDO:0004975", + "--execution", + str(path), + ] + ) + == 0 + ) + return json.loads(capsys.readouterr().out) + + async def invoke(): + registry = await mcp.get_tools() + return await registry["normalize_nodes"].run( + {"query": "MONDO:0004975", "execution": config} + ) + + result = asyncio.run(invoke()) + return result.structured_content or [part.text for part in result.content] + + first = call(config) + assert call(config) == first + assert len(calls) == 1 + replay = config | { + "run_id": "replay", + "cache": {"mode": "replay", "snapshot_run": "interface-run"}, + } + monkeypatch.setattr( + requests.sessions.Session, + "request", + lambda *a, **kw: pytest.fail("replay network"), + ) + assert call(replay) == first + assert call(replay) == first + + +def test_nested_partial_policy_inherits_required_fields(tmp_path, transport): + send, calls = transport + with execution( + {"store": str(tmp_path), "cache": {"authorization_partition": "public"}} + ): + with execution({"cache": {"scope": "cross_run"}}): + fetch(send) + assert len(calls) == 1 + + +def test_runtime_and_policy_version_are_execution_dependencies(monkeypatch, transport): + from TCT.config import RuntimeConfig + + send, calls = transport + with execution({"cache": True}) as run: + fetch(send) + monkeypatch.setattr( + "TCT.response_cache.get_runtime_config", lambda: RuntimeConfig("prod") + ) + fetch(send) + monkeypatch.setattr("TCT.response_cache.POLICY_VERSION", "future-policy") + fetch(send) + assert len(calls) == 3 + assert len({e["effective_execution_hash"] for e in run.cache_events}) == 3 + + +def test_workflow_caches_and_replays_complete_bounded_responses(monkeypatch, tmp_path): + from TCT import workflow + + fixtures = Path(__file__).parent / "fixtures" / "rollup" + payload = json.loads((fixtures / "gene.json").read_text()) + args = { + "url": "https://synthetic.invalid/query", + "timeout_seconds": 10, + "query": json.loads((fixtures / "case.json").read_text())["queries"]["genes"], + } + calls = [] + + def send(*a, **kw): + calls.append(1) + return response(payload) + + monkeypatch.setattr(requests, "post", send) + with execution({"cache": True, "store": str(tmp_path), "run_id": "workflow"}): + for outcome in ["cold", "warm"]: + record = {} + assert workflow._query(args, record) == payload + assert record["cache"]["outcome"] == outcome + monkeypatch.setattr( + requests, "post", lambda *a, **kw: pytest.fail("replay network") + ) + with execution( + { + "store": str(tmp_path), + "cache": {"mode": "replay", "snapshot_run": "workflow"}, + } + ): + assert workflow._query(args, {}) == payload + assert len(calls) == 1 + + +def test_workflow_streaming_limit_is_preserved(monkeypatch, tmp_path): + from TCT import workflow + + monkeypatch.setattr(workflow, "MAX_RESPONSE_BYTES", 3) + monkeypatch.setattr(requests, "post", lambda *a, **kw: response({"long": "body"})) + args = { + "url": "https://synthetic.invalid/query", + "timeout_seconds": 10, + "query": { + "message": { + "query_graph": { + "nodes": {"a": {}, "b": {}}, + "edges": {"e": {"subject": "a", "object": "b"}}, + } + } + }, + } + with execution({"cache": True, "store": str(tmp_path)}) as run: + with pytest.raises(ValueError, match="exceeds 3"): + workflow._query(args, {}) + assert run.cache_events[0]["status"] == "incomplete_snapshot" + assert run.cache_events[0]["snapshot_hash"] is None + + +def test_same_nlq_rollup_in_all_cache_modes(monkeypatch, tmp_path): + from TCT.config import service_url + from TCT.evidence import reduce_evidence + from TCT.interfaces import tools + from TCT.interfaces.serialization import to_jsonable + + fixtures = Path(__file__).parent / "fixtures" / "rollup" + + def load(name): + return json.loads((fixtures / (name + ".json")).read_text()) + + calls = [] + providers = load("providers") + + def frozen(session, method, url, **kw): + calls.append(url) + if url == service_url("smartapi_registry"): + return response(load("registry")) + if url.endswith("/get_normalized_nodes"): + return response(load("normalization")) + if url in providers.values(): + kind = "genes" if url == providers["gene-provider"] else "chemicals" + assert kw["json"] == load("case")["queries"][kind] + return response(load("gene" if kind == "genes" else "chemical")) + pytest.fail(f"Unexpected request {method} {url}") + + monkeypatch.setattr(requests.sessions.Session, "request", frozen) + for scenario, mode, count in [ + ("cold", "use", 4), + ("warm", "use", 0), + ("refreshed", "refresh", 4), + ("replayed", "replay", 0), + ]: + policy = {"mode": mode} + if mode == "replay": + policy["snapshot_run"] = "cold" + monkeypatch.setattr( + requests.sessions.Session, + "request", + lambda *a, **kw: pytest.fail("replay performed network I/O"), + ) + before = len(calls) + with execution( + cross(tmp_path, **policy) | {"run_id": scenario, "trapi": True} + ) as run: + outputs = [ + tools.normalize_nodes( + "MONDO:0004975", return_equivalent_identifiers=True + ), + tools.get_kp_info(), + ] + for kind, provider in [ + ("genes", "gene-provider"), + ("chemicals", "chemical-provider"), + ]: + outputs.append( + tools.query_knowledge_provider( + provider, + load("case")["queries"][kind], + outputs[1][1], + load("predicates"), + ) + ) + for actual, expected in zip(outputs, load("golden")["history"]): + assert to_jsonable(actual) == expected["output"] + assert len(calls) - before == count + assert {e["outcome"] for e in run.cache_events} == {scenario} + rollup = reduce_evidence(run, load("composition")) + assert rollup["rollup_hash"] == load("rollup-golden")["rollup_hash"] From 1f5eb8848fad1fb8ca155597873bb149e1adf386 Mon Sep 17 00:00:00 2001 From: Kenneth Bruskiewicz Date: Fri, 13 Nov 2026 12:21:57 -0700 Subject: [PATCH 8/8] Fix CI: skip vendored TRAPI schemas in codespell, update live-service tests The vendored TRAPI schemas are checksum-pinned upstream files, so codespell now skips TCT/schemas instead of editing them. NameRes on Babel 2026jul22 reports Solr stats at the top level and indexes ~331M names (2025sep1 had ~425M under "solr"). The status test accepts both shapes and uses a 300M sanity floor. Node Annotator no longer returns clinical_approval for CHEBI:15377. The expectation drops the volatile warning and clinical fields, matching #63. Co-Authored-By: Claude Opus 5.5 --- Makefile | 4 ++-- tests/test_nameres.py | 7 ++++++- tests/test_node_annotator.py | 14 ++++++++++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 6a69aa6..41143ac 100644 --- a/Makefile +++ b/Makefile @@ -10,11 +10,11 @@ test: # Run spell checking spell: - uv run codespell --skip=metaData,notebooks,htmlcov,*.json --ignore-words-list=EHR + uv run codespell --skip=metaData,notebooks,htmlcov,*.json,./TCT/schemas --ignore-words-list=EHR # Run spell checking interactively (allows fixing) spell-fix: - uv run codespell --interactive 3 --skip=metaData,notebooks,htmlcov,*.json --ignore-words-list=EHR + uv run codespell --interactive 3 --skip=metaData,notebooks,htmlcov,*.json,./TCT/schemas --ignore-words-list=EHR # Run linting lint: diff --git a/tests/test_nameres.py b/tests/test_nameres.py index 2f3e726..eddb7f7 100644 --- a/tests/test_nameres.py +++ b/tests/test_nameres.py @@ -61,7 +61,12 @@ def test_nameres_status(): assert status["status"] == "ok" assert status["babel_version"] != "" assert status["babel_version_url"] != "" - assert status["solr"]["numDocs"] > 425_000_000 + # Older NameRes releases nest Solr index stats under "solr"; newer ones + # report them at the top level. + # Babel 2025sep1 indexed ~425M names and Babel 2026jul22 ~331M; the floor + # is a sanity check that the index is loaded, not a fixed release size. + num_docs = status.get("solr", status)["numDocs"] + assert num_docs > 300_000_000 def test_nameres_incorrect(): diff --git a/tests/test_node_annotator.py b/tests/test_node_annotator.py index daa4135..79576ff 100644 --- a/tests/test_node_annotator.py +++ b/tests/test_node_annotator.py @@ -31,8 +31,18 @@ def use_node_annotator_prod(): 'curie': 'CHEBI:15377', 'expected': { 'query': 'CHEBI:15377', - 'boxed_warning': True, - 'sections': ['aeolus', 'chebi', 'chembl', 'clinical_approval', 'clinical_trials', 'drugbank', 'ndc', 'pubchem', 'unichem', 'unii'], + # Warning and clinical approval/trial fields depend on volatile + # upstream providers and are not part of the stable contract. + 'sections': [ + 'aeolus', + 'chebi', + 'chembl', + 'drugbank', + 'ndc', + 'pubchem', + 'unichem', + 'unii', + ], 'chembl.availability_type': 2, }, },