diff --git a/docs/guide/connect.md b/docs/guide/connect.md index 0cbc4a0e..75b96fd8 100644 --- a/docs/guide/connect.md +++ b/docs/guide/connect.md @@ -95,7 +95,9 @@ https://developmentfinance.org/mcp Leave off the trailing slash in every client configuration. The site answers `/mcp/` with a 308 redirect to `/mcp`, which some clients don't follow. -No token is needed. The site forwards each request to the DFO AI service with its own credentials. +No token is needed for anonymous use. The site forwards each request to the DFO AI service with its own credentials. + +An API key is optional and raises the call limit. Developer clients such as Claude Code, Cursor, and VS Code, and Claude custom connectors, can send it as a request header, either `Authorization: Bearer ` or `X-API-Key`. ChatGPT connectors cannot send a header, so ChatGPT stays anonymous. [Use an API key](../hub/build/api-keys.md) covers creating and sending one. ### Tools @@ -240,12 +242,12 @@ If the assistant answers without calling `search`, check that the client lists t ## Troubleshooting for developers -| Problem | Cause | Fix | -| ---------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | -| Redirect on connect, or "server unreachable" with a 308 in the client's logs | Wrong URL ending | Use `/mcp`, with no slash at the end | -| `rate_limited` refusal | The server is over its calls-per-second limit across all callers, or the instance already has 16 calls running | Repeat after the `retry_after_s` the refusal gives | -| HTTP 429 from the site | One address sent more requests in a minute than the site allows | Wait for the seconds given in the `Retry-After` header | -| `internal_error` refusal | A tool failed in a way with no specific refusal, and the response says only that it failed on the server | Send the time of the call and the tool's name to whoever runs the deployment | +| Problem | Cause | Fix | +| ---------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | +| Redirect on connect, or "server unreachable" with a 308 in the client's logs | Wrong URL ending | Use `/mcp`, with no slash at the end | +| `rate_limited` refusal | The server is over its calls-per-second limit across all callers, or the instance already has 16 calls running | Repeat after the `retry_after_s` the refusal gives | +| HTTP 429 from the site | One address sent more requests in a minute than the site allows. A call with a key counts against its account | Wait for the seconds in the `Retry-After` header, which every 429 carries. A key raises the limit | +| `internal_error` refusal | A tool failed in a way with no specific refusal, and the response says only that it failed on the server | Send the time of the call and the tool's name to whoever runs the deployment | ## Next diff --git a/packages/common/README.md b/packages/common/README.md index e0c44d83..4692341c 100644 --- a/packages/common/README.md +++ b/packages/common/README.md @@ -58,7 +58,7 @@ The class loads values from process environment variables and `.env` files. Unkn ## Caller identity -A caller names who an MCP call is made for. It is an opaque string with a kind prefix, sent in the `x-dfo-caller` request header: `conversation:` and 32 hex characters from the harness, the start of the SHA-256 of the principal and conversation id (`conversation_caller`), so neither leaves the process; `ip:`, `key:` and `user:` from the website's edge. The MCP transport session cannot serve as this identity. Under the 2026-07-28 protocol revision a server builds a connection per request, so FastMCP's `Context.session_id` changes on every call. +A caller names who an MCP call is made for. It is an opaque string with a kind prefix, sent in the `x-dfo-caller` request header: `conversation:` and 32 hex characters from the harness, the start of the SHA-256 of the principal and conversation id (`conversation_caller`), so neither leaves the process; `ip:` and `user:` from the website's edge (the client's address, or the account behind an API key, as 32 hex characters of an HMAC digest). The clients that call dfo-apis forward only those two forms, beside the caller principal (`WEBSITE_CALLER`). The MCP transport session cannot serve as this identity. Under the 2026-07-28 protocol revision a server builds a connection per request, so FastMCP's `Context.session_id` changes on every call. `current_caller()` returns the caller of the current request, or `None` when the request has no header, a blank one, more than one, or one longer than `MAX_CALLER_CHARS` (256), or when no HTTP request is in flight. A repeated header counts as none so that a client's copy cannot outvote the one a proxy sets. It never generates an identity in place of a missing one. The frame store records `current_writer()` as a stored frame's `writer`: the caller, or `anonymous` with none. `supplied_caller(caller)` sets the caller for a block, for tests that drive a server through an in-memory client, which sends no headers. diff --git a/packages/common/src/common/caller.py b/packages/common/src/common/caller.py index d4334295..a05960c3 100644 --- a/packages/common/src/common/caller.py +++ b/packages/common/src/common/caller.py @@ -3,7 +3,9 @@ A caller is an opaque string with a kind prefix, carried in the `x-dfo-caller` request header: `conversation:` from the harness (`conversation_caller`), and -`ip:`, `key:` or `user:` from the website's edge. A request without the +`ip:` or `user:` from the website's edge: the client's address, or the +account an API key belongs to (limits are per account), each as 32 hex digits +of an HMAC digest, never the address or account id itself. A request without the header has no caller, and `current_caller` returns `None` for it. Nothing here generates an identity in its place. @@ -28,14 +30,16 @@ CALLER_HEADER = "x-dfo-caller" # Sent beside `CALLER_HEADER` to dfo-apis on an external caller's calls, when the -# website named that caller by address: the secret dfo-apis shares with the +# website named that caller by address or account: the secret dfo-apis shares with the # website, which vouches for the caller it names # (`dfo_platform.caller.resolve_caller`). CALLER_PRINCIPAL_HEADER = "x-dfo-caller-principal" -# The form of a caller the website named by its client's address. dfo-apis -# accepts no other form beside `CALLER_PRINCIPAL_HEADER`. -ADDRESS_CALLER = re.compile(r"ip:[0-9a-f]{32}") +# The forms of a caller the website names: `ip:` for its client's address and +# `user:` for the account behind an API key, each 32 lowercase hex digits. +# dfo-apis accepts no other form beside `CALLER_PRINCIPAL_HEADER`, so a caller +# that does not match is not forwarded. +WEBSITE_CALLER = re.compile(r"(?:ip|user):[0-9a-f]{32}") # The longest caller `current_caller` accepts. The frame store writes the # caller into GCS object metadata, which holds 8 KiB in all. diff --git a/packages/common/tests/test_caller.py b/packages/common/tests/test_caller.py index b32d65b6..202525db 100644 --- a/packages/common/tests/test_caller.py +++ b/packages/common/tests/test_caller.py @@ -14,6 +14,7 @@ ANONYMOUS_WRITER, CALLER_HEADER, MAX_CALLER_CHARS, + WEBSITE_CALLER, conversation_caller, current_caller, current_writer, @@ -147,3 +148,28 @@ def test_a_conversation_caller_is_a_fixed_length_digest_of_the_key() -> None: assert "u_0123456789abcdef" not in caller assert conversation_caller(key) == caller assert conversation_caller("u/a") != conversation_caller("u/b") + + +@pytest.mark.parametrize("caller", ["ip:" + "0" * 32, "user:" + "0123456789abcdef" * 2]) +def test_website_caller_matches_the_forms_dfo_apis_accepts(caller: str) -> None: + assert WEBSITE_CALLER.fullmatch(caller) + + +@pytest.mark.parametrize( + "caller", + [ + "key:" + "0" * 32, + "conversation:" + "0" * 32, + "ip:" + "0" * 31, + "user:" + "0" * 31, + "user:" + "0" * 33, + "user:" + "g" * 32, + "user:" + "A" * 32, + "user:" + "0" * 32 + "\n", + " user:" + "0" * 32, + "user:edge", + "", + ], +) +def test_website_caller_rejects_every_other_form(caller: str) -> None: + assert WEBSITE_CALLER.fullmatch(caller) is None diff --git a/services/analysis-mcp/src/analysis_mcp/runner_client.py b/services/analysis-mcp/src/analysis_mcp/runner_client.py index 8083824b..17aa21e9 100644 --- a/services/analysis-mcp/src/analysis_mcp/runner_client.py +++ b/services/analysis-mcp/src/analysis_mcp/runner_client.py @@ -78,7 +78,7 @@ import httpx2 from common.apis_auth import ApisAuthError, AuthProvider, apis_auth_provider -from common.caller import ADDRESS_CALLER, CALLER_HEADER, CALLER_PRINCIPAL_HEADER, current_caller +from common.caller import CALLER_HEADER, CALLER_PRINCIPAL_HEADER, WEBSITE_CALLER, current_caller from common.derived_replay import ( ANALYSIS_PATH, DEFAULT_RETRY_AFTER_S, @@ -187,7 +187,7 @@ def _caller_headers(self) -> dict[str, str]: caller = current_caller() if self._caller_principal is None or caller is None: return {} - if not ADDRESS_CALLER.fullmatch(caller): + if not WEBSITE_CALLER.fullmatch(caller): return {} return {CALLER_PRINCIPAL_HEADER: self._caller_principal, CALLER_HEADER: caller} diff --git a/services/analysis-mcp/tests/test_runner.py b/services/analysis-mcp/tests/test_runner.py index 4a1be592..f7d4bab6 100644 --- a/services/analysis-mcp/tests/test_runner.py +++ b/services/analysis-mcp/tests/test_runner.py @@ -47,6 +47,7 @@ _OTHER_SPEC = "spec_" + "7" * 32 _OTHER_FRAME = {"spec_id": _OTHER_SPEC, "content_version": "cv_" + "8" * 32} _ADDRESS_CALLER = "ip:" + "a" * 32 +_ACCOUNT_CALLER = "user:" + "b" * 32 _SUM = {"by": ["time_period"], "op": "sum"} _BREAKDOWN = {"by": ["crsPurpose", "time_period"], "op": "sum", "levels": {"crsPurpose": 1}} @@ -655,10 +656,32 @@ def test_a_summary_holds_only_the_entries_its_digest_shows( "secret", {"x-dfo-caller-principal": "secret", "x-dfo-caller": _ADDRESS_CALLER}, ), + ( + _ACCOUNT_CALLER, + "secret", + {"x-dfo-caller-principal": "secret", "x-dfo-caller": _ACCOUNT_CALLER}, + ), (_ADDRESS_CALLER, None, {}), + (_ACCOUNT_CALLER, None, {}), ("conversation:" + "b" * 32, "secret", {}), + ("ip:" + "a" * 31, "secret", {}), + ("user:" + "b" * 31, "secret", {}), + ("user:" + "b" * 33, "secret", {}), + ("user:" + "g" * 32, "secret", {}), + ("user:" + "B" * 32, "secret", {}), + ], + ids=[ + "website-client", + "website-account", + "no-principal", + "no-principal-account", + "harness-conversation", + "malformed-ip-short", + "malformed-user-short", + "malformed-user-long", + "malformed-user-not-hex", + "malformed-user-uppercase", ], - ids=["website-client", "no-principal", "harness-conversation"], ) async def test_a_website_caller_is_named_beside_the_principal( store: InMemoryMaterialisationStore, diff --git a/services/data-mcp/src/data_mcp/client.py b/services/data-mcp/src/data_mcp/client.py index e99d843f..20b3bfd1 100644 --- a/services/data-mcp/src/data_mcp/client.py +++ b/services/data-mcp/src/data_mcp/client.py @@ -21,7 +21,7 @@ import httpx2 from common.apis_auth import ApisAuthError, AuthProvider -from common.caller import ADDRESS_CALLER, CALLER_HEADER, CALLER_PRINCIPAL_HEADER, current_caller +from common.caller import CALLER_HEADER, CALLER_PRINCIPAL_HEADER, WEBSITE_CALLER, current_caller from opentelemetry import trace from data_mcp.deadline import CallDeadline @@ -267,14 +267,15 @@ def caller_headers(self) -> dict[str, str]: """The headers that name the current request's caller to dfo-apis. The caller principal and the caller, when this process holds the principal - (`Settings.dfo_caller_principal`) and the website named the caller by address - (`x-dfo-caller: ip:`), so dfo-apis counts that client's allowance. + (`Settings.dfo_caller_principal`) and the website named the caller by address or account + (`x-dfo-caller: ip:` or `user:`), so dfo-apis counts that + client's or account's allowance. Empty otherwise, and dfo-apis counts the call against this service. """ caller = current_caller() if self._caller_principal is None or caller is None: return {} - if not ADDRESS_CALLER.fullmatch(caller): + if not WEBSITE_CALLER.fullmatch(caller): return {} return {CALLER_PRINCIPAL_HEADER: self._caller_principal, CALLER_HEADER: caller} diff --git a/services/data-mcp/tests/test_export_data_tool.py b/services/data-mcp/tests/test_export_data_tool.py index 4ea33a32..50a5f67a 100644 --- a/services/data-mcp/tests/test_export_data_tool.py +++ b/services/data-mcp/tests/test_export_data_tool.py @@ -34,6 +34,7 @@ SIGNED = "https://storage.googleapis.com/pinned/sha256:abc?X-Goog-Signature=secret" PRINCIPAL = "principal-secret" ADDRESS = "ip:" + "0" * 32 +ACCOUNT = "user:" + "1" * 32 FETCHED = {"spec_id": "spec_" + "1" * 32, "content_version": "cv_" + "2" * 32} DERIVED = {"spec_id": "ops_" + "3" * 32, "content_version": "ocv_" + "4" * 32} @@ -276,10 +277,28 @@ async def test_a_bare_derived_id_refuses_frame_ref_incomplete_without_a_call() - ("caller", "principal", "sent"), [ (ADDRESS, PRINCIPAL, {CALLER_PRINCIPAL_HEADER: PRINCIPAL, CALLER_HEADER: ADDRESS}), + (ACCOUNT, PRINCIPAL, {CALLER_PRINCIPAL_HEADER: PRINCIPAL, CALLER_HEADER: ACCOUNT}), (ADDRESS, None, {}), + (ACCOUNT, None, {}), (conversation_caller("c1"), PRINCIPAL, {}), + ("ip:" + "0" * 31, PRINCIPAL, {}), + ("user:" + "1" * 31, PRINCIPAL, {}), + ("user:" + "1" * 33, PRINCIPAL, {}), + ("user:" + "G" * 32, PRINCIPAL, {}), + ("user:" + "A" * 32, PRINCIPAL, {}), + ], + ids=[ + "website-caller", + "account-caller", + "no-principal", + "no-principal-account", + "harness-caller", + "malformed-ip-short", + "malformed-user-short", + "malformed-user-long", + "malformed-user-not-hex", + "malformed-user-uppercase", ], - ids=["website-caller", "no-principal", "harness-caller"], ) async def test_the_prepare_names_the_caller_the_website_named( caller: str, principal: str | None, sent: dict[str, str] diff --git a/services/presentation-mcp/src/presentation_mcp/client.py b/services/presentation-mcp/src/presentation_mcp/client.py index b47e600c..e180ed6d 100644 --- a/services/presentation-mcp/src/presentation_mcp/client.py +++ b/services/presentation-mcp/src/presentation_mcp/client.py @@ -31,7 +31,7 @@ import httpx2 from common.apis_auth import ApisAuthError, AuthProvider -from common.caller import ADDRESS_CALLER, CALLER_HEADER, CALLER_PRINCIPAL_HEADER, current_caller +from common.caller import CALLER_HEADER, CALLER_PRINCIPAL_HEADER, WEBSITE_CALLER, current_caller from common.derived_replay import ( DEFAULT_RETRY_AFTER_S, ReplayBusy, @@ -1036,15 +1036,15 @@ def _card_write_headers(self, caller: Caller) -> dict[str, str]: A first-party write, Studio's, carries the publish principal, which dfo-apis exempts. An external write carries the caller principal and the caller the website named, when this process holds the principal - and the request's `x-dfo-caller` is an `ip:` digest, so dfo-apis - counts that client. Any other external write carries neither, and + and the request's `x-dfo-caller` is an `ip:` or `user:` digest, so + dfo-apis counts that client or account. Any other external write carries neither, and dfo-apis counts it against the address it sees for this service. """ if caller is Caller.FIRST_PARTY: principal = self._publish_principal return {} if principal is None else {PRINCIPAL_HEADER: principal} named = current_caller() - if self._caller_principal is None or named is None or not ADDRESS_CALLER.fullmatch(named): + if self._caller_principal is None or named is None or not WEBSITE_CALLER.fullmatch(named): return {} return {CALLER_PRINCIPAL_HEADER: self._caller_principal, CALLER_HEADER: named} diff --git a/services/presentation-mcp/tests/test_client.py b/services/presentation-mcp/tests/test_client.py index 97f4e67f..131ab252 100644 --- a/services/presentation-mcp/tests/test_client.py +++ b/services/presentation-mcp/tests/test_client.py @@ -1222,6 +1222,7 @@ async def test_publish_call_carries_the_configured_principal_header( _CALLER_PRINCIPAL = "website-secret" _ADDRESS_CALLER = "ip:" + "0123456789abcdef" * 2 +_ACCOUNT_CALLER = "user:" + "fedcba9876543210" * 2 @pytest.mark.parametrize( @@ -1246,6 +1247,14 @@ async def test_publish_call_carries_the_configured_principal_header( {CALLER_PRINCIPAL_HEADER: _CALLER_PRINCIPAL, CALLER_HEADER: _ADDRESS_CALLER}, id="external-named-by-website", ), + pytest.param( + Caller.EXTERNAL, + _PUBLISH_PRINCIPAL, + _CALLER_PRINCIPAL, + _ACCOUNT_CALLER, + {CALLER_PRINCIPAL_HEADER: _CALLER_PRINCIPAL, CALLER_HEADER: _ACCOUNT_CALLER}, + id="external-account-named-by-website", + ), pytest.param( Caller.EXTERNAL, _PUBLISH_PRINCIPAL, @@ -1254,13 +1263,61 @@ async def test_publish_call_carries_the_configured_principal_header( {}, id="external-no-caller-principal", ), + pytest.param( + Caller.EXTERNAL, + _PUBLISH_PRINCIPAL, + None, + _ACCOUNT_CALLER, + {}, + id="external-account-no-caller-principal", + ), pytest.param( Caller.EXTERNAL, _PUBLISH_PRINCIPAL, _CALLER_PRINCIPAL, "conversation:" + "0" * 32, {}, - id="external-not-an-address", + id="external-not-a-website-caller", + ), + pytest.param( + Caller.EXTERNAL, + _PUBLISH_PRINCIPAL, + _CALLER_PRINCIPAL, + "ip:" + "0" * 31, + {}, + id="external-malformed-ip", + ), + pytest.param( + Caller.EXTERNAL, + _PUBLISH_PRINCIPAL, + _CALLER_PRINCIPAL, + "user:" + "0" * 31, + {}, + id="external-malformed-user-short", + ), + pytest.param( + Caller.EXTERNAL, + _PUBLISH_PRINCIPAL, + _CALLER_PRINCIPAL, + "user:" + "0" * 33, + {}, + id="external-malformed-user-long", + ), + pytest.param( + Caller.EXTERNAL, + _PUBLISH_PRINCIPAL, + _CALLER_PRINCIPAL, + "user:" + "g" * 32, + {}, + id="external-malformed-user-not-hex", + ), + pytest.param( + Caller.EXTERNAL, + _PUBLISH_PRINCIPAL, + _CALLER_PRINCIPAL, + "user:" + "A" * 32, + {}, + id="external-malformed-user-uppercase", ), pytest.param( Caller.EXTERNAL, _PUBLISH_PRINCIPAL, _CALLER_PRINCIPAL, None, {}, id="external-unnamed"