fix(aio): keep content out of ai events in privacy mode

.sampo/changesets/ai-privacy-mode-leaks.md

@@ -0,0 +1,12 @@
+---
+pypi/posthog: patch
+---
+
+Privacy mode now covers error text and the remaining content paths in the AI integrations.
+
+- `$ai_error` no longer carries the exception message when privacy mode is on. Provider errors and validation errors often quote the prompt or the model output, so the SDK now sends a summary built only from structured fields: the class name, HTTP status, provider error code, `param`, request ID and the class names of chained causes (for example `BadRequestError: 400 invalid_request_error param=messages[3].content request_id=req_123`). This applies to the OpenAI, Anthropic and Gemini wrappers (including Gemini embeddings), LangChain generations, spans and traces, and OpenAI Agents spans.
+- The LangChain callback handler no longer sends an `$exception` event for a failed run when privacy mode is on.
+- `$ai_tools` masks the `headers` and `authorization` of MCP tool definitions (`type: "mcp"`) with `[REDACTED]`, with or without privacy mode. The tools you pass in are not modified.
+- Gemini: a per-call `posthog_privacy_mode=False` no longer turns off privacy mode set on the client, matching the OpenAI and Anthropic wrappers.
+- OpenAI Agents: privacy mode now redacts `$ai_trace_metadata` and the `prompt` and `instructions` in transcription and speech span `model_config`.
+- OpenTelemetry: `PostHogSpanProcessor` and `PostHogTraceExporter` accept `privacy_mode` and `client`, and also honor the client's (or the module-level) `privacy_mode`. When on, exported spans drop prompt, completion, tool argument and tool result attributes, non-exception span events, exception messages and stack traces, and the span status description.

posthog/ai/anthropic/anthropic_converter.py

@@ -18,7 +18,7 @@
     TokenUsage,
     ToolInProgress,
 )
-from posthog.ai.utils import serialize_raw_usage
+from posthog.ai.utils import _redact_mcp_tool_credentials, serialize_raw_usage
 
 
 def format_anthropic_response(response: Any) -> List[FormattedMessage]:
@@ -153,7 +153,7 @@ def extract_anthropic_tools(kwargs: Dict[str, Any]) -> Optional[Any]:
         Tool definitions if present, None otherwise
     """
 
-    return kwargs.get("tools", None)
+    return _redact_mcp_tool_credentials(kwargs.get("tools", None))
 
 
 def format_anthropic_streaming_content(

posthog/ai/gemini/_shared.py

@@ -16,6 +16,7 @@
     _capture_ai_event,
     capture_streaming_event,
     finalize_ai_content,
+    _format_ai_error,
     _get_http_status,
     merge_system_prompt,
     with_privacy_mode,
@@ -176,11 +177,8 @@ def _merge_posthog_params(
             if call_distinct_id is not None
             else self._default_distinct_id
         )
-        privacy_mode = (
-            call_privacy_mode
-            if call_privacy_mode is not None
-            else self._default_privacy_mode
-        )
+        # A call can add redaction but never lift the client default.
+        privacy_mode = self._default_privacy_mode or call_privacy_mode is True
         groups = call_groups if call_groups is not None else self._default_groups
 
         properties = dict(self._default_properties)
@@ -265,7 +263,9 @@ def _capture_embedding_outcome(
 
         if error:
             event_properties["$ai_is_error"] = True
-            event_properties["$ai_error"] = str(error)
+            event_properties["$ai_error"] = _format_ai_error(
+                self._ph_client, privacy_mode, error
+            )
 
         if distinct_id is None:
             event_properties["$process_person_profile"] = False

posthog/ai/gemini/gemini.py

@@ -87,7 +87,7 @@ def __init__(
             posthog_client: PostHog client for tracking usage
             posthog_distinct_id: Default distinct ID for all calls (can be overridden per call)
             posthog_properties: Default properties for all calls (can be overridden per call)
-            posthog_privacy_mode: Default privacy mode for all calls (can be overridden per call)
+            posthog_privacy_mode: Default privacy mode for all calls (a call can enable it, not disable it)
             posthog_groups: Default groups for all calls (can be overridden per call)
             **kwargs: Additional arguments (for future compatibility)
         """
@@ -212,7 +212,7 @@ def generate_content(
             posthog_distinct_id: ID to associate with the usage event (overrides client default)
             posthog_trace_id: Trace UUID for linking events (auto-generated if not provided)
             posthog_properties: Extra properties to include in the event (merged with client defaults)
-            posthog_privacy_mode: Whether to redact sensitive information (overrides client default)
+            posthog_privacy_mode: Whether to redact sensitive information (cannot disable a client default of True)
             posthog_groups: Group analytics properties (overrides client default)
             **kwargs: Arguments passed to Gemini's generate_content
         """
@@ -384,7 +384,7 @@ def embed_content(
             posthog_distinct_id: ID to associate with the usage event (overrides client default)
             posthog_trace_id: Trace UUID for linking events (auto-generated if not provided)
             posthog_properties: Extra properties to include in the event (merged with client defaults)
-            posthog_privacy_mode: Whether to redact sensitive information (overrides client default)
+            posthog_privacy_mode: Whether to redact sensitive information (cannot disable a client default of True)
             posthog_groups: Group analytics properties (overrides client default)
             **kwargs: Arguments passed to Gemini's embed_content (e.g., config)
         """

posthog/ai/gemini/gemini_async.py

@@ -82,7 +82,7 @@ def __init__(
             posthog_client: PostHog client for tracking usage
             posthog_distinct_id: Default distinct ID for all calls (can be overridden per call)
             posthog_properties: Default properties for all calls (can be overridden per call)
-            posthog_privacy_mode: Default privacy mode for all calls (can be overridden per call)
+            posthog_privacy_mode: Default privacy mode for all calls (a call can enable it, not disable it)
             posthog_groups: Default groups for all calls (can be overridden per call)
             **kwargs: Additional arguments (for future compatibility)
         """
@@ -196,7 +196,7 @@ async def generate_content(
             posthog_distinct_id: ID to associate with the usage event (overrides client default)
             posthog_trace_id: Trace UUID for linking events (auto-generated if not provided)
             posthog_properties: Extra properties to include in the event (merged with client defaults)
-            posthog_privacy_mode: Whether to redact sensitive information (overrides client default)
+            posthog_privacy_mode: Whether to redact sensitive information (cannot disable a client default of True)
             posthog_groups: Group analytics properties (overrides client default)
             **kwargs: Arguments passed to Gemini's generate_content
         """
@@ -371,7 +371,7 @@ async def embed_content(
             posthog_distinct_id: ID to associate with the usage event (overrides client default)
             posthog_trace_id: Trace UUID for linking events (auto-generated if not provided)
             posthog_properties: Extra properties to include in the event (merged with client defaults)
-            posthog_privacy_mode: Whether to redact sensitive information (overrides client default)
+            posthog_privacy_mode: Whether to redact sensitive information (cannot disable a client default of True)
             posthog_groups: Group analytics properties (overrides client default)
             **kwargs: Arguments passed to Gemini's embed_content (e.g., config)
         """

posthog/ai/gemini/gemini_converter.py

@@ -13,7 +13,7 @@
     FormattedMessage,
     TokenUsage,
 )
-from posthog.ai.utils import serialize_raw_usage
+from posthog.ai.utils import _redact_mcp_tool_credentials, serialize_raw_usage
 
 _MEDIA_KINDS = {"image": "image", "video": "video", "audio": "audio"}
 _BARE_PART_KEYS = {
@@ -364,7 +364,7 @@ def extract_gemini_tools(kwargs: Dict[str, Any]) -> Optional[Any]:
     """
 
     if "config" in kwargs and hasattr(kwargs["config"], "tools"):
-        return kwargs["config"].tools
+        return _redact_mcp_tool_credentials(kwargs["config"].tools)
 
     return None
 

posthog/ai/langchain/callbacks.py

@@ -47,7 +47,10 @@
     _capture_ai_event,
     _extract_cache_creation_ttl_breakdown,
     finalize_ai_content,
+    _format_ai_error,
     get_model_params,
+    _privacy_mode_enabled,
+    _redact_mcp_tool_credentials,
     _responses_stop_reason,
     with_privacy_mode,
 )
@@ -560,14 +563,12 @@ def _capture_trace_or_span(
             event_properties.update(self._properties)
 
         if isinstance(outputs, BaseException):
-            event_properties["$ai_error"] = _stringify_exception(outputs)
+            event_properties["$ai_error"] = _format_ai_error(
+                self._ph_client, self._privacy_mode, outputs, _stringify_exception
+            )
             event_properties["$ai_is_error"] = True
-            event_properties = _capture_exception_and_update_properties(
-                self._ph_client,
-                outputs,
-                self._distinct_id,
-                self._groups,
-                event_properties,
+            event_properties = self._capture_exception_and_update_properties(
+                outputs, event_properties
             )
 
         elif outputs is not None:
@@ -588,6 +589,25 @@ def _capture_trace_or_span(
             groups=self._groups,
         )
 
+    def _capture_exception_and_update_properties(
+        self, exception: BaseException, event_properties: Dict[str, Any]
+    ) -> Dict[str, Any]:
+        # An `$exception` event carries the message and stack, which echo prompts and output.
+        if not self._ph_client.enable_exception_autocapture or _privacy_mode_enabled(
+            self._ph_client, self._privacy_mode
+        ):
+            return event_properties
+
+        exception_id = self._ph_client.capture_exception(
+            exception,
+            distinct_id=self._distinct_id,
+            groups=self._groups,
+            properties=event_properties,
+        )
+        if exception_id:
+            event_properties["$exception_event_id"] = exception_id
+        return event_properties
+
     def _pop_run_and_capture_generation(
         self,
         run_id: UUID,
@@ -657,7 +677,7 @@ def _capture_generation(
             event_properties.update(run.posthog_properties)
 
         if run.tools:
-            event_properties["$ai_tools"] = run.tools
+            event_properties["$ai_tools"] = _redact_mcp_tool_credentials(run.tools)
 
         if self._properties:
             event_properties.update(self._properties)
@@ -667,15 +687,13 @@ def _capture_generation(
 
         if isinstance(output, BaseException):
             event_properties["$ai_http_status"] = _get_http_status(output)
-            event_properties["$ai_error"] = _stringify_exception(output)
+            event_properties["$ai_error"] = _format_ai_error(
+                self._ph_client, self._privacy_mode, output, _stringify_exception
+            )
             event_properties["$ai_is_error"] = True
 
-            event_properties = _capture_exception_and_update_properties(
-                self._ph_client,
-                output,
-                self._distinct_id,
-                self._groups,
-                event_properties,
+            event_properties = self._capture_exception_and_update_properties(
+                output, event_properties
             )
         else:
             # Add usage
@@ -1011,27 +1029,6 @@ def _parse_usage(
     return llm_usage
 
 
-def _capture_exception_and_update_properties(
-    client: Client,
-    exception: BaseException,
-    distinct_id: Optional[Union[str, int, UUID]],
-    groups: Optional[Dict[str, Any]],
-    event_properties: Dict[str, Any],
-):
-    if client.enable_exception_autocapture:
-        exception_id = client.capture_exception(
-            exception,
-            distinct_id=distinct_id,
-            groups=groups,
-            properties=event_properties,
-        )
-
-        if exception_id:
-            event_properties["$exception_event_id"] = exception_id
-
-    return event_properties
-
-
 def _get_http_status(error: BaseException) -> int:
     # OpenAI: https://github.com/openai/openai-python/blob/main/src/openai/_exceptions.py
     # Anthropic: https://github.com/anthropics/anthropic-sdk-python/blob/main/src/anthropic/_exceptions.py

posthog/ai/langchain/middleware.py

@@ -21,6 +21,7 @@
 from langgraph.errors import GraphBubbleUp
 from langgraph.types import Command
 from ...client import Client
+from ..utils import _with_privacy_safe_message
 from .callbacks import (
     CallbackHandler,
     GenerationMetadata,
@@ -329,13 +330,10 @@ def _privacy_mode_enabled(self) -> bool:
             or getattr(self._callback._ph_client, "privacy_mode", False)
         )
 
-    def _returned_tool_error(self, response: ToolMessage) -> RuntimeError:
-        message = (
-            "Tool returned an error"
-            if self._privacy_mode_enabled()
-            else str(response.content)
-        )
-        return RuntimeError(message)
+    def _returned_tool_error(self, response: ToolMessage) -> BaseException:
+        if self._privacy_mode_enabled():
+            return _with_privacy_safe_message(RuntimeError("Tool returned an error"))
+        return RuntimeError(str(response.content))
 
     @staticmethod
     def _normalize_tool(tool: Any) -> dict[str, Any]:

posthog/ai/openai/openai_converter.py

@@ -17,7 +17,11 @@
     FormattedTextContent,
     TokenUsage,
 )
-from posthog.ai.utils import _responses_stop_reason, serialize_raw_usage
+from posthog.ai.utils import (
+    _responses_stop_reason,
+    _redact_mcp_tool_credentials,
+    serialize_raw_usage,
+)
 
 
 def _item_attr(item: Any, name: str, default: Any = None) -> Any:
@@ -299,7 +303,7 @@ def extract_openai_tools(kwargs: Dict[str, Any]) -> Optional[Any]:
 
     # Check for tools parameter (newer API)
     if "tools" in kwargs:
-        return kwargs["tools"]
+        return _redact_mcp_tool_credentials(kwargs["tools"])
 
     # Check for functions parameter (older API)
     if "functions" in kwargs:

posthog/ai/openai_agents/processor.py

@@ -25,12 +25,16 @@
 from posthog.ai.utils import (
     _capture_processor_event,
     finalize_ai_content,
+    _format_error_summary,
+    _privacy_mode_enabled,
     with_privacy_mode as _with_privacy_mode,
 )
 from posthog.client import Client
 
 log = logging.getLogger("posthog")
 
+_AUDIO_MODEL_CONFIG_CONTENT_KEYS = frozenset({"prompt", "instructions"})
+
 
 def _parse_iso_timestamp(iso_str: Optional[str]) -> Optional[float]:
     """Parse ISO timestamp to Unix timestamp."""
@@ -125,6 +129,15 @@ def _with_privacy_mode(self, value: Any) -> Any:
         """Apply privacy mode redaction if enabled."""
         return _with_privacy_mode(self._client, self._privacy_mode, value)
 
+    def _privacy_enabled(self) -> bool:
+        return _privacy_mode_enabled(self._client, self._privacy_mode)
+
+    @staticmethod
+    def _summarize_span_error(error_info: Any) -> str:
+        # Agents SDK span errors are plain dicts, not exceptions: only a type name is structured.
+        error_type = error_info.get("type") if isinstance(error_info, dict) else None
+        return _format_error_summary(error_type)
+
     def _evict_stale_entries(self) -> None:
         """Evict oldest entries if dicts exceed max size to prevent unbounded growth."""
         if len(self._span_start_times) > self._max_tracked_entries:
@@ -226,7 +239,9 @@ def on_trace_end(self, trace: Trace) -> None:
 
             # Include trace metadata if present
             if metadata:
-                properties["$ai_trace_metadata"] = _ensure_serializable(metadata)
+                properties["$ai_trace_metadata"] = self._with_privacy_mode(
+                    _ensure_serializable(metadata)
+                )
 
             if distinct_id is None:
                 properties["$process_person_profile"] = False
@@ -313,7 +328,11 @@ def on_span_end(self, span: Span[Any]) -> None:
 
                 error_properties = {
                     "$ai_is_error": True,
-                    "$ai_error": error_message,
+                    "$ai_error": (
+                        self._summarize_span_error(error_info)
+                        if self._privacy_enabled()
+                        else error_message
+                    ),
                     "$ai_error_type": error_type,
                 }
 
@@ -778,7 +797,9 @@ def _handle_audio_span(
 
         # Add model config if available (pass-through property)
         if hasattr(span_data, "model_config") and span_data.model_config:
-            properties["model_config"] = _ensure_serializable(span_data.model_config)
+            properties["model_config"] = self._redact_audio_model_config(
+                _ensure_serializable(span_data.model_config)
+            )
 
         # Add time to first audio byte for speech spans (pass-through property)
         if hasattr(span_data, "first_content_at") and span_data.first_content_at:
@@ -825,6 +846,15 @@ def _handle_audio_span(
 
         self._capture_event("$ai_span", properties, distinct_id)
 
+    def _redact_audio_model_config(self, model_config: Any) -> Any:
+        # The STT prompt and TTS instructions are user-authored text.
+        if not isinstance(model_config, dict) or not self._privacy_enabled():
+            return model_config
+        return {
+            key: (None if key in _AUDIO_MODEL_CONFIG_CONTENT_KEYS else value)
+            for key, value in model_config.items()
+        }
+
     def _handle_mcp_span(
         self,
         span_data: MCPListToolsSpanData,