diff --git a/.changelog/5332.added b/.changelog/5332.added new file mode 100644 index 0000000000..d8e9a2dd66 --- /dev/null +++ b/.changelog/5332.added @@ -0,0 +1 @@ +`opentelemetry-sdk`: add support for parameter `exclude_attribute_keys` in `View` diff --git a/opentelemetry-configuration/src/opentelemetry/configuration/_meter_provider.py b/opentelemetry-configuration/src/opentelemetry/configuration/_meter_provider.py index d7eb75f780..e7d0084bad 100644 --- a/opentelemetry-configuration/src/opentelemetry/configuration/_meter_provider.py +++ b/opentelemetry-configuration/src/opentelemetry/configuration/_meter_provider.py @@ -231,14 +231,13 @@ def _create_view(config: ViewConfig) -> View: if instrument_type is None: raise ConfigurationError(f"Unknown instrument type: {selector.instrument_type!r}") - attribute_keys: set[str] | None = None + attribute_keys: list[str] | None = None + exclude_attribute_keys: list[str] | None = None if stream.attribute_keys is not None: - if stream.attribute_keys.excluded: - _logger.warning( - "attribute_keys.excluded is not supported by the Python SDK View; the exclusion list will be ignored." - ) + if stream.attribute_keys.excluded is not None: + exclude_attribute_keys = stream.attribute_keys.excluded if stream.attribute_keys.included is not None: - attribute_keys = set(stream.attribute_keys.included) + attribute_keys = stream.attribute_keys.included aggregation = None if stream.aggregation is not None: @@ -255,6 +254,7 @@ def _create_view(config: ViewConfig) -> View: description=stream.description, attribute_keys=attribute_keys, aggregation=aggregation, + exclude_attribute_keys=exclude_attribute_keys, ) diff --git a/opentelemetry-configuration/tests/test_meter_provider.py b/opentelemetry-configuration/tests/test_meter_provider.py index 2c5fceb6c0..b24423a8b1 100644 --- a/opentelemetry-configuration/tests/test_meter_provider.py +++ b/opentelemetry-configuration/tests/test_meter_provider.py @@ -868,11 +868,14 @@ def test_stream_attribute_keys_included(self): ) self.assertEqual(view._attribute_keys, {"key1", "key2"}) - def test_stream_attribute_keys_excluded_logs_warning(self): + def test_stream_attribute_keys_excluded_is_applied(self): config = self._make_view_config(stream_kwargs={"attribute_keys": IncludeExclude(excluded=["key1"])}) - with self.assertLogs("opentelemetry.configuration._meter_provider", level="WARNING") as log: - create_meter_provider(config) - self.assertTrue(any("excluded" in msg for msg in log.output)) + meter_provider = create_meter_provider(config) + views = meter_provider._sdk_config.views + self.assertEqual(len(views), 1) + view = views[0] + self.assertEqual(view._exclude_attribute_keys, frozenset({"key1"})) + self.assertIsNone(view._attribute_keys) def test_stream_aggregation_drop(self): view = self._get_view(self._make_view_config(stream_kwargs={"aggregation": AggregationConfig(drop={})})) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/_view_instrument_match.py b/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/_view_instrument_match.py index 06a45f8d2d..ea5ce841da 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/_view_instrument_match.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/_view_instrument_match.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from collections.abc import Mapping, Sequence +from fnmatch import fnmatchcase from logging import getLogger from threading import Lock from time import time_ns @@ -130,7 +131,19 @@ def consume_measurement(self, measurement: Measurement, should_sample_exemplar: # leading to unexpected behavior, but deep copying is expensive. attributes = dict(measurement.attributes) if self._view._attribute_keys is not None: - attributes = {k: v for k, v in attributes.items() if k in self._view._attribute_keys} + attributes = { + key: value + for key, value in attributes.items() + if any(fnmatchcase(key, pattern) for pattern in self._view._attribute_keys) + } + elif self._view._exclude_attribute_keys: + attributes = { + key: value + for key, value in attributes.items() + if not any(fnmatchcase(key, pattern) for pattern in self._view._exclude_attribute_keys) + } + else: + attributes = {} aggr_key = _hash_attributes(attributes) diff --git a/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/view.py b/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/view.py index 9b60a28d62..a9d840b9f0 100644 --- a/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/view.py +++ b/opentelemetry-sdk/src/opentelemetry/sdk/metrics/_internal/view.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 -from collections.abc import Callable +from collections.abc import Callable, Iterable from fnmatch import fnmatchcase from logging import getLogger @@ -90,6 +90,16 @@ class View: instrument_unit: This is an instrument matching attribute: the unit the instrument must have to match the view. + exclude_attribute_keys: This is a metric stream customizing attribute: this is + a set of attribute keys. If not `None` then measurement attributes whose + keys are in ``exclude_attribute_keys`` will be removed before identifying + the metric stream. + Matching is case-sensitive. Values are evaluated to match as follows: + - If the value exactly matches. + - If the value matches the wildcard pattern, where '?' matches any single + character and '*' matches any number of characters including none. + + This class is not intended to be subclassed by the user. """ @@ -104,10 +114,11 @@ def __init__( meter_schema_url: str | None = None, name: str | None = None, description: str | None = None, - attribute_keys: set[str] | None = None, + attribute_keys: Iterable[str] | None = None, aggregation: Aggregation | None = None, exemplar_reservoir_factory: Callable[[type[_Aggregation]], ExemplarReservoirBuilder] | None = None, instrument_unit: str | None = None, + exclude_attribute_keys: Iterable[str] | None = None, ): if ( instrument_type @@ -130,7 +141,28 @@ def __init__( if any(c in instrument_name for c in "*?["): # pylint: disable=broad-exception-raised raise Exception(f"View {name} declared with wildcard characters in instrument_name") + + attribute_keys = frozenset(attribute_keys) if attribute_keys is not None else None + + exclude_attribute_keys = frozenset(exclude_attribute_keys) if exclude_attribute_keys is not None else None + + if attribute_keys is not None and exclude_attribute_keys is not None: + overlap = [ + (attribute_key, exclude_attribute_key) + for attribute_key in attribute_keys + for exclude_attribute_key in exclude_attribute_keys + if ( + fnmatchcase(attribute_key, exclude_attribute_key) + or fnmatchcase( + exclude_attribute_key, + attribute_key, + ) + ) + ] + if overlap: + # pylint: disable=broad-exception-raised + raise ValueError(f"attribute_keys and exclude_attribute_keys contain overlapping patterns: {overlap}") # _name, _description, _aggregation, _exemplar_reservoir_factory and # _attribute_keys will be accessed when instantiating a _ViewInstrumentMatch. self._name = name @@ -145,6 +177,7 @@ def __init__( self._attribute_keys = attribute_keys self._aggregation = aggregation or self._default_aggregation self._exemplar_reservoir_factory = exemplar_reservoir_factory or _default_reservoir_factory + self._exclude_attribute_keys = exclude_attribute_keys # pylint: disable=too-many-return-statements # pylint: disable=too-many-branches diff --git a/opentelemetry-sdk/tests/metrics/test_view.py b/opentelemetry-sdk/tests/metrics/test_view.py index f24e035d0a..7ef6b57097 100644 --- a/opentelemetry-sdk/tests/metrics/test_view.py +++ b/opentelemetry-sdk/tests/metrics/test_view.py @@ -127,6 +127,47 @@ def test_view_name(self): self.assertEqual(view._name, "name") self.assertEqual(view._instrument_name, "instrument_name_1") + def test_attribute_keys_and_exclude_attribute_keys_overlap(self): + test_cases = [ + { + "name": "exact overlap", + "attribute_keys": ("method", "status_code"), + "exclude_attribute_keys": ("method", "user_id"), + }, + { + "name": "wildcard include overlaps exact exclude", + "attribute_keys": ("k8s.*",), + "exclude_attribute_keys": ("k8s.node",), + }, + { + "name": "exact include overlaps wildcard exclude", + "attribute_keys": ("k8s.node",), + "exclude_attribute_keys": ("k8s.*",), + }, + { + "name": "wildcard overlaps wildcard", + "attribute_keys": ("k8s.*",), + "exclude_attribute_keys": ("k8s.n*",), + }, + ] + + for case in test_cases: + with self.subTest(case=case["name"]): + with self.assertRaises(ValueError): + View( + instrument_name="instrument_name", + attribute_keys=case["attribute_keys"], + exclude_attribute_keys=case["exclude_attribute_keys"], + ) + + def test_attribute_keys_and_exclude_attribute_keys_disjoint(self): + view = View( + instrument_name="instrument_name", + attribute_keys=("method", "status_code"), + exclude_attribute_keys=("user_id",), + ) + + self.assertIsNotNone(view) def test_view_name_without_instrument_name(self): cases = [ ("custom_name", {"instrument_type": Mock}), diff --git a/opentelemetry-sdk/tests/metrics/test_view_instrument_match.py b/opentelemetry-sdk/tests/metrics/test_view_instrument_match.py index 8985a339d3..6df0ab7cc8 100644 --- a/opentelemetry-sdk/tests/metrics/test_view_instrument_match.py +++ b/opentelemetry-sdk/tests/metrics/test_view_instrument_match.py @@ -73,6 +73,90 @@ def setUpClass(cls): views=[], ) + def test_consume_measurement_with_exclude_attribute_keys(self): + test_cases = [ + { + "name": "excluded attribute removed", + "exclude_attribute_keys": {"f"}, + "measurement_attributes": {"c": "d", "f": "g"}, + "expected_attributes": {"c": "d"}, + }, + { + "name": "wildcard exclusion", + "exclude_attribute_keys": {"k8s.*"}, + "measurement_attributes": { + "k8s.node": "node1", + "k8s.cluster": "cluster1", + "service.name": "svc", + }, + "expected_attributes": { + "service.name": "svc", + }, + }, + { + "name": "question mark exclusion", + "exclude_attribute_keys": {"node?"}, + "measurement_attributes": { + "node1": "a", + "node2": "b", + "node12": "c", + }, + "expected_attributes": { + "node12": "c", + }, + }, + { + "name": "wildcard exact exclusion", + "exclude_attribute_keys": {"*.node"}, + "measurement_attributes": { + "k8s.node": "node1", + "k8s.cluster": "cluster1", + }, + "expected_attributes": { + "k8s.cluster": "cluster1", + }, + }, + { + "name": "none attributes become empty attributes", + "exclude_attribute_keys": {"f"}, + "measurement_attributes": None, + "expected_attributes": {}, + }, + ] + + for case in test_cases: + with self.subTest(case=case["name"]): + instrument1 = Mock(name="instrument1") + instrument1.instrumentation_scope = self.mock_instrumentation_scope + + view_instrument_match = _ViewInstrumentMatch( + view=View( + instrument_name="instrument1", + name="name", + aggregation=self.mock_aggregation_factory, + exclude_attribute_keys=case["exclude_attribute_keys"], + ), + instrument=instrument1, + instrument_class_aggregation=MagicMock(**{"__getitem__.return_value": DefaultAggregation()}), + ) + + view_instrument_match.consume_measurement( + Measurement( + value=0, + time_unix_nano=time_ns(), + instrument=instrument1, + context=Context(), + attributes=case["measurement_attributes"], + ) + ) + + self.assertEqual( + view_instrument_match._attributes_aggregation, + { + _hash_attributes(case["expected_attributes"]): (self.mock_created_aggregation), + }, + ) + def test_consume_measurement(self): instrument1 = Mock(name="instrument1") instrument1.instrumentation_scope = self.mock_instrumentation_scope @@ -119,6 +203,44 @@ def test_consume_measurement(self): }, ) + # wildcard attribute_keys should match multiple attributes + view_instrument_match = _ViewInstrumentMatch( + view=View( + instrument_name="instrument1", + name="name", + aggregation=self.mock_aggregation_factory, + attribute_keys={"k8s.*"}, + ), + instrument=instrument1, + instrument_class_aggregation=MagicMock(**{"__getitem__.return_value": DefaultAggregation()}), + ) + + view_instrument_match.consume_measurement( + Measurement( + value=0, + time_unix_nano=time_ns(), + instrument=instrument1, + context=Context(), + attributes={ + "k8s.node": "node1", + "k8s.cluster": "cluster1", + "service.name": "svc", + }, + ) + ) + + self.assertEqual( + view_instrument_match._attributes_aggregation, + { + _hash_attributes( + { + "k8s.node": "node1", + "k8s.cluster": "cluster1", + } + ): self.mock_created_aggregation, + }, + ) + # setup new instrument match without `attribute_keys` set, defaults to keeping # all attributes. view_instrument_match = _ViewInstrumentMatch(