From 65f23c10791d2e2043897a191ee66781166efc27 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 28 Sep 2026 16:37:07 +0000 Subject: [PATCH 1/2] fix(otel): send lazily resolved API key with opentelemetry-exporter-otlp-proto-http 1.45 opentelemetry-exporter-otlp-proto-http 1.45.0 moved request headers off the exporter (`_headers` / `_session`) and onto an internal `_client`. When the Braintrust API key was resolved after the exporter was constructed (e.g. from `.env.braintrust` or an env var set later), `OtelExporter._set_api_key_header` updated attributes the exporter no longer reads, so spans were exported without an `Authorization` header. Instead of patching upstream's private header storage, re-run the public `OTLPSpanExporter.__init__` with the resolved `Authorization` header. Track `shutdown()` so resolving a key cannot revive an exporter that was already shut down. Replace the tests that asserted on private exporter state with tests that export to a local OTLP collector (the shared `scripted_server` helper) and check the headers it receives. Add an OpenTelemetry version matrix for `test_otel` (latest=1.45.0, 1.44.0, 1.16.0) so both the pre- and post-1.45 exporter layouts stay covered. 1.16.0 is the oldest release that imports without pkg_resources; it is skipped on Python 3.14 because OpenTelemetry <1.28 requires protobuf<5. Co-Authored-By: Claude Opus 5.5 --- py/noxfile.py | 14 ++- py/pyproject.toml | 19 ++++ py/scripts/session-weights.json | 4 +- py/src/braintrust/otel/__init__.py | 29 +++--- py/src/braintrust/test_otel.py | 147 ++++++++++++++++++++++++----- 5 files changed, 174 insertions(+), 39 deletions(-) diff --git a/py/noxfile.py b/py/noxfile.py index 58df546f0..dc06bb9d1 100644 --- a/py/noxfile.py +++ b/py/noxfile.py @@ -822,12 +822,20 @@ def test_cli(session): _run_tests(session, DEVSERVER_DIR) +OTEL_VERSIONS = _get_matrix_versions("opentelemetry-sdk") + + @nox.session() -def test_otel(session): +@nox.parametrize("version", OTEL_VERSIONS, ids=OTEL_VERSIONS) +def test_otel(session, version): """Test OtelExporter with OpenTelemetry installed.""" + if version != LATEST and Version(version) < Version("1.28.0") and sys.version_info >= (3, 14): + session.skip("OpenTelemetry <1.28 requires protobuf<5, which does not support Python 3.14") _install_test_deps(session) - session.install(".[otel]") - _run_tests(session, "braintrust/test_otel.py") + _install_matrix_dep(session, "opentelemetry-api", version) + _install_matrix_dep(session, "opentelemetry-sdk", version) + _install_matrix_dep(session, "opentelemetry-exporter-otlp-proto-http", version) + _run_tests(session, "braintrust/test_otel.py", version=version) @nox.session() diff --git a/py/pyproject.toml b/py/pyproject.toml index d2aefa700..22cc895c1 100644 --- a/py/pyproject.toml +++ b/py/pyproject.toml @@ -554,6 +554,25 @@ latest = "boto3==1.43.98" latest = "botocore==1.43.98" "1.34.116" = "botocore==1.34.116" +# OpenTelemetry api/sdk/exporter release in lockstep; keep their keys in sync. +# 1.45.0 moved OTLP HTTP exporter headers onto an internal client, so keep a +# pre-1.45 version covered alongside latest. 1.16.0 is the oldest release that +# imports without pkg_resources; test_otel skips it on Python 3.14 (protobuf<5). +[tool.braintrust.matrix.opentelemetry-api] +latest = "opentelemetry-api==1.45.0" +"1.44.0" = "opentelemetry-api==1.44.0" +"1.16.0" = "opentelemetry-api==1.16.0" + +[tool.braintrust.matrix.opentelemetry-sdk] +latest = "opentelemetry-sdk==1.45.0" +"1.44.0" = "opentelemetry-sdk==1.44.0" +"1.16.0" = "opentelemetry-sdk==1.16.0" + +[tool.braintrust.matrix.opentelemetry-exporter-otlp-proto-http] +latest = "opentelemetry-exporter-otlp-proto-http==1.45.0" +"1.44.0" = "opentelemetry-exporter-otlp-proto-http==1.44.0" +"1.16.0" = "opentelemetry-exporter-otlp-proto-http==1.16.0" + # --------------------------------------------------------------------------- # Vendor packages — optional third-party packages the SDK can work without. # Keys are matrix keys; values are Python import names. The noxfile uses this diff --git a/py/scripts/session-weights.json b/py/scripts/session-weights.json index 44dc7274a..a475b6f4f 100644 --- a/py/scripts/session-weights.json +++ b/py/scripts/session-weights.json @@ -71,7 +71,9 @@ "test_openai_http2_streaming(latest)": 12, "test_openrouter(0.6.0)": 8, "test_openrouter(latest)": 12, - "test_otel": 9, + "test_otel(1.16.0)": 9, + "test_otel(1.44.0)": 9, + "test_otel(latest)": 9, "test_otel_not_installed": 5, "test_pydantic_ai_integration(1.10.0)": 34, "test_pydantic_ai_integration(latest)": 27, diff --git a/py/src/braintrust/otel/__init__.py b/py/src/braintrust/otel/__init__.py index 84946a412..7d458eb75 100644 --- a/py/src/braintrust/otel/__init__.py +++ b/py/src/braintrust/otel/__init__.py @@ -39,6 +39,9 @@ def export(self, *args, **kwargs): def force_flush(self, *args, **kwargs): raise ImportError(INSTALL_ERR_MSG) + def shutdown(self, *args, **kwargs): + raise ImportError(INSTALL_ERR_MSG) + class BatchSpanProcessor: def __init__(self, *args, **kwargs): raise ImportError(INSTALL_ERR_MSG) @@ -248,7 +251,6 @@ def __init__( ) self._braintrust_api_key_arg = api_key_arg - self._braintrust_headers_override_authorization = "Authorization" in headers self._braintrust_has_api_key = bool(api_key and api_key.strip()) exporter_headers = {} @@ -261,20 +263,17 @@ def __init__( self.parent = parent - super().__init__(endpoint=endpoint, headers=exporter_headers, **kwargs) + self._braintrust_exporter_kwargs = {"endpoint": endpoint, "headers": exporter_headers, **kwargs} + self._braintrust_shutdown = False + super().__init__(**self._braintrust_exporter_kwargs) def _set_api_key_header(self, api_key: str) -> None: - if not self._braintrust_headers_override_authorization: - authorization = {"Authorization": f"Bearer {api_key}"} - exporter_headers = getattr(self, "_headers", None) - if isinstance(exporter_headers, dict): - exporter_headers.update(authorization) - else: - self._headers = {**dict(exporter_headers or {}), **authorization} - - session = getattr(self, "_session", None) - if session is not None: - session.headers.update(authorization) + exporter_kwargs = self._braintrust_exporter_kwargs + if "Authorization" not in exporter_kwargs["headers"] and not self._braintrust_shutdown: + # Re-run the upstream constructor instead of patching its private header + # storage, which moved in opentelemetry-exporter-otlp-proto-http 1.45. + headers = {"Authorization": f"Bearer {api_key}", **exporter_kwargs["headers"]} + super().__init__(**{**exporter_kwargs, "headers": headers}) self._braintrust_has_api_key = True def _ensure_api_key(self) -> None: @@ -298,6 +297,10 @@ def force_flush(self, timeout_millis=30000): self._ensure_api_key() return super().force_flush(timeout_millis) + def shutdown(self): + self._braintrust_shutdown = True + return super().shutdown() + def add_braintrust_span_processor( tracer_provider, diff --git a/py/src/braintrust/test_otel.py b/py/src/braintrust/test_otel.py index 2ad206b35..9dbfd1c6b 100644 --- a/py/src/braintrust/test_otel.py +++ b/py/src/braintrust/test_otel.py @@ -1,8 +1,10 @@ # pylint: disable=not-context-manager import json import sys +from collections.abc import Iterator import pytest +from braintrust.api._test_server import scripted_server def _check_otel_installed(): @@ -20,6 +22,38 @@ def _check_otel_installed(): OTEL_INSTALLED = _check_otel_installed() +_collector_requests: list[tuple[str, dict[str, str]]] = [] + + +def _record_otlp_request(method, path, body, headers): + _collector_requests.append((path, {key.lower(): value for key, value in headers.items()})) + return 200, {}, b"" + + +@pytest.fixture +def otlp_collector() -> Iterator[str]: + """Local OTLP/HTTP endpoint that records the path and headers of each export request.""" + with scripted_server(_record_otlp_request) as (url, _): + yield url + + +def _export_span(exporter) -> list[tuple[str, dict[str, str]]]: + """Export a single span through *exporter* and return the (path, lowercased headers) the collector received.""" + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import SimpleSpanProcessor + + _collector_requests.clear() + provider = TracerProvider(shutdown_on_exit=False) + provider.add_span_processor(SimpleSpanProcessor(exporter)) + provider.get_tracer(__name__).start_span("test-span").end() + return list(_collector_requests) + + +def _export_span_and_get_request(exporter) -> tuple[str, dict[str, str]]: + [request] = _export_span(exporter) + return request + + @pytest.fixture def uninstall_braintrust_otel(): sys.modules.pop("braintrust.otel", None) @@ -71,14 +105,14 @@ def test_otel_exporter_creation(tmp_path): OtelExporter(api_key="fake-key") -def test_otel_exporter_with_explicit_params(): +def test_otel_exporter_with_explicit_params(otlp_collector): if not _check_otel_installed(): pytest.skip("OpenTelemetry SDK not fully installed, skipping test") from braintrust.otel import OtelExporter exporter = OtelExporter( - url="https://custom.example.com/otel/v1/traces", + url=f"{otlp_collector}/custom/v1/traces", api_key="explicit-api-key", parent="project_name:explicit-test", headers={"custom-header": "custom-value"}, @@ -86,17 +120,14 @@ def test_otel_exporter_with_explicit_params(): assert exporter.parent == "project_name:explicit-test" - # Check endpoint and headers - assert exporter._endpoint == "https://custom.example.com/otel/v1/traces" - expected_headers = { - "Authorization": "Bearer explicit-api-key", - "x-bt-parent": "project_name:explicit-test", - "custom-header": "custom-value", - } - assert exporter._headers == expected_headers + path, headers = _export_span_and_get_request(exporter) + assert path == "/custom/v1/traces" + assert headers["authorization"] == "Bearer explicit-api-key" + assert headers["x-bt-parent"] == "project_name:explicit-test" + assert headers["custom-header"] == "custom-value" -def test_otel_exporter_uses_env_braintrust_api_key(tmp_path): +def test_otel_exporter_uses_env_braintrust_api_key(tmp_path, otlp_collector): if not _check_otel_installed(): pytest.skip("OpenTelemetry SDK not fully installed, skipping test") @@ -107,10 +138,64 @@ def test_otel_exporter_uses_env_braintrust_api_key(tmp_path): m.chdir(tmp_path) (tmp_path / ".env.braintrust").write_text("BRAINTRUST_API_KEY=file-api-key\n") - exporter = OtelExporter(parent="project_name:test") - exporter.force_flush() + exporter = OtelExporter(url=f"{otlp_collector}/otel/v1/traces", parent="project_name:test") + + _, headers = _export_span_and_get_request(exporter) + assert headers["authorization"] == "Bearer file-api-key" + assert headers["x-bt-parent"] == "project_name:test" + + +def test_otel_exporter_sends_api_key_resolved_after_construction(otlp_collector): + if not _check_otel_installed(): + pytest.skip("OpenTelemetry SDK not fully installed, skipping test") + + from braintrust.otel import OtelExporter + + with pytest.MonkeyPatch.context() as m: + m.delenv("BRAINTRUST_API_KEY", raising=False) + exporter = OtelExporter(url=f"{otlp_collector}/otel/v1/traces", parent="project_name:test") + + m.setenv("BRAINTRUST_API_KEY", "late-api-key") + _, headers = _export_span_and_get_request(exporter) + assert headers["authorization"] == "Bearer late-api-key" + + # Subsequent exports keep sending the resolved key. + _, headers = _export_span_and_get_request(exporter) + assert headers["authorization"] == "Bearer late-api-key" + + +def test_otel_exporter_resolving_api_key_does_not_revive_shutdown_exporter(otlp_collector): + if not _check_otel_installed(): + pytest.skip("OpenTelemetry SDK not fully installed, skipping test") + + from braintrust.otel import OtelExporter + + with pytest.MonkeyPatch.context() as m: + m.delenv("BRAINTRUST_API_KEY", raising=False) + exporter = OtelExporter(url=f"{otlp_collector}/otel/v1/traces", parent="project_name:test") + exporter.shutdown() + + m.setenv("BRAINTRUST_API_KEY", "late-api-key") + assert _export_span(exporter) == [] + + +def test_otel_exporter_lazy_api_key_does_not_override_explicit_authorization_header(otlp_collector): + if not _check_otel_installed(): + pytest.skip("OpenTelemetry SDK not fully installed, skipping test") + + from braintrust.otel import OtelExporter + + with pytest.MonkeyPatch.context() as m: + m.delenv("BRAINTRUST_API_KEY", raising=False) + exporter = OtelExporter( + url=f"{otlp_collector}/otel/v1/traces", + parent="project_name:test", + headers={"Authorization": "Bearer custom-auth"}, + ) - assert exporter._headers["Authorization"] == "Bearer file-api-key" + m.setenv("BRAINTRUST_API_KEY", "late-api-key") + _, headers = _export_span_and_get_request(exporter) + assert headers["authorization"] == "Bearer custom-auth" def test_braintrust_span_processor_merges_span_origin_with_context_json_set_after_start(): @@ -360,8 +445,6 @@ def test_braintrust_api_url_env_var(): exporter = OtelExporter() assert exporter._endpoint == "https://api.braintrust.dev/otel/v1/traces" - expected_headers = {"Authorization": "Bearer test-api-key", "x-bt-parent": "project_name:test"} - assert exporter._headers == expected_headers # Test custom API URL with pytest.MonkeyPatch.context() as m: @@ -372,8 +455,6 @@ def test_braintrust_api_url_env_var(): exporter = OtelExporter() assert exporter._endpoint == "https://custom.braintrust.dev/otel/v1/traces" - expected_headers = {"Authorization": "Bearer custom-key", "x-bt-parent": "project_name:default-otel-project"} - assert exporter._headers == expected_headers # Test custom API URL with trailing slash with pytest.MonkeyPatch.context() as m: @@ -386,6 +467,25 @@ def test_braintrust_api_url_env_var(): assert exporter._endpoint == "https://custom.example.com/otel/v1/traces" +def test_otel_exporter_sends_env_api_key_and_default_parent(otlp_collector): + if not _check_otel_installed(): + pytest.skip("OpenTelemetry SDK not fully installed, skipping test") + + from braintrust.otel import OtelExporter + + with pytest.MonkeyPatch.context() as m: + m.setenv("BRAINTRUST_API_KEY", "env-api-key") + m.setenv("BRAINTRUST_API_URL", otlp_collector) + m.delenv("BRAINTRUST_PARENT", raising=False) + + exporter = OtelExporter() + + path, headers = _export_span_and_get_request(exporter) + assert path == "/otel/v1/traces" + assert headers["authorization"] == "Bearer env-api-key" + assert headers["x-bt-parent"] == "project_name:default-otel-project" + + def test_braintrust_otel_filter_ai_spans_environment_variable(): if not _check_otel_installed(): pytest.skip("OpenTelemetry SDK not fully installed, skipping test") @@ -434,7 +534,7 @@ def test_braintrust_otel_filter_ai_spans_environment_variable(): os.environ.pop("BRAINTRUST_OTEL_FILTER_AI_SPANS", None) -def test_braintrust_span_processor_class(): +def test_braintrust_span_processor_class(otlp_collector): if not _check_otel_installed(): pytest.skip("OpenTelemetry SDK not fully installed, skipping test") @@ -483,7 +583,7 @@ def custom_filter(span): processor_custom = BraintrustSpanProcessor( api_key="explicit-key", parent="project:test", - api_url="https://custom.example.com", + api_url=otlp_collector, filter_ai_spans=True, custom_filter=custom_filter, headers={"X-Test-Header": "test"}, @@ -498,8 +598,11 @@ def custom_filter(span): # Check that the exporter was created with the right parameters exporter = processor_custom.exporter assert exporter.parent == "project:test" - assert exporter._endpoint == "https://custom.example.com/otel/v1/traces" - assert exporter._headers["Authorization"] == "Bearer explicit-key" + path, headers = _export_span_and_get_request(exporter) + assert path == "/otel/v1/traces" + assert headers["authorization"] == "Bearer explicit-key" + assert headers["x-bt-parent"] == "project:test" + assert headers["x-test-header"] == "test" class TestSpanFiltering: From e5490c11d44a9815d2e20e21c7f82462878b84ad Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Mon, 28 Sep 2026 17:59:17 +0000 Subject: [PATCH 2/2] fix(otel): synchronize lazy API key re-init with shutdown When the first lazy-key export raced with shutdown() (e.g. a SimpleSpanProcessor ending a span on one thread while the provider shuts down on another), shutdown could land between the shutdown check and the upstream re-init. Re-running OTLPSpanExporter.__init__ then reset the upstream shutdown state and recreated the HTTP client, so the exporter kept sending after it had been shut down. Guard the check/re-init and the shutdown transition with one lock, and re-check the resolved-key latch under it so concurrent first exports only re-init once. Co-Authored-By: Claude Opus 5.5 --- py/src/braintrust/otel/__init__.py | 25 ++++++++++++-------- py/src/braintrust/test_otel.py | 37 ++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/py/src/braintrust/otel/__init__.py b/py/src/braintrust/otel/__init__.py index 7d458eb75..85ac21599 100644 --- a/py/src/braintrust/otel/__init__.py +++ b/py/src/braintrust/otel/__init__.py @@ -1,6 +1,7 @@ import json import logging import os +import threading import warnings from urllib.parse import urljoin @@ -265,16 +266,21 @@ def __init__( self._braintrust_exporter_kwargs = {"endpoint": endpoint, "headers": exporter_headers, **kwargs} self._braintrust_shutdown = False + # Guards lazy re-init against a concurrent shutdown(), which re-init would undo. + self._braintrust_lifecycle_lock = threading.Lock() super().__init__(**self._braintrust_exporter_kwargs) def _set_api_key_header(self, api_key: str) -> None: - exporter_kwargs = self._braintrust_exporter_kwargs - if "Authorization" not in exporter_kwargs["headers"] and not self._braintrust_shutdown: - # Re-run the upstream constructor instead of patching its private header - # storage, which moved in opentelemetry-exporter-otlp-proto-http 1.45. - headers = {"Authorization": f"Bearer {api_key}", **exporter_kwargs["headers"]} - super().__init__(**{**exporter_kwargs, "headers": headers}) - self._braintrust_has_api_key = True + with self._braintrust_lifecycle_lock: + if self._braintrust_has_api_key: + return + exporter_kwargs = self._braintrust_exporter_kwargs + if "Authorization" not in exporter_kwargs["headers"] and not self._braintrust_shutdown: + # Re-run the upstream constructor instead of patching its private header + # storage, which moved in opentelemetry-exporter-otlp-proto-http 1.45. + headers = {"Authorization": f"Bearer {api_key}", **exporter_kwargs["headers"]} + super().__init__(**{**exporter_kwargs, "headers": headers}) + self._braintrust_has_api_key = True def _ensure_api_key(self) -> None: if self._braintrust_has_api_key: @@ -298,8 +304,9 @@ def force_flush(self, timeout_millis=30000): return super().force_flush(timeout_millis) def shutdown(self): - self._braintrust_shutdown = True - return super().shutdown() + with self._braintrust_lifecycle_lock: + self._braintrust_shutdown = True + return super().shutdown() def add_braintrust_span_processor( diff --git a/py/src/braintrust/test_otel.py b/py/src/braintrust/test_otel.py index 9dbfd1c6b..5c9688c2f 100644 --- a/py/src/braintrust/test_otel.py +++ b/py/src/braintrust/test_otel.py @@ -179,6 +179,43 @@ def test_otel_exporter_resolving_api_key_does_not_revive_shutdown_exporter(otlp_ assert _export_span(exporter) == [] +def test_otel_exporter_shutdown_racing_lazy_api_key_reinit_stays_shutdown(otlp_collector): + if not _check_otel_installed(): + pytest.skip("OpenTelemetry SDK not fully installed, skipping test") + + import threading + + from braintrust.otel import OtelExporter + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + + upstream_init = OTLPSpanExporter.__init__ + + def init_with_concurrent_shutdown(self, *args, **kwargs): + # Run shutdown() on another thread while the lazy-key re-init is in progress. + # Give it a bounded chance to finish first, so the test also terminates when + # shutdown correctly waits for the re-init. + shutdown_thread = threading.Thread(target=self.shutdown) + shutdown_thread.start() + shutdown_thread.join(timeout=0.5) + upstream_init(self, *args, **kwargs) + shutdown_threads.append(shutdown_thread) + + shutdown_threads = [] + with pytest.MonkeyPatch.context() as m: + m.delenv("BRAINTRUST_API_KEY", raising=False) + exporter = OtelExporter(url=f"{otlp_collector}/otel/v1/traces", parent="project_name:test") + + m.setenv("BRAINTRUST_API_KEY", "late-api-key") + m.setattr(OTLPSpanExporter, "__init__", init_with_concurrent_shutdown) + _export_span(exporter) + for shutdown_thread in shutdown_threads: + shutdown_thread.join(timeout=5) + assert not shutdown_thread.is_alive() + + assert shutdown_threads + assert _export_span(exporter) == [] + + def test_otel_exporter_lazy_api_key_does_not_override_explicit_authorization_header(otlp_collector): if not _check_otel_installed(): pytest.skip("OpenTelemetry SDK not fully installed, skipping test")