From cd9a50322ba948d427e33477a31c11471eed91f8 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Fri, 18 Jul 2025 15:39:59 -0400 Subject: [PATCH 1/4] fix --- ddtrace/llmobs/_integrations/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ddtrace/llmobs/_integrations/utils.py b/ddtrace/llmobs/_integrations/utils.py index eb374c5ac17..707618d0c26 100644 --- a/ddtrace/llmobs/_integrations/utils.py +++ b/ddtrace/llmobs/_integrations/utils.py @@ -1189,7 +1189,7 @@ def _compute_completion_tokens(completions_or_messages, model_name): estimated = False num_completion_tokens = 0 for choice in completions_or_messages: - content = choice.get("content", "") or choice.get("text", "") + content = _get_attr(choice, "content", "") or _get_attr(choice, "text", "") estimated, completion_tokens = _compute_token_count(content, model_name) num_completion_tokens += completion_tokens return estimated, num_completion_tokens From 8cedce9e0bab661d50c7fbfdc50c8c82b37c5593 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Fri, 18 Jul 2025 15:48:36 -0400 Subject: [PATCH 2/4] better fix specific to openai --- ddtrace/llmobs/_integrations/openai.py | 6 +++--- ddtrace/llmobs/_integrations/utils.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ddtrace/llmobs/_integrations/openai.py b/ddtrace/llmobs/_integrations/openai.py index 2b51d39ec85..dda7aff852b 100644 --- a/ddtrace/llmobs/_integrations/openai.py +++ b/ddtrace/llmobs/_integrations/openai.py @@ -153,9 +153,9 @@ def _extract_llmobs_metrics_tags( # in the streamed responses case, `resp` is a list with `usage` being stored in the first element if resp and isinstance(resp, list) and _get_attr(resp[0], "usage", None): - token_usage = resp[0].get("usage", {}) - elif resp and getattr(resp, "usage", None): - token_usage = resp.usage + token_usage = _get_attr(resp[0], "usage", {}) + elif resp and _get_attr(resp, "usage", None): + token_usage = _get_attr(resp, "usage", {}) if token_usage is not None: prompt_tokens = _get_attr(token_usage, "prompt_tokens", 0) completion_tokens = _get_attr(token_usage, "completion_tokens", 0) diff --git a/ddtrace/llmobs/_integrations/utils.py b/ddtrace/llmobs/_integrations/utils.py index 707618d0c26..eb374c5ac17 100644 --- a/ddtrace/llmobs/_integrations/utils.py +++ b/ddtrace/llmobs/_integrations/utils.py @@ -1189,7 +1189,7 @@ def _compute_completion_tokens(completions_or_messages, model_name): estimated = False num_completion_tokens = 0 for choice in completions_or_messages: - content = _get_attr(choice, "content", "") or _get_attr(choice, "text", "") + content = choice.get("content", "") or choice.get("text", "") estimated, completion_tokens = _compute_token_count(content, model_name) num_completion_tokens += completion_tokens return estimated, num_completion_tokens From 376c1e86c68393e2e0279d521dc01e73f97a21e5 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Fri, 18 Jul 2025 16:18:10 -0400 Subject: [PATCH 3/4] test --- .riot/requirements/1911f94.txt | 51 ++++++++++++++++++++++ riotfile.py | 2 +- tests/contrib/openai/test_openai_llmobs.py | 35 +-------------- 3 files changed, 54 insertions(+), 34 deletions(-) create mode 100644 .riot/requirements/1911f94.txt diff --git a/.riot/requirements/1911f94.txt b/.riot/requirements/1911f94.txt new file mode 100644 index 00000000000..e84f071768a --- /dev/null +++ b/.riot/requirements/1911f94.txt @@ -0,0 +1,51 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1911f94.in +# +annotated-types==0.7.0 +anyio==4.9.0 +attrs==25.3.0 +certifi==2025.7.14 +charset-normalizer==3.4.2 +coverage[toml]==7.9.2 +distro==1.9.0 +exceptiongroup==1.3.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.1.0 +jiter==0.10.0 +mock==5.2.0 +multidict==6.6.3 +openai==1.66.0 +opentracing==2.4.0 +packaging==25.0 +pillow==11.3.0 +pluggy==1.6.0 +propcache==0.3.2 +pydantic==2.11.7 +pydantic-core==2.33.2 +pygments==2.19.2 +pytest==8.4.1 +pytest-asyncio==0.21.1 +pytest-cov==6.2.1 +pytest-mock==3.14.1 +pytest-randomly==3.16.0 +pyyaml==6.0.2 +regex==2024.11.6 +requests==2.32.4 +sniffio==1.3.1 +sortedcontainers==2.4.0 +tiktoken==0.9.0 +tomli==2.2.1 +tqdm==4.67.1 +typing-extensions==4.14.1 +typing-inspection==0.4.1 +urllib3==1.26.20 +vcrpy==7.0.0 +wrapt==1.17.2 +yarl==1.20.1 diff --git a/riotfile.py b/riotfile.py index e16cc54d8da..e856e326085 100644 --- a/riotfile.py +++ b/riotfile.py @@ -2628,7 +2628,7 @@ def select_pys(min_version: str = MIN_PYTHON_VERSION, max_version: str = MAX_PYT Venv( pys=select_pys(min_version="3.8"), pkgs={ - "openai": [latest, "~=1.76.2"], + "openai": [latest, "~=1.76.2", "==1.66.0"], "tiktoken": latest, "pillow": latest, }, diff --git a/tests/contrib/openai/test_openai_llmobs.py b/tests/contrib/openai/test_openai_llmobs.py index c2bce9172f5..e8bed5a64da 100644 --- a/tests/contrib/openai/test_openai_llmobs.py +++ b/tests/contrib/openai/test_openai_llmobs.py @@ -1737,43 +1737,12 @@ def test_responses_reasoning_stream(self, openai, ddtrace_global_config, mock_ll for event in stream: pass - span = mock_tracer.pop_traces()[0][0] - assert mock_llmobs_writer.enqueue.call_count == 1 - mock_llmobs_writer.enqueue.assert_called_with( - _expected_llmobs_llm_span_event( - span, - model_name="o4-mini-2025-04-16", - model_provider="openai", - input_messages=[{"content": "If one plus a number is 10, what is the number?", "role": "user"}], - output_messages=[ - {"role": "reasoning", "content": mock.ANY}, - {"role": "assistant", "content": "The number is 9, since 1 + x = 10 ⇒ x = 10 − 1 = 9."}, - ], - metadata={ - "reasoning": {"effort": "medium", "summary": "detailed"}, - "stream": True, - "temperature": 1.0, - "top_p": 1.0, - "tools": [], - "tool_choice": "auto", - "truncation": "disabled", - "text": {"format": {"type": "text"}}, - "reasoning_tokens": 128, - }, - token_metrics={ - "input_tokens": mock.ANY, - "output_tokens": mock.ANY, - "total_tokens": mock.ANY, - "cache_read_input_tokens": mock.ANY, - }, - tags={"ml_app": "", "service": "tests.contrib.openai"}, - ) - ) - # special assertion on rough reasoning content span_event = mock_llmobs_writer.enqueue.call_args[0][0] reasoning_content = json.loads(span_event["meta"]["output"]["messages"][0]["content"]) + assistant_content = span_event["meta"]["output"]["messages"][1]["content"] assert reasoning_content["summary"] is not None + assert assistant_content == "The number is 9, since 1 + x = 10 ⇒ x = 10 − 1 = 9." @pytest.mark.skipif( parse_version(openai_module.version.VERSION) < (1, 66), reason="Response options only available openai >= 1.66" From 5cdcc5d56b4d961fa34e0f333002ea117c32eb6d Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Fri, 18 Jul 2025 16:26:42 -0400 Subject: [PATCH 4/4] scripts/compile-and-prune-test-requirements --- .riot/requirements/1a0657d.txt | 49 +++++++++++++++++++++++++++++++ .riot/requirements/85ff44d.txt | 49 +++++++++++++++++++++++++++++++ .riot/requirements/b68c552.txt | 51 ++++++++++++++++++++++++++++++++ .riot/requirements/c5399d2.txt | 49 +++++++++++++++++++++++++++++++ .riot/requirements/e9c67e1.txt | 53 ++++++++++++++++++++++++++++++++++ 5 files changed, 251 insertions(+) create mode 100644 .riot/requirements/1a0657d.txt create mode 100644 .riot/requirements/85ff44d.txt create mode 100644 .riot/requirements/b68c552.txt create mode 100644 .riot/requirements/c5399d2.txt create mode 100644 .riot/requirements/e9c67e1.txt diff --git a/.riot/requirements/1a0657d.txt b/.riot/requirements/1a0657d.txt new file mode 100644 index 00000000000..71749fcce22 --- /dev/null +++ b/.riot/requirements/1a0657d.txt @@ -0,0 +1,49 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/1a0657d.in +# +annotated-types==0.7.0 +anyio==4.9.0 +attrs==25.3.0 +certifi==2025.7.14 +charset-normalizer==3.4.2 +coverage[toml]==7.9.2 +distro==1.9.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.1.0 +jiter==0.10.0 +mock==5.2.0 +multidict==6.6.3 +openai==1.66.0 +opentracing==2.4.0 +packaging==25.0 +pillow==11.3.0 +pluggy==1.6.0 +propcache==0.3.2 +pydantic==2.11.7 +pydantic-core==2.33.2 +pygments==2.19.2 +pytest==8.4.1 +pytest-asyncio==0.21.1 +pytest-cov==6.2.1 +pytest-mock==3.14.1 +pytest-randomly==3.16.0 +pyyaml==6.0.2 +regex==2024.11.6 +requests==2.32.4 +sniffio==1.3.1 +sortedcontainers==2.4.0 +tiktoken==0.9.0 +tqdm==4.67.1 +typing-extensions==4.14.1 +typing-inspection==0.4.1 +urllib3==1.26.20 +vcrpy==7.0.0 +wrapt==1.17.2 +yarl==1.20.1 diff --git a/.riot/requirements/85ff44d.txt b/.riot/requirements/85ff44d.txt new file mode 100644 index 00000000000..244597ff29f --- /dev/null +++ b/.riot/requirements/85ff44d.txt @@ -0,0 +1,49 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/85ff44d.in +# +annotated-types==0.7.0 +anyio==4.9.0 +attrs==25.3.0 +certifi==2025.7.14 +charset-normalizer==3.4.2 +coverage[toml]==7.9.2 +distro==1.9.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.1.0 +jiter==0.10.0 +mock==5.2.0 +multidict==6.6.3 +openai==1.66.0 +opentracing==2.4.0 +packaging==25.0 +pillow==11.3.0 +pluggy==1.6.0 +propcache==0.3.2 +pydantic==2.11.7 +pydantic-core==2.33.2 +pygments==2.19.2 +pytest==8.4.1 +pytest-asyncio==0.21.1 +pytest-cov==6.2.1 +pytest-mock==3.14.1 +pytest-randomly==3.16.0 +pyyaml==6.0.2 +regex==2024.11.6 +requests==2.32.4 +sniffio==1.3.1 +sortedcontainers==2.4.0 +tiktoken==0.9.0 +tqdm==4.67.1 +typing-extensions==4.14.1 +typing-inspection==0.4.1 +urllib3==1.26.20 +vcrpy==7.0.0 +wrapt==1.17.2 +yarl==1.20.1 diff --git a/.riot/requirements/b68c552.txt b/.riot/requirements/b68c552.txt new file mode 100644 index 00000000000..723fa90b9bc --- /dev/null +++ b/.riot/requirements/b68c552.txt @@ -0,0 +1,51 @@ +# +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/b68c552.in +# +annotated-types==0.7.0 +anyio==4.5.2 +attrs==25.3.0 +certifi==2025.7.14 +charset-normalizer==3.4.2 +coverage[toml]==7.6.1 +distro==1.9.0 +exceptiongroup==1.3.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +hypothesis==6.45.0 +idna==3.10 +importlib-metadata==8.5.0 +iniconfig==2.1.0 +jiter==0.9.1 +mock==5.2.0 +multidict==6.1.0 +openai==1.66.0 +opentracing==2.4.0 +packaging==25.0 +pillow==10.4.0 +pluggy==1.5.0 +propcache==0.2.0 +pydantic==2.10.6 +pydantic-core==2.27.2 +pytest==8.3.5 +pytest-asyncio==0.21.1 +pytest-cov==5.0.0 +pytest-mock==3.14.1 +pytest-randomly==3.15.0 +pyyaml==6.0.2 +regex==2024.11.6 +requests==2.32.4 +sniffio==1.3.1 +sortedcontainers==2.4.0 +tiktoken==0.7.0 +tomli==2.2.1 +tqdm==4.67.1 +typing-extensions==4.13.2 +urllib3==1.26.20 +vcrpy==6.0.2 +wrapt==1.17.2 +yarl==1.15.2 +zipp==3.20.2 diff --git a/.riot/requirements/c5399d2.txt b/.riot/requirements/c5399d2.txt new file mode 100644 index 00000000000..186f21ddd46 --- /dev/null +++ b/.riot/requirements/c5399d2.txt @@ -0,0 +1,49 @@ +# +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/c5399d2.in +# +annotated-types==0.7.0 +anyio==4.9.0 +attrs==25.3.0 +certifi==2025.7.14 +charset-normalizer==3.4.2 +coverage[toml]==7.9.2 +distro==1.9.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +hypothesis==6.45.0 +idna==3.10 +iniconfig==2.1.0 +jiter==0.10.0 +mock==5.2.0 +multidict==6.6.3 +openai==1.66.0 +opentracing==2.4.0 +packaging==25.0 +pillow==11.3.0 +pluggy==1.6.0 +propcache==0.3.2 +pydantic==2.11.7 +pydantic-core==2.33.2 +pygments==2.19.2 +pytest==8.4.1 +pytest-asyncio==0.21.1 +pytest-cov==6.2.1 +pytest-mock==3.14.1 +pytest-randomly==3.16.0 +pyyaml==6.0.2 +regex==2024.11.6 +requests==2.32.4 +sniffio==1.3.1 +sortedcontainers==2.4.0 +tiktoken==0.9.0 +tqdm==4.67.1 +typing-extensions==4.14.1 +typing-inspection==0.4.1 +urllib3==1.26.20 +vcrpy==7.0.0 +wrapt==1.17.2 +yarl==1.20.1 diff --git a/.riot/requirements/e9c67e1.txt b/.riot/requirements/e9c67e1.txt new file mode 100644 index 00000000000..daa1f61fdcb --- /dev/null +++ b/.riot/requirements/e9c67e1.txt @@ -0,0 +1,53 @@ +# +# This file is autogenerated by pip-compile with Python 3.9 +# by the following command: +# +# pip-compile --allow-unsafe --no-annotate .riot/requirements/e9c67e1.in +# +annotated-types==0.7.0 +anyio==4.9.0 +attrs==25.3.0 +certifi==2025.7.14 +charset-normalizer==3.4.2 +coverage[toml]==7.9.2 +distro==1.9.0 +exceptiongroup==1.3.0 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +hypothesis==6.45.0 +idna==3.10 +importlib-metadata==8.7.0 +iniconfig==2.1.0 +jiter==0.10.0 +mock==5.2.0 +multidict==6.6.3 +openai==1.66.0 +opentracing==2.4.0 +packaging==25.0 +pillow==11.3.0 +pluggy==1.6.0 +propcache==0.3.2 +pydantic==2.11.7 +pydantic-core==2.33.2 +pygments==2.19.2 +pytest==8.4.1 +pytest-asyncio==0.21.1 +pytest-cov==6.2.1 +pytest-mock==3.14.1 +pytest-randomly==3.16.0 +pyyaml==6.0.2 +regex==2024.11.6 +requests==2.32.4 +sniffio==1.3.1 +sortedcontainers==2.4.0 +tiktoken==0.9.0 +tomli==2.2.1 +tqdm==4.67.1 +typing-extensions==4.14.1 +typing-inspection==0.4.1 +urllib3==1.26.20 +vcrpy==7.0.0 +wrapt==1.17.2 +yarl==1.20.1 +zipp==3.23.0