fix(llamaindex): structured llm model and temperature parsing (#3159)

Author: galkleinman

packages/opentelemetry-instrumentation-llamaindex/opentelemetry/instrumentation/llamaindex/span_utils.py

@@ -31,6 +31,11 @@ def set_llm_chat_request_model_attributes(event, span):
 
     model_dict = event.model_dict
     span.set_attribute(SpanAttributes.LLM_REQUEST_TYPE, LLMRequestTypeValues.CHAT.value)
+
+    # For StructuredLLM, the model and temperature are nested under model_dict.llm
+    if "llm" in model_dict:
+        model_dict = model_dict.get("llm", {})
+
     span.set_attribute(SpanAttributes.LLM_REQUEST_MODEL, model_dict.get("model"))
     span.set_attribute(
         SpanAttributes.LLM_REQUEST_TEMPERATURE, model_dict.get("temperature")

packages/opentelemetry-instrumentation-llamaindex/tests/cassettes/test_structured_llm/test_structured_llm_achat_model_attributes.yaml

@@ -0,0 +1,116 @@
+interactions:
+- request:
+    body: '{"messages":[{"role":"system","content":"Extract invoice information from
+      the following text."},{"role":"user","content":"Invoice #67890 for $299.99 to
+      Jane Doe"}],"model":"gpt-4o","stream":false,"temperature":0.5,"tool_choice":"required","tools":[{"type":"function","function":{"name":"Invoice","description":"Example
+      model for structured output testing.","parameters":{"properties":{"invoice_id":{"description":"Invoice
+      identifier","title":"Invoice Id","type":"string"},"amount":{"description":"Invoice
+      amount","title":"Amount","type":"number"},"customer_name":{"description":"Customer
+      name","title":"Customer Name","type":"string"}},"required":["invoice_id","amount","customer_name"],"type":"object","additionalProperties":false},"strict":false}}]}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '751'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      traceparent:
+      - 00-8a61275987df582f1f4634434316e052-29c62ce5233ca6c1-01
+      user-agent:
+      - AsyncOpenAI/Python 1.58.1
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - async:asyncio
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.58.1
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.9.5
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//jFNRb5swEH7nV1j3nFQhJITwOG2Tmm1au2mqlFEh1xzgxdjMNlnbKP99
+        MiSBpJk0HpB13313330+7zxCgGcQE2AltayqxfjdH/97Pt8+LMJv4X1w+/l+XX35+PvDWr2Wn0oY
+        OYZ6+oXMHlk3TFW1QMuV7GCmkVp0Vf3FPPDDKJj5LVCpDIWjFbUdz9R4OpnOxpNoPAkPxFJxhgZi
+        8tMjhJBd+3cSZYbPEJPJ6Bip0BhaIMSnJEJAK+EiQI3hxlJpYdSDTEmL0qmWjRADwColUkaF6Bt3
+        325w7n2iQqTB5us6Kld3myh6jrTmP+4eypfVPB/060q/1K2gvJHs5M8AP8Xji2aEgKRVy72VW+fK
+        BZUQoLpoKpTWyYZdArxLTHmWQJxAuIiWkwRGCdBKNdImEE+Xy5vlcpQAa4xVFerUNWmzV1Qiea8w
+        gT2cNdp7186PA/805o2h4q2xVEplqZuvdfbxgOxPlyhUUWv1ZC6okHPJTZlqpKb1ZnhF3lFIKwGa
+        sy2AWquqtqlVG2ybLmddUeh3tAeniwNolaWij/tTf3SlXJqhpbzdktNiMspKzHpqv6C0ybgaAN5g
+        9LdqrtXuxuey+J/yPcAY1haztNaYcXY+cZ+m0T3hf6WdTG4Fg0G9dZtlOWp3HRnmtBHd6wLzYixW
+        ac5lgbrWvH1ikNdpEC7mURhk0RS8vfcXAAD//wMAz9iN+WsEAAA=
+    headers:
+      CF-RAY:
+      - 9631210b48c59b09-TLV
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Tue, 22 Jul 2025 07:12:22 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=85Pnuwdxp60XG9gTYk6PvKiayYA.O7_GyYvIqszoFK4-1753168342-1.0.1.1-U3UrVzfu85BkIH.H5MYad4ZmN7puPBD7KQW.azMffta5E4H.xC_CG5z8TrUQgzbuCSgy2usWVYL2yAcSoMeLGGO4EEBzHVfENLWDNLBsUK8;
+        path=/; expires=Tue, 22-Jul-25 07:42:22 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=8vwo6_jaBCZTKBaXzggpnVdGNTrb_E1G2_w3V4_aTfw-1753168342175-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Strict-Transport-Security:
+      - max-age=31536000; includeSubDomains; preload
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      cf-cache-status:
+      - DYNAMIC
+      openai-organization:
+      - traceloop
+      openai-processing-ms:
+      - '741'
+      openai-project:
+      - proj_tzz1TbPPOXaf6j9tEkVUBIAa
+      openai-version:
+      - '2020-10-01'
+      x-envoy-upstream-service-time:
+      - '744'
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '30000000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '29999973'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - 9a674902-3edd-41ec-b409-4d348446f812
+    status:
+      code: 200
+      message: OK
+version: 1

packages/opentelemetry-instrumentation-llamaindex/tests/cassettes/test_structured_llm/test_structured_llm_model_attributes.yaml

@@ -0,0 +1,116 @@
+interactions:
+- request:
+    body: '{"messages":[{"role":"system","content":"Extract invoice information from
+      the following text."},{"role":"user","content":"Invoice #12345 for $199.99 to
+      John Smith"}],"model":"gpt-4o","stream":false,"temperature":0.7,"tool_choice":"required","tools":[{"type":"function","function":{"name":"Invoice","description":"Example
+      model for structured output testing.","parameters":{"properties":{"invoice_id":{"description":"Invoice
+      identifier","title":"Invoice Id","type":"string"},"amount":{"description":"Invoice
+      amount","title":"Amount","type":"number"},"customer_name":{"description":"Customer
+      name","title":"Customer Name","type":"string"}},"required":["invoice_id","amount","customer_name"],"type":"object","additionalProperties":false},"strict":false}}]}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '753'
+      content-type:
+      - application/json
+      host:
+      - api.openai.com
+      traceparent:
+      - 00-fea82171c54bdef7cc2453298f3b9c96-2e3d335209cf620a-01
+      user-agent:
+      - OpenAI/Python 1.58.1
+      x-stainless-arch:
+      - arm64
+      x-stainless-async:
+      - 'false'
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.58.1
+      x-stainless-retry-count:
+      - '0'
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.9.5
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//jFPRbtowFH3PV1j3OVQkAQJ56zpNo1pVTUjbtKWKXOdCTB07s50Oivj3
+        yQmQQJm0PETWPT7nHh9f7zxCgOeQEGAFtaysxODDn2DBNl8+vf14GG43xe0LXdR1/PPjOg5/fwff
+        MdTzGpk9sm6YKiuBlivZwkwjtehUg3gcBZNpFE0boFQ5CkdbVXYwUoNwGI4Gw+lgODkQC8UZGkjI
+        L48QQnbN31mUOW4gIUP/WCnRGLpCSE6bCAGthKsANYYbS6UFvwOZkhalcy1rIXqAVUpkjArRNW6/
+        XW/d5USFyPJq/rmOH9nD/Nvdbayi+0cze17ffe31a6W3VWNoWUt2yqeHn+rJRTNCQNKy4c7lq0vl
+        gkoIUL2qS5TW2YZdCrRUtbQpJMFsdjOb+Smw2lhVos6cVgpJCveqkGRRcluk4KfAW+2M5w0ahNFo
+        nMIezlrtvWvrp16CGpe1oeJ9tFRKZak7YZPt0wHZn65RqFWl1bO5oMKSS26KTCM1TTr9S/KORhoL
+        UJ/NAVRalZXNrHrBpuls1IpCN6UdGMYH0CpLRVcPwsC/IpflaClv5uQ0moyyAvOO2o0orXOueoDX
+        O/p7N9e02+Nzufof+Q5gDCuLeVZpzDk7P3G3TaN7xP/adgq5MQwG9asbFMtRu+vIcUlr0b4vMFtj
+        scyWXK5QV5o3jwyWVRZN4vF0EuXTELy99xcAAP//AwAYSN8XbQQAAA==
+    headers:
+      CF-RAY:
+      - 963120fffc22efea-TLV
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Tue, 22 Jul 2025 07:12:19 GMT
+      Server:
+      - cloudflare
+      Set-Cookie:
+      - __cf_bm=KVzJcC68SQuziL071b4WwO20k3mVcAnXeO74ikQvbTk-1753168339-1.0.1.1-VjNOeRtShSwdWPgUcGHVVipel_nbk82XRkGT5m__qbU7aBiCFLU6Oj8DNlL53ZnnKptoG4CaJSvVt6Zh5YLChq9Ad7vpo5IhXwUtoeR3ws0;
+        path=/; expires=Tue, 22-Jul-25 07:42:19 GMT; domain=.api.openai.com; HttpOnly;
+        Secure; SameSite=None
+      - _cfuvid=B236WyNvITlBmvDyAycWz.GvuU_hKcSHN1HBQ8O2fa4-1753168339363-0.0.1.1-604800000;
+        path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      cf-cache-status:
+      - DYNAMIC
+      openai-organization:
+      - traceloop
+      openai-processing-ms:
+      - '890'
+      openai-project:
+      - proj_tzz1TbPPOXaf6j9tEkVUBIAa
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-envoy-upstream-service-time:
+      - '897'
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '30000000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '29999973'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 0s
+      x-request-id:
+      - req_d35a94342d3f797254ffcd31f2a6b423
+    status:
+      code: 200
+      message: OK
+version: 1

packages/opentelemetry-instrumentation-llamaindex/tests/test_structured_llm.py

@@ -0,0 +1,104 @@
+import pytest
+from llama_index.core.llms import ChatMessage
+from llama_index.llms.openai import OpenAI
+from pydantic import BaseModel, Field
+from opentelemetry.semconv_ai import SpanAttributes, LLMRequestTypeValues
+
+
+class Invoice(BaseModel):
+    """Example model for structured output testing."""
+
+    invoice_id: str = Field(description="Invoice identifier")
+    amount: float = Field(description="Invoice amount")
+    customer_name: str = Field(description="Customer name")
+
+
+@pytest.mark.vcr()
+def test_structured_llm_model_attributes(instrument_with_content, span_exporter):
+    """
+    Test that StructuredLLM correctly sets model attributes.
+
+    This test reproduces the issue where set_llm_chat_request_model_attributes
+    fails to access model and temperature from StructuredLLM because it tries
+    to access model_dict.model instead of model_dict.llm.model.
+    """
+    llm = OpenAI(model="gpt-4o", temperature=0.7)
+    structured_llm = llm.as_structured_llm(Invoice)
+
+    messages = [
+        ChatMessage(
+            role="system",
+            content="Extract invoice information from the following text.",
+        ),
+        ChatMessage(role="user", content="Invoice #12345 for $199.99 to John Smith"),
+    ]
+
+    response = structured_llm.chat(messages)
+
+    assert response is not None
+
+    spans = span_exporter.get_finished_spans()
+    assert len(spans) > 0
+
+    llm_span = None
+    for span in spans:
+        if (
+            span.attributes.get(SpanAttributes.LLM_REQUEST_TYPE)
+            == LLMRequestTypeValues.CHAT.value
+        ):
+            llm_span = span
+            break
+
+    assert llm_span is not None, "Should have an LLM span"
+
+    attributes = llm_span.attributes
+    assert "gen_ai.request.model" in attributes
+    assert attributes["gen_ai.request.model"] == "gpt-4o"
+    assert "gen_ai.request.temperature" in attributes
+    assert attributes["gen_ai.request.temperature"] == 0.7
+
+
+@pytest.mark.vcr()
+@pytest.mark.asyncio
+async def test_structured_llm_achat_model_attributes(
+    instrument_with_content, span_exporter
+):
+    """
+    Test that StructuredLLM achat method correctly sets model attributes.
+
+    This is the async version of the test that reproduces the original issue.
+    """
+    llm = OpenAI(model="gpt-4o", temperature=0.5)
+    structured_llm = llm.as_structured_llm(Invoice)
+
+    messages = [
+        ChatMessage(
+            role="system",
+            content="Extract invoice information from the following text.",
+        ),
+        ChatMessage(role="user", content="Invoice #67890 for $299.99 to Jane Doe"),
+    ]
+
+    response = await structured_llm.achat(messages)
+
+    assert response is not None
+
+    spans = span_exporter.get_finished_spans()
+    assert len(spans) > 0
+
+    llm_span = None
+    for span in spans:
+        if (
+            span.attributes.get(SpanAttributes.LLM_REQUEST_TYPE)
+            == LLMRequestTypeValues.CHAT.value
+        ):
+            llm_span = span
+            break
+
+    assert llm_span is not None, "Should have an LLM span"
+
+    attributes = llm_span.attributes
+    assert "gen_ai.request.model" in attributes
+    assert attributes["gen_ai.request.model"] == "gpt-4o"
+    assert "gen_ai.request.temperature" in attributes
+    assert attributes["gen_ai.request.temperature"] == 0.5