fix: observability fixes for OTEL tracing and logging (#7240)

Signed-off-by: Jie Hao <jihao@nvidia.com>

deploy/observability/tempo-datasource.yml

@@ -20,7 +20,9 @@ datasources:
         filterByTraceID: false
         filterBySpanID: false
         customQuery: true
-        query: '{service_name=~".+"} | trace_id = "$${__span.traceId}"'
+        query: '{service_name=~".+"} | trace_id = "$${__trace.traceId}"'
+        spanStartTimeShift: "-1h"
+        spanEndTimeShift: "1h"
       search:
         hide: false
       nodeGraph:

lib/bindings/python/src/dynamo/runtime/logging.py

@@ -89,12 +89,15 @@ class VllmColorFormatter(logging.Formatter):
         else:
             target = record.module
         msg = record.getMessage()
-        return (
+        result = (
             f"{self._DIM}{ts}{self._RESET} "
             f"{color}{level:>5}{self._RESET} "
             f"{self._DIM}{target}{self._RESET}{self._DIM}:{self._RESET} "
             f"{msg}"
         )
+        if record.exc_info and record.exc_info[0] is not None:
+            result += "\n" + self.formatException(record.exc_info)
+        return result
 
 
 # Configure the Python logger to use the NimLogHandler

pyproject.toml

@@ -50,7 +50,7 @@ trtllm =[
 vllm = [
     "uvloop",
     "nixl[cu12]<=0.10.1",
-    "vllm[flashinfer,runai]==0.17.1",
+    "vllm[flashinfer,runai,otel]==0.17.1",
     # vllm-omni 0.16.0 is now on PyPI; install only future rc builds from source in container builds
     # (see container/deps/vllm/install_vllm.sh). pip install ai-dynamo[vllm] will
     # not include vllm-omni — install it separately from source if needed.