Unverified Commit 8a04d7b2 authored by mohammedabdulwahhab's avatar mohammedabdulwahhab Committed by GitHub
Browse files

fix: revert placement of logging init if otel exports is disabled (#3797)


Signed-off-by: default avatarmohammedabdulwahhab <furkhan324@berkeley.edu>
parent 69fffdba
...@@ -30,6 +30,8 @@ Dynamo's tracing is configured via environment variables. For complete logging d ...@@ -30,6 +30,8 @@ Dynamo's tracing is configured via environment variables. For complete logging d
| `OTEL_EXPORT_ENDPOINT` | OTLP gRPC endpoint for Tempo | `http://localhost:4317` (local) or `http://tempo:4317` (docker) | | `OTEL_EXPORT_ENDPOINT` | OTLP gRPC endpoint for Tempo | `http://localhost:4317` (local) or `http://tempo:4317` (docker) |
| `OTEL_SERVICE_NAME` | Service name for identifying components | `dynamo-frontend`, `dynamo-worker-prefill`, `dynamo-worker-decode` | | `OTEL_SERVICE_NAME` | Service name for identifying components | `dynamo-frontend`, `dynamo-worker-prefill`, `dynamo-worker-decode` |
**Note:** When `OTEL_EXPORT_ENABLED=1`, logging initialization is deferred until the runtime is available (required by the OTEL exporter). This means some early logs will be dropped. This will be fixed in a future release.
### Example Configuration ### Example Configuration
```bash ```bash
......
...@@ -118,6 +118,18 @@ fn create_request_context( ...@@ -118,6 +118,18 @@ fn create_request_context(
/// import the module. /// import the module.
#[pymodule] #[pymodule]
fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> { fn _core(m: &Bound<'_, PyModule>) -> PyResult<()> {
// Initialize logging early unless OTEL export is enabled (which requires tokio runtime)
if std::env::var("OTEL_EXPORT_ENABLED")
.map(|v| v == "1")
.unwrap_or(false)
{
eprintln!(
"Warning: OTEL_EXPORT_ENABLED=1 detected. Logging initialization deferred until runtime is available. Early logs may be dropped."
);
} else {
rs::logging::init();
}
m.add_function(wrap_pyfunction!(llm::kv::compute_block_hash_for_seq_py, m)?)?; m.add_function(wrap_pyfunction!(llm::kv::compute_block_hash_for_seq_py, m)?)?;
m.add_function(wrap_pyfunction!(log_message, m)?)?; m.add_function(wrap_pyfunction!(log_message, m)?)?;
m.add_function(wrap_pyfunction!(register_llm, m)?)?; m.add_function(wrap_pyfunction!(register_llm, m)?)?;
...@@ -425,9 +437,14 @@ impl DistributedRuntime { ...@@ -425,9 +437,14 @@ impl DistributedRuntime {
// Initialize logging in context where tokio runtime is available // Initialize logging in context where tokio runtime is available
// otel exporter requires it // otel exporter requires it
if std::env::var("OTEL_EXPORT_ENABLED")
.map(|v| v == "1")
.unwrap_or(false)
{
runtime.secondary().block_on(async { runtime.secondary().block_on(async {
rs::logging::init(); rs::logging::init();
}); });
}
let inner = let inner =
if is_static { if is_static {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment