feat(runtime): Support tokio-console (#1986)

.cargo/config.toml

+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[build]
+# tokio-console needs this
+rustflags = ["--cfg", "tokio_unstable"]

Cargo.lock

@@ -551,6 +551,12 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
 
+[[package]]
+name = "base64"
+version = "0.21.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
+
 [[package]]
 name = "base64"
 version = "0.22.1"
@@ -1084,6 +1090,45 @@ dependencies = [
  "windows-sys 0.59.0",
 ]
 
+[[package]]
+name = "console-api"
+version = "0.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857"
+dependencies = [
+ "futures-core",
+ "prost 0.13.5",
+ "prost-types",
+ "tonic",
+ "tracing-core",
+]
+
+[[package]]
+name = "console-subscriber"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6539aa9c6a4cd31f4b1c040f860a1eac9aa80e7df6b05d506a6e7179936d6a01"
+dependencies = [
+ "console-api",
+ "crossbeam-channel",
+ "crossbeam-utils",
+ "futures-task",
+ "hdrhistogram",
+ "humantime",
+ "hyper-util",
+ "prost 0.13.5",
+ "prost-types",
+ "serde",
+ "serde_json",
+ "thread_local",
+ "tokio",
+ "tokio-stream",
+ "tonic",
+ "tracing",
+ "tracing-core",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "const-oid"
 version = "0.9.6"
@@ -1896,6 +1941,7 @@ dependencies = [
  "blake3",
  "bytes",
  "chrono",
+ "console-subscriber",
  "derive-getters",
  "derive_builder",
  "educe",
@@ -2873,6 +2919,19 @@ dependencies = [
  "foldhash",
 ]
 
+[[package]]
+name = "hdrhistogram"
+version = "7.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
+dependencies = [
+ "base64 0.21.7",
+ "byteorder",
+ "flate2",
+ "nom",
+ "num-traits",
+]
+
 [[package]]
 name = "heck"
 version = "0.4.1"
@@ -7060,6 +7119,7 @@ dependencies = [
  "slab",
  "socket2",
  "tokio-macros",
+ "tracing",
  "windows-sys 0.52.0",
 ]
 

Cargo.toml

 # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 
 [workspace]
 members = [

docs/guides/dynamo_run.md

@@ -669,3 +669,12 @@ More fully-featured Backend engines (used by `dynamo-run`):
 - [vllm](https://github.com/ai-dynamo/dynamo/blob/main/launch/dynamo-run/src/subprocess/vllm_inc.py)
 - [sglang](https://github.com/ai-dynamo/dynamo/blob/main/launch/dynamo-run/src/subprocess/sglang_inc.py)
 
+### Debugging
+
+`dynamo-run` and `dynamo-runtime` support [tokio-console](https://github.com/tokio-rs/console). Build with the feature to enable:
+```
+cargo build --features cuda,tokio-console -p dynamo-run
+```
+
+The listener uses the default tokio console port, and all interfaces (0.0.0.0).
+

launch/dynamo-run/Cargo.toml

@@ -23,6 +23,8 @@ metal = ["dynamo-engine-llamacpp/metal", "dynamo-engine-mistralrs/metal"]
 vulkan = ["dynamo-engine-llamacpp/vulkan"]
 openmp = ["dynamo-engine-llamacpp/openmp"]
 
+tokio-console = ["dynamo-runtime/tokio-console"]
+
 [dependencies]
 dynamo-llm = { workspace = true }
 dynamo-runtime = { workspace = true }

launch/dynamo-run/src/main.rs

 // SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use std::env;
 

lib/runtime/Cargo.toml

 # SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
 
 [package]
 name = "dynamo-runtime"
@@ -27,8 +15,8 @@ description = "Dynamo Runtime Library"
 [features]
 default = []
 integration = []
-# Tests that require an active ETCD server
-testing-etcd = []
+testing-etcd = [] # Tests that require an active ETCD server
+tokio-console = ["dep:console-subscriber", "tokio/tracing"]
 
 [dependencies]
 # Use workspace dependencies where available
@@ -64,6 +52,7 @@ xxhash-rust = { workspace = true }
 
 arc-swap = { version = "1" }
 async-once-cell = { version = "0.5.4" }
+console-subscriber = { version = "0.4", optional = true }
 educe = { version = "0.6.0" }
 figment = { version = "0.10.19", features = ["env", "json", "toml", "test"] }
 local-ip-address = { version = "0.6.3" }
@@ -81,10 +70,3 @@ env_logger = { version = "0.11" }
 reqwest = { workspace = true }
 rstest = { version = "0.23.0" }
 temp-env = { version = "0.3.6" }
-
-# These patches are to address issues in reqwest, which is used in the HTTP server test (but not on servers).
-# These are transitive dependencies to use secure versions and mitigate known vulnerabilities.
-[patch.crates-io]
-tokio  = { version = "1.18.4" }   # addresses RUSTSEC-2023-0001
-h2     = { version = "0.4.4"   }   # addresses RUSTSEC-2024-0332
-rustls = { version = "0.23.18" }   # addresses RUSTSEC-2024-0399

lib/runtime/src/logging.rs

 // SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 //! Dynamo Distributed Logging Module.
 //!
@@ -46,6 +34,7 @@ use figment::{
     Figment,
 };
 use serde::{Deserialize, Serialize};
+use tracing::level_filters::LevelFilter;
 use tracing::{Event, Subscriber};
 use tracing_subscriber::fmt::time::FormatTime;
 use tracing_subscriber::fmt::time::LocalTime;
@@ -58,6 +47,8 @@ use tracing_subscriber::registry::LookupSpan;
 use tracing_subscriber::EnvFilter;
 use tracing_subscriber::{filter::Directive, fmt};
 
+use crate::config::{disable_ansi_logging, jsonl_logging_enabled};
+
 /// ENV used to set the log level
 const FILTER_ENV: &str = "DYN_LOG";
 
@@ -98,12 +89,58 @@ impl Default for LoggingConfig {
 
 /// Initialize the logger
 pub fn init() {
-    INIT.call_once(|| {
-        let config = load_config();
+    INIT.call_once(setup_logging);
+}
+
+#[cfg(feature = "tokio-console")]
+fn setup_logging() {
+    // Start tokio-console server. Returns a tracing-subscriber Layer.
+    let tokio_console_layer = console_subscriber::ConsoleLayer::builder()
+        .with_default_env()
+        .server_addr(([0, 0, 0, 0], console_subscriber::Server::DEFAULT_PORT))
+        .spawn();
+    let tokio_console_target = tracing_subscriber::filter::Targets::new()
+        .with_default(LevelFilter::ERROR)
+        .with_target("runtime", LevelFilter::TRACE)
+        .with_target("tokio", LevelFilter::TRACE);
+    let l = fmt::layer()
+        .with_ansi(!disable_ansi_logging())
+        .event_format(fmt::format().compact().with_timer(TimeFormatter::new()))
+        .with_writer(std::io::stderr)
+        .with_filter(filter(load_config));
+    tracing_subscriber::registry()
+        .with(l)
+        .with(tokio_console_layer.with_filter(tokio_console_target))
+        .init();
+}
+
+#[cfg(not(feature = "tokio-console"))]
+fn setup_logging() {
+    let f = filters(load_config());
+    // The generics mean we have to repeat everything. Each builder method returns a
+    // specialized type.
+    if jsonl_logging_enabled() {
+        // JSON logger for NIM
+
+        let l = fmt::layer()
+            .with_ansi(false)
+            .event_format(CustomJsonFormatter::new())
+            .with_writer(std::io::stderr)
+            .with_filter(f);
+        tracing_subscriber::registry().with(l).init();
+    } else {
+        // Normal logging
+
+        let l = fmt::layer()
+            .with_ansi(!disable_ansi_logging())
+            .event_format(fmt::format().compact().with_timer(TimeFormatter::new()))
+            .with_writer(std::io::stderr)
+            .with_filter(f);
+        tracing_subscriber::registry().with(l).init();
+    }
+}
 
-        // Examples to remove noise
-        // .add_directive("rustls=warn".parse()?)
-        // .add_directive("tokio_util::codec=warn".parse()?)
+fn filters(config: LoggingConfig) -> EnvFilter {
     let mut filter_layer = EnvFilter::builder()
         .with_default_directive(config.log_level.parse().unwrap())
         .with_env_var(FILTER_ENV)
@@ -120,23 +157,7 @@ pub fn init() {
             }
         }
     }
-
-        if crate::config::jsonl_logging_enabled() {
-            let l = fmt::layer()
-                .with_ansi(false) // ansi terminal escapes and colors always disabled
-                .event_format(CustomJsonFormatter::new())
-                .with_writer(std::io::stderr)
-                .with_filter(filter_layer);
-            tracing_subscriber::registry().with(l).init();
-        } else {
-            let l = fmt::layer()
-                .with_ansi(!crate::config::disable_ansi_logging())
-                .event_format(fmt::format().compact().with_timer(TimeFormatter::new()))
-                .with_writer(std::io::stderr)
-                .with_filter(filter_layer);
-            tracing_subscriber::registry().with(l).init();
-        };
-    });
+    filter_layer
 }
 
 /// Log a message with file and line info