feat: transport agnostic request plane for dynamo - natless (#4246)

lib/runtime/src/utils.rs

@@ -4,6 +4,7 @@
 pub use tokio::time::{Duration, Instant};
 
 pub mod graceful_shutdown;
+pub mod ip_resolver;
 pub mod leader_worker_barrier;
 pub mod pool;
 pub mod stream;
@@ -12,3 +13,4 @@ pub mod tasks;
 pub mod typed_prefix_watcher;
 
 pub use graceful_shutdown::GracefulShutdownTracker;
+pub use ip_resolver::{get_http_rpc_host_from_env, get_tcp_rpc_host_from_env};

lib/runtime/src/utils/ip_resolver.rs

+// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! IP resolution utilities for getting local IP addresses with fallback support
+
+use crate::pipeline::network::tcp::server::{DefaultIpResolver, IpResolver};
+use local_ip_address::Error;
+use std::net::IpAddr;
+
+/// Get the local IP address for HTTP RPC host binding, using IpResolver with fallback to 127.0.0.1
+///
+/// This function attempts to resolve the local IP address using the provided resolver.
+/// If resolution fails, it falls back to 127.0.0.1 (localhost).
+///
+/// # Arguments
+/// * `resolver` - An implementation of IpResolver trait for getting local IP addresses
+///
+/// # Returns
+/// A string representation of the resolved IP address
+pub fn get_http_rpc_host_with_resolver<R: IpResolver>(resolver: R) -> String {
+    let resolved_ip = resolver.local_ip().or_else(|err| match err {
+        Error::LocalIpAddressNotFound => resolver.local_ipv6(),
+        _ => Err(err),
+    });
+
+    match resolved_ip {
+        Ok(addr) => addr,
+        Err(Error::LocalIpAddressNotFound) => IpAddr::from([127, 0, 0, 1]),
+        Err(_) => IpAddr::from([127, 0, 0, 1]), // Fallback for any other error
+    }
+    .to_string()
+}
+
+/// Get the local IP address for HTTP RPC host binding using the default resolver
+///
+/// This is a convenience function that uses the DefaultIpResolver.
+/// It follows the same logic as the TcpStreamServer for IP resolution.
+///
+/// # Returns
+/// A string representation of the resolved IP address, with fallback to "127.0.0.1"
+pub fn get_http_rpc_host() -> String {
+    get_http_rpc_host_with_resolver(DefaultIpResolver)
+}
+
+/// Get the HTTP RPC host from environment variable or resolve local IP as fallback
+///
+/// This function checks the DYN_HTTP_RPC_HOST environment variable first.
+/// If not set, it uses IP resolution to determine the local IP address.
+///
+/// # Returns
+/// A string representation of the HTTP RPC host address
+pub fn get_http_rpc_host_from_env() -> String {
+    std::env::var("DYN_HTTP_RPC_HOST").unwrap_or_else(|_| get_http_rpc_host())
+}
+
+/// Get the TCP RPC host from environment variable or resolve local IP as fallback
+///
+/// This function checks the DYN_TCP_RPC_HOST environment variable first.
+/// If not set, it uses IP resolution to determine the local IP address.
+///
+/// # Returns
+/// A string representation of the TCP RPC host address
+pub fn get_tcp_rpc_host_from_env() -> String {
+    std::env::var("DYN_TCP_RPC_HOST").unwrap_or_else(|_| get_http_rpc_host())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use local_ip_address::Error;
+
+    // Mock resolver for testing
+    struct MockIpResolver {
+        ipv4_result: Result<IpAddr, Error>,
+        ipv6_result: Result<IpAddr, Error>,
+    }
+
+    impl IpResolver for MockIpResolver {
+        fn local_ip(&self) -> Result<IpAddr, Error> {
+            match &self.ipv4_result {
+                Ok(addr) => Ok(*addr),
+                Err(Error::LocalIpAddressNotFound) => Err(Error::LocalIpAddressNotFound),
+                Err(_) => Err(Error::LocalIpAddressNotFound), // Simplify for testing
+            }
+        }
+
+        fn local_ipv6(&self) -> Result<IpAddr, Error> {
+            match &self.ipv6_result {
+                Ok(addr) => Ok(*addr),
+                Err(Error::LocalIpAddressNotFound) => Err(Error::LocalIpAddressNotFound),
+                Err(_) => Err(Error::LocalIpAddressNotFound), // Simplify for testing
+            }
+        }
+    }
+
+    #[test]
+    fn test_get_http_rpc_host_with_successful_ipv4() {
+        let resolver = MockIpResolver {
+            ipv4_result: Ok(IpAddr::from([192, 168, 1, 100])),
+            ipv6_result: Ok(IpAddr::from([0, 0, 0, 0, 0, 0, 0, 1])),
+        };
+
+        let result = get_http_rpc_host_with_resolver(resolver);
+        assert_eq!(result, "192.168.1.100");
+    }
+
+    #[test]
+    fn test_get_http_rpc_host_with_ipv4_fail_ipv6_success() {
+        let resolver = MockIpResolver {
+            ipv4_result: Err(Error::LocalIpAddressNotFound),
+            ipv6_result: Ok(IpAddr::from([0x2001, 0xdb8, 0, 0, 0, 0, 0, 1])),
+        };
+
+        let result = get_http_rpc_host_with_resolver(resolver);
+        assert_eq!(result, "2001:db8::1");
+    }
+
+    #[test]
+    fn test_get_http_rpc_host_with_both_fail() {
+        let resolver = MockIpResolver {
+            ipv4_result: Err(Error::LocalIpAddressNotFound),
+            ipv6_result: Err(Error::LocalIpAddressNotFound),
+        };
+
+        let result = get_http_rpc_host_with_resolver(resolver);
+        assert_eq!(result, "127.0.0.1");
+    }
+
+    #[test]
+    fn test_get_http_rpc_host_from_env_with_env_var() {
+        // Set environment variable
+        unsafe {
+            std::env::set_var("DYN_HTTP_RPC_HOST", "10.0.0.1");
+        }
+
+        let result = get_http_rpc_host_from_env();
+        assert_eq!(result, "10.0.0.1");
+
+        // Clean up
+        unsafe {
+            std::env::remove_var("DYN_HTTP_RPC_HOST");
+        }
+    }
+
+    #[test]
+    fn test_get_http_rpc_host_from_env_without_env_var() {
+        // Note: We can't reliably unset environment variables in tests
+        // This test assumes DYN_HTTP_RPC_HOST is not set to a specific test value
+
+        let result = get_http_rpc_host_from_env();
+        // Should return some IP address (either resolved or fallback)
+        assert!(!result.is_empty());
+
+        // Should be a valid IP address
+        let _: IpAddr = result.parse().expect("Should be a valid IP address");
+    }
+}

tests/serve/test_vllm.py

@@ -48,6 +48,30 @@ vllm_configs = {
             metric_payload_default(min_num_requests=6, backend="vllm"),
         ],
     ),
+    "agg-request-plane-tcp": VLLMConfig(
+        name="agg-request-plane-tcp",
+        directory=vllm_dir,
+        script_name="agg_request_planes.sh",
+        marks=[pytest.mark.gpu_1],
+        model="Qwen/Qwen3-0.6B",
+        script_args=["--tcp"],
+        request_payloads=[
+            chat_payload_default(),
+            completion_payload_default(),
+        ],
+    ),
+    "agg-request-plane-http": VLLMConfig(
+        name="agg-request-plane-http",
+        directory=vllm_dir,
+        script_name="agg_request_planes.sh",
+        marks=[pytest.mark.gpu_1],
+        model="Qwen/Qwen3-0.6B",
+        script_args=["--http"],
+        request_payloads=[
+            chat_payload_default(),
+            completion_payload_default(),
+        ],
+    ),
     "agg-router": VLLMConfig(
         name="agg-router",
         directory=vllm_dir,