feat: Rename dynamo_component_concurrent_requests (#2515)

Signed-off-by: Tzu-Ling Kan <tzulingk@nvidia.com>

deploy/metrics/README.md

@@ -40,7 +40,7 @@ As of Q2 2025, Dynamo HTTP Frontend metrics are exposed when you build container
 
 The core Dynamo backend system automatically exposes metrics with the `dynamo_component_*` prefix for all components that use the `DistributedRuntime` framework:
 
-- `dynamo_component_concurrent_requests`: Requests currently being processed (gauge)
+- `dynamo_component_inflight_requests`: Requests currently being processed (gauge)
 - `dynamo_component_request_bytes_total`: Total bytes received in requests (counter)
 - `dynamo_component_request_duration_seconds`: Request processing time (histogram)
 - `dynamo_component_requests_total`: Total requests processed (counter)

lib/runtime/examples/system_metrics/README.md

@@ -62,7 +62,7 @@ The `dynamo_component_errors_total` metric includes the following error types:
 - `dynamo_component_request_duration_seconds` - Request processing time
 
 ### Gauges
-- `dynamo_component_concurrent_requests` - Number of requests currently being processed
+- `dynamo_component_inflight_requests` - Number of requests currently being processed
 
 ### Custom Metrics (Optional)
 - `dynamo_component_bytes_processed_total` - Total data bytes processed by system handler (example)
@@ -80,9 +80,9 @@ These labels are prefixed with "dynamo_" to avoid collisions with Kubernetes and
 When the system is running, you'll see metrics from http://<ip>:<port>/metrics like this:
 
 ```prometheus
-# HELP dynamo_component_concurrent_requests Number of requests currently being processed by component endpoint
-# TYPE dynamo_component_concurrent_requests gauge
-dynamo_component_concurrent_requests{dynamo_component="example_component",dynamo_endpoint="example_endpoint9881",dynamo_namespace="example_namespace"} 0
+# HELP dynamo_component_inflight_requests Number of requests currently being processed by component endpoint
+# TYPE dynamo_component_inflight_requests gauge
+dynamo_component_inflight_requests{dynamo_component="example_component",dynamo_endpoint="example_endpoint9881",dynamo_namespace="example_namespace"} 0
 
 # HELP dynamo_component_bytes_processed_total Example of a custom metric. Total number of data bytes processed by system handler
 # TYPE dynamo_component_bytes_processed_total counter

lib/runtime/examples/system_metrics/tests/integration_test.rs

@@ -118,7 +118,7 @@ async fn test_backend_with_metrics() -> Result<()> {
                 println!("{}", metrics_content);
                 println!("=== END METRICS CONTENT ===");
 
-                // Parse and verify ingress metrics are greater than 0 (except concurrent_requests)
+                // Parse and verify ingress metrics are greater than 0 (except inflight_requests)
                 verify_ingress_metrics_greater_than_0(&metrics_content);
 
                 println!("Successfully retrieved and verified metrics!");
@@ -143,7 +143,7 @@ async fn test_backend_with_metrics() -> Result<()> {
 }
 
 fn verify_ingress_metrics_greater_than_0(metrics_content: &str) {
-    // Define the work handler metrics we want to verify (excluding concurrent_requests which can be 0)
+    // Define the work handler metrics we want to verify (excluding inflight_requests which can be 0)
     let metrics_to_verify = [
         "my_custom_bytes_processed_total",
         "requests_total",

lib/runtime/src/metrics.rs

@@ -1651,11 +1651,7 @@ mod test_metricsregistry_nats {
                 18000.0,
                 23000.0,
             ), // ~75-125% of 20660
-            (
-                build_metric_name(work_handler::CONCURRENT_REQUESTS),
-                0.0,
-                1.0,
-            ), // 0 or very low
+            (build_metric_name(work_handler::INFLIGHT_REQUESTS), 0.0, 1.0), // 0 or very low
             // Histograms have _{count,sum} suffixes
             (
                 format!(

lib/runtime/src/metrics/prometheus_names.rs

@@ -127,7 +127,7 @@ pub mod work_handler {
     pub const RESPONSE_BYTES_TOTAL: &str = "response_bytes_total";
 
     /// Number of requests currently being processed by work handler
-    pub const CONCURRENT_REQUESTS: &str = "concurrent_requests";
+    pub const INFLIGHT_REQUESTS: &str = "inflight_requests";
 
     /// Time spent processing requests by work handler (histogram)
     pub const REQUEST_DURATION_SECONDS: &str = "request_duration_seconds";

lib/runtime/src/pipeline/network/ingress/push_handler.rs

@@ -26,7 +26,7 @@ use tracing::Instrument;
 pub struct WorkHandlerMetrics {
     pub request_counter: IntCounter,
     pub request_duration: Histogram,
-    pub concurrent_requests: IntGauge,
+    pub inflight_requests: IntGauge,
     pub request_bytes: IntCounter,
     pub response_bytes: IntCounter,
     pub error_counter: IntCounterVec,
@@ -36,7 +36,7 @@ impl WorkHandlerMetrics {
     pub fn new(
         request_counter: IntCounter,
         request_duration: Histogram,
-        concurrent_requests: IntGauge,
+        inflight_requests: IntGauge,
         request_bytes: IntCounter,
         response_bytes: IntCounter,
         error_counter: IntCounterVec,
@@ -44,7 +44,7 @@ impl WorkHandlerMetrics {
         Self {
             request_counter,
             request_duration,
-            concurrent_requests,
+            inflight_requests,
             request_bytes,
             response_bytes,
             error_counter,
@@ -68,8 +68,8 @@ impl WorkHandlerMetrics {
             None,
         )?;
 
-        let concurrent_requests = endpoint.create_intgauge(
-            "concurrent_requests",
+        let inflight_requests = endpoint.create_intgauge(
+            "inflight_requests",
             "Number of requests currently being processed by work handler",
             &[],
         )?;
@@ -96,7 +96,7 @@ impl WorkHandlerMetrics {
         Ok(Self::new(
             request_counter,
             request_duration,
-            concurrent_requests,
+            inflight_requests,
             request_bytes,
             response_bytes,
             error_counter,
@@ -121,7 +121,7 @@ where
 
         if let Some(m) = self.metrics() {
             m.request_counter.inc();
-            m.concurrent_requests.inc();
+            m.inflight_requests.inc();
             m.request_bytes.inc_by(payload.len() as u64);
         }
 
@@ -289,7 +289,7 @@ where
         if let Some(m) = self.metrics() {
             let duration = start_time.elapsed();
             m.request_duration.observe(duration.as_secs_f64());
-            m.concurrent_requests.dec();
+            m.inflight_requests.dec();
         }
 
         Ok(())

tests/router/test_router_e2e_with_mockers.py

@@ -131,7 +131,7 @@ async def send_request_with_retry(url: str, payload: dict, max_retries: int = 4)
     return False
 
 
-async def send_concurrent_requests(urls: list, payload: dict, num_requests: int):
+async def send_inflight_requests(urls: list, payload: dict, num_requests: int):
     """Send multiple requests concurrently, alternating between URLs if multiple provided"""
 
     # First, send test requests with retry to ensure all systems are ready
@@ -228,7 +228,7 @@ def test_mocker_kv_router(request, runtime_services):
 
         # Use async to send requests concurrently for better performance
         asyncio.run(
-            send_concurrent_requests(
+            send_inflight_requests(
                 [
                     f"http://localhost:{frontend_port}/v1/chat/completions"
                 ],  # Pass as list
@@ -301,7 +301,7 @@ def test_mocker_two_kv_router(request, runtime_services):
 
         # Use async to send requests concurrently, alternating between routers
         asyncio.run(
-            send_concurrent_requests(
+            send_inflight_requests(
                 router_urls,
                 TEST_PAYLOAD,
                 NUM_REQUESTS,
@@ -404,7 +404,7 @@ def test_mocker_kv_router_overload_503(request, runtime_services):
 
         # First, send one request with retry to ensure system is ready
         logger.info("Sending initial request to ensure system is ready...")
-        asyncio.run(send_concurrent_requests([url], test_payload_503, 1))
+        asyncio.run(send_inflight_requests([url], test_payload_503, 1))
 
         # Now send 50 concurrent requests to exhaust resources, then verify 503
         logger.info("Sending 50 concurrent requests to exhaust resources...")