feat(kv-router): pluggable scheduling policy for router queue [DYN-2454] (#7260)

Signed-off-by: PeaBrane <yanrpei@gmail.com>

lib/llm/src/kv_router/scheduler.rs

 // SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
 
+pub use dynamo_kv_router::scheduling::policy::RouterSchedulingPolicy;
 pub use dynamo_kv_router::scheduling::{
     KvSchedulerError, PotentialLoad, SchedulingRequest, SchedulingResponse,
 };
@@ -100,12 +101,20 @@ impl KvScheduler {
         let (request_tx, request_rx) = tokio::sync::mpsc::channel::<SchedulingRequest>(1024);
         let scheduler_cancel_token = component.drt().primary_token();
 
+        let policy =
+            RouterSchedulingPolicy::new(kv_router_config.router_queue_policy, block_size as usize);
+        tracing::info!(
+            "Router queue policy: {}",
+            kv_router_config.router_queue_policy
+        );
+
         let queue = Arc::new(SchedulerQueue::new(
             slots.clone(),
             workers_with_configs.clone(),
             kv_router_config.router_queue_threshold,
             block_size,
             selector,
+            policy,
         ));
         let queue_clone = queue.clone();
 

lib/mocker/src/kv_manager/vllm_backend.rs

@@ -213,7 +213,7 @@ impl KvManager {
                     // If at max capacity, evict the oldest entry from inactive blocks
                     if self.cache.is_at_capacity() {
                         let Some(evicted) = self.cache.evict_inactive() else {
-                            return allocated;
+                            break;
                         };
                         tracing::trace!(
                             "Evicting block from inactive pool: {evicted:?}, dp_rank={}",

lib/runtime/src/config/environment_names.rs

@@ -331,6 +331,16 @@ pub mod model {
     }
 }
 
+/// KV Router configuration environment variables
+pub mod router {
+    /// Queue threshold fraction for prefill token capacity.
+    /// When set, requests are queued if all workers exceed this fraction of max_num_batched_tokens.
+    pub const DYN_ROUTER_QUEUE_THRESHOLD: &str = "DYN_ROUTER_QUEUE_THRESHOLD";
+
+    /// Scheduling policy for the router queue ("fcfs" or "wspt").
+    pub const DYN_ROUTER_QUEUE_POLICY: &str = "DYN_ROUTER_QUEUE_POLICY";
+}
+
 /// Event Plane transport environment variables
 pub mod event_plane {
     /// Event transport selection: "zmq" or "nats". Default: "nats"
@@ -481,6 +491,9 @@ mod tests {
             model::huggingface::HF_HUB_CACHE,
             model::huggingface::HF_HOME,
             model::huggingface::HF_HUB_OFFLINE,
+            // Router
+            router::DYN_ROUTER_QUEUE_THRESHOLD,
+            router::DYN_ROUTER_QUEUE_POLICY,
             // Event Plane
             event_plane::DYN_EVENT_PLANE,
             event_plane::DYN_EVENT_PLANE_CODEC,