fix: recovery etc order, full recovery, thundering herd on same worker ids in same order. (#8016)

Signed-off-by: michaelfeil <63565275+michaelfeil@users.noreply.github.com>

lib/llm/src/kv_router/indexer/worker_query.rs

@@ -18,6 +18,7 @@ use dynamo_runtime::protocols::maybe_error::MaybeError;
 use dynamo_runtime::stream;
 use dynamo_runtime::traits::DistributedRuntimeProvider;
 use futures::StreamExt;
+use rand::Rng;
 use tokio::sync::{Mutex, Semaphore};
 
 use super::Indexer;
@@ -459,6 +460,14 @@ impl WorkerQueryClient {
         let client = self.clone();
 
         tokio::spawn(async move {
+            // Add jitter only for full-restore (start_event_id is None)
+            // to permute semaphore acquisition order and reduce thundering herd risk on initial discovery.
+            // This distributes load when multiple routers start simultaneously.
+            if start_event_id.is_none() {
+                let jitter_us = rand::rng().random_range(0..3000u64);
+                tokio::time::sleep(Duration::from_micros(jitter_us)).await;
+            }
+
             let Ok(_permit) = client.recovery_semaphore.clone().acquire_owned().await else {
                 return;
             };