Skip to content

GitLab

"docs/vscode:/vscode.git/clone" did not exist on "5f6e22dc3b5058a6f4635ccac45ca99efa4cb6d7"
Unverified Commit 3a8428ec authored by Byron Hsu's avatar Byron Hsu Committed by GitHub
Browse files

[router] Expose worker startup interval (#3019)

parent 0311ce8e
Hide whitespace changes
Inline Side-by-side
Showing with 72 additions and 13 deletions
sgl-router/py_src/sglang_router/launch_router.py
View file @ 3a8428ec
......@@ -34,6 +34,7 @@ class RouterArgs:
# Routing policy
policy: str = "cache_aware"
worker_startup_timeout_secs: int = 300
worker_startup_check_interval: int = 10
cache_threshold: float = 0.5
balance_abs_threshold: int = 32
balance_rel_threshold: float = 1.0001
......@@ -94,6 +95,12 @@ class RouterArgs:
default=RouterArgs.worker_startup_timeout_secs,
help="Timeout in seconds for worker startup",
)
parser.add_argument(
f"--{prefix}worker-startup-check-interval",
type=int,
default=RouterArgs.worker_startup_check_interval,
help="Interval in seconds between checks for worker startup",
)
parser.add_argument(
f"--{prefix}cache-threshold",
type=float,
......@@ -157,6 +164,9 @@ class RouterArgs:
worker_startup_timeout_secs=getattr(
args, f"{prefix}worker_startup_timeout_secs"
),
worker_startup_check_interval=getattr(
args, f"{prefix}worker_startup_check_interval"
),
cache_threshold=getattr(args, f"{prefix}cache_threshold"),
balance_abs_threshold=getattr(args, f"{prefix}balance_abs_threshold"),
balance_rel_threshold=getattr(args, f"{prefix}balance_rel_threshold"),
......@@ -202,6 +212,7 @@ def launch_router(args: argparse.Namespace) -> Optional[Router]:
port=router_args.port,
policy=policy_from_str(router_args.policy),
worker_startup_timeout_secs=router_args.worker_startup_timeout_secs,
worker_startup_check_interval=router_args.worker_startup_check_interval,
cache_threshold=router_args.cache_threshold,
balance_abs_threshold=router_args.balance_abs_threshold,
balance_rel_threshold=router_args.balance_rel_threshold,
......
sgl-router/py_src/sglang_router/router.py
View file @ 3a8428ec
......@@ -18,6 +18,7 @@ class Router:
host: Host address to bind the router server. Default: '127.0.0.1'
port: Port number to bind the router server. Default: 3001
worker_startup_timeout_secs: Timeout in seconds for worker startup. Default: 300
worker_startup_check_interval: Interval in seconds between checks for worker initialization. Default: 10
cache_threshold: Cache threshold (0.0-1.0) for cache-aware routing. Routes to cached worker
if the match rate exceeds threshold, otherwise routes to the worker with the smallest
tree. Default: 0.5
......@@ -39,6 +40,7 @@ class Router:
host: str = "127.0.0.1",
port: int = 3001,
worker_startup_timeout_secs: int = 300,
worker_startup_check_interval: int = 10,
cache_threshold: float = 0.50,
balance_abs_threshold: int = 32,
balance_rel_threshold: float = 1.0001,
......@@ -53,6 +55,7 @@ class Router:
host=host,
port=port,
worker_startup_timeout_secs=worker_startup_timeout_secs,
worker_startup_check_interval=worker_startup_check_interval,
cache_threshold=cache_threshold,
balance_abs_threshold=balance_abs_threshold,
balance_rel_threshold=balance_rel_threshold,
......
sgl-router/py_test/test_launch_router.py
View file @ 3a8428ec
......@@ -29,6 +29,7 @@ class TestLaunchRouter(unittest.TestCase):
port=30000,
policy="cache_aware",
worker_startup_timeout_secs=600,
worker_startup_check_interval=10,
cache_threshold=0.5,
balance_abs_threshold=32,
balance_rel_threshold=1.0001,
......
sgl-router/src/lib.rs
View file @ 3a8428ec
......@@ -18,6 +18,7 @@ struct Router {
worker_urls: Vec<String>,
policy: PolicyType,
worker_startup_timeout_secs: u64,
worker_startup_check_interval: u64,
cache_threshold: f32,
balance_abs_threshold: usize,
balance_rel_threshold: f32,
......@@ -36,6 +37,7 @@ impl Router {
host = String::from("127.0.0.1"),
port = 3001,
worker_startup_timeout_secs = 300,
worker_startup_check_interval = 10,
cache_threshold = 0.50,
balance_abs_threshold = 32,
balance_rel_threshold = 1.0001,
......@@ -50,6 +52,7 @@ impl Router {
host: String,
port: u16,
worker_startup_timeout_secs: u64,
worker_startup_check_interval: u64,
cache_threshold: f32,
balance_abs_threshold: usize,
balance_rel_threshold: f32,
......@@ -64,6 +67,7 @@ impl Router {
worker_urls,
policy,
worker_startup_timeout_secs,
worker_startup_check_interval,
cache_threshold,
balance_abs_threshold,
balance_rel_threshold,
......@@ -78,12 +82,15 @@ impl Router {
let policy_config = match &self.policy {
PolicyType::Random => router::PolicyConfig::RandomConfig {
timeout_secs: self.worker_startup_timeout_secs,
interval_secs: self.worker_startup_check_interval,
},
PolicyType::RoundRobin => router::PolicyConfig::RoundRobinConfig {
timeout_secs: self.worker_startup_timeout_secs,
interval_secs: self.worker_startup_check_interval,
},
PolicyType::CacheAware => router::PolicyConfig::CacheAwareConfig {
timeout_secs: self.worker_startup_timeout_secs,
interval_secs: self.worker_startup_check_interval,
cache_threshold: self.cache_threshold,
balance_abs_threshold: self.balance_abs_threshold,
balance_rel_threshold: self.balance_rel_threshold,
......
sgl-router/src/router.rs
View file @ 3a8428ec
......@@ -18,10 +18,12 @@ pub enum Router {
worker_urls: Arc<RwLock<Vec<String>>>,
current_index: AtomicUsize,
timeout_secs: u64,
interval_secs: u64,
},
Random {
worker_urls: Arc<RwLock<Vec<String>>>,
timeout_secs: u64,
interval_secs: u64,
},
CacheAware {
/*
......@@ -92,6 +94,7 @@ pub enum Router {
balance_abs_threshold: usize,
balance_rel_threshold: f32,
timeout_secs: u64,
interval_secs: u64,
_eviction_thread: Option<thread::JoinHandle<()>>,
},
}
......@@ -100,9 +103,11 @@ pub enum Router {
pub enum PolicyConfig {
RandomConfig {
timeout_secs: u64,
interval_secs: u64,
},
RoundRobinConfig {
timeout_secs: u64,
interval_secs: u64,
},
CacheAwareConfig {
cache_threshold: f32,
......@@ -111,31 +116,50 @@ pub enum PolicyConfig {
eviction_interval_secs: u64,
max_tree_size: usize,
timeout_secs: u64,
interval_secs: u64,
},
}
impl Router {
pub fn new(worker_urls: Vec<String>, policy_config: PolicyConfig) -> Result<Self, String> {
// Get timeout from policy config
let timeout_secs = match &policy_config {
PolicyConfig::RandomConfig { timeout_secs } => *timeout_secs,
PolicyConfig::RoundRobinConfig { timeout_secs } => *timeout_secs,
PolicyConfig::CacheAwareConfig { timeout_secs, .. } => *timeout_secs,
// Get timeout and interval from policy config
let (timeout_secs, interval_secs) = match &policy_config {
PolicyConfig::RandomConfig {
timeout_secs,
interval_secs,
} => (*timeout_secs, *interval_secs),
PolicyConfig::RoundRobinConfig {
timeout_secs,
interval_secs,
} => (*timeout_secs, *interval_secs),
PolicyConfig::CacheAwareConfig {
timeout_secs,
interval_secs,
..
} => (*timeout_secs, *interval_secs),
};
// Wait until all workers are healthy
Self::wait_for_healthy_workers(&worker_urls, timeout_secs, 10)?;
Self::wait_for_healthy_workers(&worker_urls, timeout_secs, interval_secs)?;
// Create router based on policy...
Ok(match policy_config {
PolicyConfig::RandomConfig { timeout_secs } => Router::Random {
PolicyConfig::RandomConfig {
timeout_secs,
interval_secs,
} => Router::Random {
worker_urls: Arc::new(RwLock::new(worker_urls)),
timeout_secs,
interval_secs,
},
PolicyConfig::RoundRobinConfig { timeout_secs } => Router::RoundRobin {
PolicyConfig::RoundRobinConfig {
timeout_secs,
interval_secs,
} => Router::RoundRobin {
worker_urls: Arc::new(RwLock::new(worker_urls)),
current_index: std::sync::atomic::AtomicUsize::new(0),
timeout_secs,
interval_secs,
},
PolicyConfig::CacheAwareConfig {
cache_threshold,
......@@ -144,6 +168,7 @@ impl Router {
eviction_interval_secs,
max_tree_size,
timeout_secs,
interval_secs,
} => {
let mut running_queue = HashMap::new();
for url in &worker_urls {
......@@ -195,6 +220,7 @@ impl Router {
balance_abs_threshold,
balance_rel_threshold,
timeout_secs,
interval_secs,
_eviction_thread: Some(eviction_thread),
}
}
......@@ -594,11 +620,22 @@ impl Router {
}
pub async fn add_worker(&self, worker_url: &str) -> Result<String, String> {
let interval_secs = 10; // check every 10 seconds
let timeout_secs = match self {
Router::Random { timeout_secs, .. } => *timeout_secs,
Router::RoundRobin { timeout_secs, .. } => *timeout_secs,
Router::CacheAware { timeout_secs, .. } => *timeout_secs,
let (timeout_secs, interval_secs) = match self {
Router::Random {
timeout_secs,
interval_secs,
..
} => (*timeout_secs, *interval_secs),
Router::RoundRobin {
timeout_secs,
interval_secs,
..
} => (*timeout_secs, *interval_secs),
Router::CacheAware {
timeout_secs,
interval_secs,
..
} => (*timeout_secs, *interval_secs),
};
let start_time = std::time::Instant::now();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment