lib.rs 3.28 KB
Newer Older
1
2
use pyo3::prelude::*;
pub mod router;
3
pub mod server;
4
pub mod tree;
5

6
7
8
9
10
#[pyclass(eq)]
#[derive(Clone, PartialEq)]
pub enum PolicyType {
    Random,
    RoundRobin,
11
    CacheAware,
12
13
}

14
15
16
17
18
#[pyclass]
struct Router {
    host: String,
    port: u16,
    worker_urls: Vec<String>,
19
    policy: PolicyType,
20
    worker_startup_timeout_secs: u64,
21
    cache_threshold: f32,
22
23
    balance_abs_threshold: usize,
    balance_rel_threshold: f32,
24
25
    eviction_interval_secs: u64,
    max_tree_size: usize,
26
    max_payload_size: usize,
27
    verbose: bool,
28
29
30
31
32
}

#[pymethods]
impl Router {
    #[new]
33
34
35
36
37
    #[pyo3(signature = (
        worker_urls,
        policy = PolicyType::RoundRobin,
        host = String::from("127.0.0.1"),
        port = 3001,
38
        worker_startup_timeout_secs = 300,
39
        cache_threshold = 0.50,
40
41
        balance_abs_threshold = 32,
        balance_rel_threshold = 1.0001,
42
        eviction_interval_secs = 60,
43
        max_tree_size = 2usize.pow(24),
44
        max_payload_size = 4 * 1024 * 1024,
45
        verbose = false
46
47
48
49
50
51
    ))]
    fn new(
        worker_urls: Vec<String>,
        policy: PolicyType,
        host: String,
        port: u16,
52
        worker_startup_timeout_secs: u64,
53
        cache_threshold: f32,
54
55
        balance_abs_threshold: usize,
        balance_rel_threshold: f32,
56
57
        eviction_interval_secs: u64,
        max_tree_size: usize,
58
        max_payload_size: usize,
59
        verbose: bool,
60
61
    ) -> PyResult<Self> {
        Ok(Router {
62
63
64
            host,
            port,
            worker_urls,
65
            policy,
66
            worker_startup_timeout_secs,
67
            cache_threshold,
68
69
            balance_abs_threshold,
            balance_rel_threshold,
70
71
            eviction_interval_secs,
            max_tree_size,
72
            max_payload_size,
73
            verbose,
74
        })
75
76
77
    }

    fn start(&self) -> PyResult<()> {
78
        let policy_config = match &self.policy {
79
80
81
82
83
84
            PolicyType::Random => router::PolicyConfig::RandomConfig {
                timeout_secs: self.worker_startup_timeout_secs,
            },
            PolicyType::RoundRobin => router::PolicyConfig::RoundRobinConfig {
                timeout_secs: self.worker_startup_timeout_secs,
            },
85
            PolicyType::CacheAware => router::PolicyConfig::CacheAwareConfig {
86
                timeout_secs: self.worker_startup_timeout_secs,
87
                cache_threshold: self.cache_threshold,
88
89
                balance_abs_threshold: self.balance_abs_threshold,
                balance_rel_threshold: self.balance_rel_threshold,
90
91
                eviction_interval_secs: self.eviction_interval_secs,
                max_tree_size: self.max_tree_size,
92
93
            },
        };
94
95

        actix_web::rt::System::new().block_on(async move {
96
97
98
99
100
101
            server::startup(server::ServerConfig {
                host: self.host.clone(),
                port: self.port,
                worker_urls: self.worker_urls.clone(),
                policy_config,
                verbose: self.verbose,
102
                max_payload_size: self.max_payload_size,
103
104
            })
            .await
105
106
107
            .map_err(|e| pyo3::exceptions::PyRuntimeError::new_err(e.to_string()))?;
            Ok(())
        })
108
109
110
111
    }
}

#[pymodule]
112
fn sglang_router_rs(m: &Bound<'_, PyModule>) -> PyResult<()> {
113
    m.add_class::<PolicyType>()?;
114
115
    m.add_class::<Router>()?;
    Ok(())
116
}