Skip to content

GitLab

Commit de290537 authored by Ryan Olson's avatar Ryan Olson Committed by GitHub
Browse files

refactor: prometheus upgrade (#452)

parent 5f9d1fc3
Show whitespace changes
Inline Side-by-side
Showing with 95 additions and 162 deletions
Cargo.lock
View file @ de290537
......@@ -160,9 +160,9 @@ dependencies = [
[[package]]
name = "async-nats"
version = "0.38.0"
version = "0.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76433c4de73442daedb3a59e991d94e85c14ebfc33db53dfcd347a21cd6ef4f8"
checksum = "e23419d455dc57d3ae60a2f4278cf561fc74fe866e548e14d2b0ad3e1b8ca0b2"
dependencies = [
"base64 0.22.1",
"bytes",
......@@ -3481,20 +3481,17 @@ dependencies = [
name = "metrics"
version = "0.1.0"
dependencies = [
"async-nats",
"axum 0.6.20",
"clap",
"dynamo-llm",
"dynamo-runtime",
"futures",
"opentelemetry",
"opentelemetry-prometheus",
"prometheus",
"rand 0.9.0",
"reqwest",
"serde",
"serde_json",
"thiserror 1.0.69",
"thiserror 2.0.12",
"tokio",
"tracing",
]
......@@ -4191,80 +4188,12 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e"
[[package]]
name = "opentelemetry"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54"
dependencies = [
"opentelemetry_api",
"opentelemetry_sdk",
]
[[package]]
name = "opentelemetry-prometheus"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7d81bc254e2d572120363a2b16cdb0d715d301b5789be0cfc26ad87e4e10e53"
dependencies = [
"once_cell",
"opentelemetry_api",
"opentelemetry_sdk",
"prometheus",
"protobuf",
]
[[package]]
name = "opentelemetry_api"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b"
dependencies = [
"futures-channel",
"futures-util",
"indexmap 1.9.3",
"js-sys",
"once_cell",
"pin-project-lite",
"thiserror 1.0.69",
"urlencoding",
]
[[package]]
name = "opentelemetry_sdk"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026"
dependencies = [
"async-trait",
"crossbeam-channel",
"futures-channel",
"futures-executor",
"futures-util",
"once_cell",
"opentelemetry_api",
"ordered-float",
"percent-encoding",
"rand 0.8.5",
"regex",
"thiserror 1.0.69",
]
[[package]]
name = "option-ext"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d"
[[package]]
name = "ordered-float"
version = "3.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc"
dependencies = [
"num-traits",
]
[[package]]
name = "overload"
version = "0.1.1"
......@@ -4576,9 +4505,9 @@ dependencies = [
[[package]]
name = "prometheus"
version = "0.13.4"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1"
checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
dependencies = [
"cfg-if 1.0.0",
"fnv",
......@@ -4586,7 +4515,7 @@ dependencies = [
"memchr",
"parking_lot",
"protobuf",
"thiserror 1.0.69",
"thiserror 2.0.12",
]
[[package]]
......@@ -4686,9 +4615,23 @@ dependencies = [
[[package]]
name = "protobuf"
version = "2.28.0"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
dependencies = [
"once_cell",
"protobuf-support",
"thiserror 1.0.69",
]
[[package]]
name = "protobuf-support"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "pulp"
......@@ -5428,7 +5371,6 @@ version = "0.102.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
dependencies = [
"ring",
"rustls-pki-types",
"untrusted",
]
......@@ -6417,11 +6359,11 @@ dependencies = [
"httparse",
"rand 0.8.5",
"ring",
"rustls-native-certs 0.8.1",
"rustls-pki-types",
"tokio",
"tokio-rustls",
"tokio-util",
"webpki-roots",
]
[[package]]
......@@ -6866,12 +6808,6 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "urlencoding"
version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da"
[[package]]
name = "utf16_iter"
version = "1.0.5"
......
Cargo.toml
View file @ de290537
......@@ -40,7 +40,7 @@ dynamo-llm = { path = "lib/llm" }
# External dependencies
anyhow = { version = "1" }
async-nats = { version = "0.38", features = ["service"] }
async-nats = { version = "0.40", features = ["service"] }
async-stream = { version = "0.3" }
async-trait = { version = "0.1" }
async_zmq = { version = "0.4.0" }
......@@ -53,6 +53,7 @@ either = { version = "1.13", features = ["serde"] }
futures = { version = "0.3" }
hf-hub = { version = "0.4.2", default-features = false, features = ["tokio", "rustls-tls"] }
humantime = { version = "2.2.0" }
prometheus = { version = "0.14" }
rand = { version = "0.9.0" }
serde = { version = "1", features = ["derive"] }
serde_json = { version = "1" }
......@@ -66,7 +67,6 @@ validator = { version = "0.20.0", features = ["derive"] }
uuid = { version = "1", features = ["v4", "serde"] }
xxhash-rust = { version = "0.8", features = ["xxh3", "const_xxh3"] }
strum = { version = "0.27", features = ["derive"] }
prometheus = { version = "0.13" }
[profile.dev.package]
insta.opt-level = 3
......
components/metrics/Cargo.toml
View file @ de290537
......@@ -27,17 +27,15 @@ dynamo-llm = { workspace = true }
dynamo-runtime = { workspace = true }
futures = { workspace = true }
prometheus = { workspace = true }
rand = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
tracing = { workspace = true }
async-nats = { version = "0.38", features = ["service"] }
# TODO: Update axum to 0.8
axum = { version = "0.6" }
clap = { version = "4.5", features = ["derive", "env"] }
opentelemetry = { version = "0.20" }
opentelemetry-prometheus = { version = "0.13" }
prometheus = { version = "0.13" }
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
thiserror = { version = "1.0" }
components/metrics/src/bin/mock_worker.rs
View file @ de290537
......@@ -13,12 +13,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use async_nats::service::endpoint::Stats;
use dynamo_llm::kv_router::{
protocols::ForwardPassMetrics, scheduler::KVHitRateEvent, KV_HIT_RATE_SUBJECT,
};
use dynamo_runtime::{
component::Namespace,
component::{service::EndpointStats, Namespace},
logging,
pipeline::{
async_trait, network::Ingress, AsyncEngine, AsyncEngineContextProvider, Error, ManyOut,
......@@ -107,7 +106,7 @@ async fn mock_event_publisher(namespace: Namespace) {
}
/// Generates mock forward pass metrics for stats handler
fn mock_stats_handler(_stats: Stats) -> serde_json::Value {
fn mock_stats_handler(_stats: EndpointStats) -> serde_json::Value {
let request_total_slots = 100;
let request_active_slots = rand::rng().random_range(0..=request_total_slots);
let kv_total_blocks = 100;
......
components/metrics/src/lib.rs
View file @ de290537
......@@ -429,7 +429,7 @@ impl PrometheusMetrics {
&self,
gauge: &prometheus::GaugeVec,
config: &LLMWorkerLoadCapacityConfig,
worker_id: &str,
worker_id: &String,
value: f64,
) {
gauge
......@@ -442,7 +442,7 @@ impl PrometheusMetrics {
&self,
counter: &prometheus::CounterVec,
config: &LLMWorkerLoadCapacityConfig,
worker_id: &str,
worker_id: &String,
value: f64,
) {
counter
......
lib/bindings/python/Cargo.lock
View file @ de290537
......@@ -138,9 +138,9 @@ dependencies = [
[[package]]
name = "async-nats"
version = "0.38.0"
version = "0.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76433c4de73442daedb3a59e991d94e85c14ebfc33db53dfcd347a21cd6ef4f8"
checksum = "e23419d455dc57d3ae60a2f4278cf561fc74fe866e548e14d2b0ad3e1b8ca0b2"
dependencies = [
"base64 0.22.1",
"bytes",
......@@ -3144,9 +3144,9 @@ dependencies = [
[[package]]
name = "prometheus"
version = "0.13.4"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1"
checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
dependencies = [
"cfg-if 1.0.0",
"fnv",
......@@ -3154,7 +3154,7 @@ dependencies = [
"memchr",
"parking_lot",
"protobuf",
"thiserror 1.0.69",
"thiserror 2.0.12",
]
[[package]]
......@@ -3211,9 +3211,23 @@ dependencies = [
[[package]]
name = "protobuf"
version = "2.28.0"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
dependencies = [
"once_cell",
"protobuf-support",
"thiserror 1.0.69",
]
[[package]]
name = "protobuf-support"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "pulp"
......@@ -3781,7 +3795,6 @@ version = "0.102.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
dependencies = [
"ring",
"rustls-pki-types",
"untrusted",
]
......@@ -4481,11 +4494,11 @@ dependencies = [
"httparse",
"rand 0.8.5",
"ring",
"rustls-native-certs 0.8.1",
"rustls-pki-types",
"tokio",
"tokio-rustls",
"tokio-util",
"webpki-roots",
]
[[package]]
......
lib/runtime/examples/Cargo.lock
View file @ de290537
......@@ -61,9 +61,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50"
[[package]]
name = "async-nats"
version = "0.38.0"
version = "0.40.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "76433c4de73442daedb3a59e991d94e85c14ebfc33db53dfcd347a21cd6ef4f8"
checksum = "e23419d455dc57d3ae60a2f4278cf561fc74fe866e548e14d2b0ad3e1b8ca0b2"
dependencies = [
"base64",
"bytes",
......@@ -77,7 +77,7 @@ dependencies = [
"rand 0.8.5",
"regex",
"ring",
"rustls-native-certs 0.7.3",
"rustls-native-certs",
"rustls-pemfile",
"rustls-webpki 0.102.8",
"serde",
......@@ -378,16 +378,6 @@ dependencies = [
"libc",
]
[[package]]
name = "core-foundation"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b55271e5c8c478ad3f38ad24ef34923091e0548492a266d19b3c0b4d82574c63"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "core-foundation-sys"
version = "0.8.7"
......@@ -1846,9 +1836,9 @@ dependencies = [
[[package]]
name = "prometheus"
version = "0.13.4"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d33c28a30771f7f96db69893f78b857f7450d7e0237e9c8fc6427a81bae7ed1"
checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a"
dependencies = [
"cfg-if 1.0.0",
"fnv",
......@@ -1856,7 +1846,7 @@ dependencies = [
"memchr",
"parking_lot",
"protobuf",
"thiserror 1.0.69",
"thiserror 2.0.12",
]
[[package]]
......@@ -1913,9 +1903,23 @@ dependencies = [
[[package]]
name = "protobuf"
version = "2.28.0"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94"
checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
dependencies = [
"once_cell",
"protobuf-support",
"thiserror 1.0.69",
]
[[package]]
name = "protobuf-support"
version = "3.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
dependencies = [
"thiserror 1.0.69",
]
[[package]]
name = "quote"
......@@ -2131,19 +2135,7 @@ dependencies = [
"rustls-pemfile",
"rustls-pki-types",
"schannel",
"security-framework 2.11.1",
]
[[package]]
name = "rustls-native-certs"
version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3"
dependencies = [
"openssl-probe",
"rustls-pki-types",
"schannel",
"security-framework 3.2.0",
"security-framework",
]
[[package]]
......@@ -2167,7 +2159,6 @@ version = "0.102.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9"
dependencies = [
"ring",
"rustls-pki-types",
"untrusted",
]
......@@ -2226,20 +2217,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
dependencies = [
"bitflags 2.9.0",
"core-foundation 0.9.4",
"core-foundation-sys",
"libc",
"security-framework-sys",
]
[[package]]
name = "security-framework"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "271720403f46ca04f7ba6f55d438f8bd878d6b8ca0a1046e8228c4145bcbb316"
dependencies = [
"bitflags 2.9.0",
"core-foundation 0.10.0",
"core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
......@@ -2684,11 +2662,11 @@ dependencies = [
"httparse",
"rand 0.8.5",
"ring",
"rustls-native-certs 0.8.1",
"rustls-pki-types",
"tokio",
"tokio-rustls",
"tokio-util",
"webpki-roots",
]
[[package]]
......@@ -3107,6 +3085,15 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "webpki-roots"
version = "0.26.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2210b291f7ea53617fbafcc4939f10914214ec15aace5ba62293a668f322c5c9"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "winapi"
version = "0.2.8"
......
lib/runtime/src/component.rs
View file @ de290537
......@@ -66,7 +66,7 @@ mod component;
mod endpoint;
mod namespace;
mod registry;
mod service;
pub mod service;
pub use client::Client;
......
lib/runtime/src/component/endpoint.rs
View file @ de290537
......@@ -17,6 +17,8 @@ use derive_getters::Dissolve;
use super::*;
pub use async_nats::service::endpoint::Stats as EndpointStats;
#[derive(Educe, Builder, Dissolve)]
#[educe(Debug)]
#[builder(pattern = "owned", build_fn(private, name = "build_internal"))]
......@@ -47,7 +49,7 @@ impl EndpointConfigBuilder {
pub fn stats_handler<F>(self, handler: F) -> Self
where
F: FnMut(async_nats::service::endpoint::Stats) -> serde_json::Value + Send + Sync + 'static,
F: FnMut(EndpointStats) -> serde_json::Value + Send + Sync + 'static,
{
self._stats_handler(Some(Box::new(handler)))
}
......
lib/runtime/src/component/service.rs
View file @ de290537
......@@ -19,13 +19,11 @@ use std::sync::Mutex;
use super::*;
use async_nats::service::{endpoint, Service};
pub use super::endpoint::EndpointStats;
pub type StatsHandler =
Box<dyn FnMut(String, endpoint::Stats) -> serde_json::Value + Send + Sync + 'static>;
Box<dyn FnMut(String, EndpointStats) -> serde_json::Value + Send + Sync + 'static>;
pub type EndpointStatsHandler =
Box<dyn FnMut(endpoint::Stats) -> serde_json::Value + Send + Sync + 'static>;
Box<dyn FnMut(EndpointStats) -> serde_json::Value + Send + Sync + 'static>;
pub const PROJECT_NAME: &str = "Dynamo";
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment