metrics.rs 42.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
// SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//! Metric Registry Framework for Dynamo.
//!
//! This module provides registry classes for Prometheus metrics
19
20
//! that auto populates the labels with the component-endpoint hierarchy.
//! All metrics are prefixed with "dynamo_component_" to avoid collisions with Kubernetes and other monitoring system labels.
21

22
23
use once_cell::sync::Lazy;
use regex::Regex;
24
25
26
27
use std::any::Any;
use std::collections::HashMap;
use std::sync::{Arc, Mutex};

28
29
// If set to true, then metrics will be labeled with the dynamo_namespace, dynamo_component, and dynamo_endpoint.
// These labels are prefixed with "dynamo_" to avoid collisions with Kubernetes and other monitoring system labels.
30
31
32
33
34
pub const USE_AUTO_LABELS: bool = true;

// Prometheus imports
use prometheus::Encoder;

35
36
fn build_metric_name(metric_name: &str) -> String {
    format!("dynamo_component_{}", metric_name)
37
}
38

39
40
41
42
43
44
45
/// Lints a metric name component by stripping off invalid characters and validating Prometheus naming pattern
/// Prometheus doesn't provide a built-in function to validate metric names, but the specification requires
/// names to follow the pattern [a-zA-Z_:][a-zA-Z0-9_:]*. This function implements that validation.
/// Returns error if sanitized name doesn't follow the required pattern.
fn lint_prometheus_name(name: &str) -> anyhow::Result<String> {
    if name.is_empty() {
        return Ok("".to_string());
46
    }
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65

    static INVALID_CHARS_PATTERN: Lazy<Regex> =
        Lazy::new(|| Regex::new(r"[^a-zA-Z0-9_:]").unwrap());

    static PROMETHEUS_NAME_PATTERN: Lazy<Regex> =
        Lazy::new(|| Regex::new(r"^[a-zA-Z_:][a-zA-Z0-9_:]*$").unwrap());

    // Remove all invalid characters (everything except alphanumeric, colons, and underscores)
    let sanitized = INVALID_CHARS_PATTERN.replace_all(name, "").to_string();

    // Check if the sanitized name follows Prometheus naming pattern
    if !sanitized.is_empty() && !PROMETHEUS_NAME_PATTERN.is_match(&sanitized) {
        return Err(anyhow::anyhow!(
            "Sanitized name '{}' does not follow Prometheus naming pattern [a-zA-Z_:][a-zA-Z0-9_:]*",
            sanitized
        ));
    }

    Ok(sanitized)
66
67
}

68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
/// Validate that a label slice has no duplicate keys.
/// Returns Ok(()) when all keys are unique; otherwise returns an error naming the duplicate key.
fn validate_no_duplicate_label_keys(labels: &[(&str, &str)]) -> anyhow::Result<()> {
    let mut seen_keys = std::collections::HashSet::new();
    for (key, _) in labels {
        if !seen_keys.insert(*key) {
            return Err(anyhow::anyhow!(
                "Duplicate label key '{}' found in labels",
                key
            ));
        }
    }
    Ok(())
}

83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
/// Trait that defines common behavior for Prometheus metric types
pub trait PrometheusMetric: prometheus::core::Collector + Clone + Send + Sync + 'static {
    /// Create a new metric with the given options
    fn with_opts(opts: prometheus::Opts) -> Result<Self, prometheus::Error>
    where
        Self: Sized;

    /// Create a new metric with histogram options and custom buckets
    /// This is a default implementation that will panic for non-histogram metrics
    fn with_histogram_opts_and_buckets(
        _opts: prometheus::HistogramOpts,
        _buckets: Option<Vec<f64>>,
    ) -> Result<Self, prometheus::Error>
    where
        Self: Sized,
    {
        panic!("with_histogram_opts_and_buckets is not implemented for this metric type");
    }

    /// Create a new metric with counter options and label names (for CounterVec)
    /// This is a default implementation that will panic for non-countervec metrics
    fn with_opts_and_label_names(
        _opts: prometheus::Opts,
        _label_names: &[&str],
    ) -> Result<Self, prometheus::Error>
    where
        Self: Sized,
    {
        panic!("with_opts_and_label_names is not implemented for this metric type");
    }
}

// Implement the trait for Counter, IntCounter, and Gauge
impl PrometheusMetric for prometheus::Counter {
    fn with_opts(opts: prometheus::Opts) -> Result<Self, prometheus::Error> {
        prometheus::Counter::with_opts(opts)
    }
}

impl PrometheusMetric for prometheus::IntCounter {
    fn with_opts(opts: prometheus::Opts) -> Result<Self, prometheus::Error> {
        prometheus::IntCounter::with_opts(opts)
    }
}

impl PrometheusMetric for prometheus::Gauge {
    fn with_opts(opts: prometheus::Opts) -> Result<Self, prometheus::Error> {
        prometheus::Gauge::with_opts(opts)
    }
}

impl PrometheusMetric for prometheus::IntGauge {
    fn with_opts(opts: prometheus::Opts) -> Result<Self, prometheus::Error> {
        prometheus::IntGauge::with_opts(opts)
    }
}

impl PrometheusMetric for prometheus::IntGaugeVec {
    fn with_opts(_opts: prometheus::Opts) -> Result<Self, prometheus::Error> {
        Err(prometheus::Error::Msg(
            "IntGaugeVec requires label names, use with_opts_and_label_names instead".to_string(),
        ))
    }

    fn with_opts_and_label_names(
        opts: prometheus::Opts,
        label_names: &[&str],
    ) -> Result<Self, prometheus::Error> {
        prometheus::IntGaugeVec::new(opts, label_names)
    }
}

155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
impl PrometheusMetric for prometheus::IntCounterVec {
    fn with_opts(_opts: prometheus::Opts) -> Result<Self, prometheus::Error> {
        Err(prometheus::Error::Msg(
            "IntCounterVec requires label names, use with_opts_and_label_names instead".to_string(),
        ))
    }

    fn with_opts_and_label_names(
        opts: prometheus::Opts,
        label_names: &[&str],
    ) -> Result<Self, prometheus::Error> {
        prometheus::IntCounterVec::new(opts, label_names)
    }
}

170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
// Implement the trait for Histogram
impl PrometheusMetric for prometheus::Histogram {
    fn with_opts(opts: prometheus::Opts) -> Result<Self, prometheus::Error> {
        // Convert Opts to HistogramOpts
        let histogram_opts = prometheus::HistogramOpts::new(opts.name, opts.help);
        prometheus::Histogram::with_opts(histogram_opts)
    }

    fn with_histogram_opts_and_buckets(
        mut opts: prometheus::HistogramOpts,
        buckets: Option<Vec<f64>>,
    ) -> Result<Self, prometheus::Error> {
        if let Some(custom_buckets) = buckets {
            opts = opts.buckets(custom_buckets);
        }
        prometheus::Histogram::with_opts(opts)
    }
}

// Implement the trait for CounterVec
impl PrometheusMetric for prometheus::CounterVec {
    fn with_opts(_opts: prometheus::Opts) -> Result<Self, prometheus::Error> {
        // This will panic - CounterVec needs label names
        panic!("CounterVec requires label names, use with_opts_and_label_names instead");
    }

    fn with_opts_and_label_names(
        opts: prometheus::Opts,
        label_names: &[&str],
    ) -> Result<Self, prometheus::Error> {
        prometheus::CounterVec::new(opts, label_names)
    }
}

/// Private helper function to create metrics - not accessible to trait implementors
fn create_metric<T: PrometheusMetric, R: MetricsRegistry + ?Sized>(
    registry: &R,
    metric_name: &str,
    metric_desc: &str,
    labels: &[(&str, &str)],
    buckets: Option<Vec<f64>>,
    const_labels: Option<&[&str]>,
212
) -> anyhow::Result<T> {
213
    // Validate that user-provided labels don't have duplicate keys
214
215
216
217
218
219
220
221
222
223
    validate_no_duplicate_label_keys(labels)?;
    // Validate that user-provided labels don't conflict with stored labels
    for (key, _) in registry.stored_labels() {
        if labels.iter().any(|(k, _)| *k == key) {
            return Err(anyhow::anyhow!(
                "Label key '{}' already exists in registry.",
                key
            ));
        }
    }
224
225
226
227

    let basename = registry.basename();
    let parent_hierarchy = registry.parent_hierarchy();

228
229
230
    // Build hierarchy: parent_hierarchy + [basename]
    let hierarchy = [parent_hierarchy.clone(), vec![basename.clone()]].concat();

231
    let metric_name = build_metric_name(metric_name);
232

233
    // Build updated_labels: auto-labels first, then `labels` + stored labels
234
235
236
237
238
    let mut updated_labels: Vec<(String, String)> = Vec::new();

    if USE_AUTO_LABELS {
        // Validate that user-provided labels don't conflict with auto-generated labels
        for (key, _) in labels {
239
240
            if *key == "dynamo_namespace" || *key == "dynamo_component" || *key == "dynamo_endpoint"
            {
241
242
243
244
245
246
247
                return Err(anyhow::anyhow!(
                    "Label '{}' is automatically added by auto_label feature and cannot be manually set",
                    key
                ));
            }
        }

248
        // Add auto-generated labels with sanitized values
249
250
251
252
253
254
255
256
        if hierarchy.len() > 1 {
            let namespace = &hierarchy[1];
            if !namespace.is_empty() {
                let valid_namespace = lint_prometheus_name(namespace)?;
                if !valid_namespace.is_empty() {
                    updated_labels.push(("dynamo_namespace".to_string(), valid_namespace));
                }
            }
257
258
259
260
        }
        if hierarchy.len() > 2 {
            let component = &hierarchy[2];
            if !component.is_empty() {
261
262
                let valid_component = lint_prometheus_name(component)?;
                if !valid_component.is_empty() {
263
                    updated_labels.push(("dynamo_component".to_string(), valid_component));
264
                }
265
266
267
268
269
            }
        }
        if hierarchy.len() > 3 {
            let endpoint = &hierarchy[3];
            if !endpoint.is_empty() {
270
271
                let valid_endpoint = lint_prometheus_name(endpoint)?;
                if !valid_endpoint.is_empty() {
272
                    updated_labels.push(("dynamo_endpoint".to_string(), valid_endpoint));
273
                }
274
275
276
277
278
279
280
281
282
283
            }
        }
    }

    // Add user labels
    updated_labels.extend(
        labels
            .iter()
            .map(|(k, v)| ((*k).to_string(), (*v).to_string())),
    );
284
285
286
287
288
289
290
    // Add stored labels (safe because overlaps were rejected above)
    updated_labels.extend(
        registry
            .stored_labels()
            .into_iter()
            .map(|(k, v)| (k.to_string(), v.to_string())),
    );
291
292

    // Handle different metric types
293
294
295
    let prometheus_metric = if std::any::TypeId::of::<T>()
        == std::any::TypeId::of::<prometheus::Histogram>()
    {
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
        // Special handling for Histogram with custom buckets
        // buckets parameter is valid for Histogram, const_labels is not used
        if const_labels.is_some() {
            return Err(anyhow::anyhow!(
                "const_labels parameter is not valid for Histogram"
            ));
        }
        let mut opts = prometheus::HistogramOpts::new(&metric_name, metric_desc);
        for (key, value) in &updated_labels {
            opts = opts.const_label(key.clone(), value.clone());
        }
        T::with_histogram_opts_and_buckets(opts, buckets)?
    } else if std::any::TypeId::of::<T>() == std::any::TypeId::of::<prometheus::CounterVec>() {
        // Special handling for CounterVec with label names
        // const_labels parameter is required for CounterVec
        if buckets.is_some() {
            return Err(anyhow::anyhow!(
                "buckets parameter is not valid for CounterVec"
            ));
        }
        let mut opts = prometheus::Opts::new(&metric_name, metric_desc);
        for (key, value) in &updated_labels {
            opts = opts.const_label(key.clone(), value.clone());
        }
        let label_names = const_labels
            .ok_or_else(|| anyhow::anyhow!("CounterVec requires const_labels parameter"))?;
        T::with_opts_and_label_names(opts, label_names)?
    } else if std::any::TypeId::of::<T>() == std::any::TypeId::of::<prometheus::IntGaugeVec>() {
        // Special handling for IntGaugeVec with label names
        // const_labels parameter is required for IntGaugeVec
        if buckets.is_some() {
            return Err(anyhow::anyhow!(
                "buckets parameter is not valid for IntGaugeVec"
            ));
        }
        let mut opts = prometheus::Opts::new(&metric_name, metric_desc);
        for (key, value) in &updated_labels {
            opts = opts.const_label(key.clone(), value.clone());
        }
        let label_names = const_labels
            .ok_or_else(|| anyhow::anyhow!("IntGaugeVec requires const_labels parameter"))?;
        T::with_opts_and_label_names(opts, label_names)?
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
    } else if std::any::TypeId::of::<T>() == std::any::TypeId::of::<prometheus::IntCounterVec>() {
        // Special handling for IntCounterVec with label names
        // const_labels parameter is required for IntCounterVec
        if buckets.is_some() {
            return Err(anyhow::anyhow!(
                "buckets parameter is not valid for IntCounterVec"
            ));
        }
        let mut opts = prometheus::Opts::new(&metric_name, metric_desc);
        for (key, value) in &updated_labels {
            opts = opts.const_label(key.clone(), value.clone());
        }
        let label_names = const_labels
            .ok_or_else(|| anyhow::anyhow!("IntCounterVec requires const_labels parameter"))?;
        T::with_opts_and_label_names(opts, label_names)?
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
    } else {
        // Standard handling for Counter, IntCounter, Gauge, IntGauge
        // buckets and const_labels parameters are not valid for these types
        if buckets.is_some() {
            return Err(anyhow::anyhow!(
                "buckets parameter is not valid for Counter, IntCounter, Gauge, or IntGauge"
            ));
        }
        if const_labels.is_some() {
            return Err(anyhow::anyhow!(
                "const_labels parameter is not valid for Counter, IntCounter, Gauge, or IntGauge"
            ));
        }
        let mut opts = prometheus::Opts::new(&metric_name, metric_desc);
        for (key, value) in &updated_labels {
            opts = opts.const_label(key.clone(), value.clone());
        }
        T::with_opts(opts)?
    };

    // Iterate over the DRT's registry and register this metric across all hierarchical levels.
374
    // The prefixed_hierarchy is structured as: ["", "testnamespace", "testnamespace_testcomponent", "testnamespace_testcomponent_testendpoint"]
375
376
377
378
379
380
381
382
383
    // This prefixing is essential to differentiate between the names of children and grandchildren.
    let mut prometheus_registry = registry
        .drt()
        .prometheus_registries_by_prefix
        .lock()
        .unwrap();

    // Build prefixed hierarchy and register metrics in a single loop
    // current_prefix accumulates the hierarchical path as we iterate through hierarchy
384
    // For example, if hierarchy = ["", "testnamespace", "testcomponent"], then:
385
    // - Iteration 1: current_prefix = "" (empty string from DRT)
386
387
    // - Iteration 2: current_prefix = "testnamespace"
    // - Iteration 3: current_prefix = "testnamespace_testcomponent"
388
389
390
391
392
393
394
395
    let mut current_prefix = String::new();
    for name in &hierarchy {
        if !current_prefix.is_empty() && !name.is_empty() {
            current_prefix.push('_');
        }
        current_prefix.push_str(name);

        // Register metric at this hierarchical level
396
        let collector: Box<dyn prometheus::core::Collector> = Box::new(prometheus_metric.clone());
397
398
399
400
401
402
        let _ = prometheus_registry
            .entry(current_prefix.clone())
            .or_default()
            .register(collector);
    }

403
    Ok(prometheus_metric)
404
405
406
407
408
409
410
411
412
}

/// This trait should be implemented by all metric registries, including Prometheus, Envy, OpenTelemetry, and others.
/// It offers a unified interface for creating and managing metrics, organizing sub-registries, and
/// generating output in Prometheus text format.
pub trait MetricsRegistry: Send + Sync + crate::traits::DistributedRuntimeProvider {
    // Get the name of this registry (without any prefix)
    fn basename(&self) -> String;

413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
    /// Get any stored labels for this registry
    fn stored_labels(&self) -> Vec<(&str, &str)> {
        Vec::new()
    }

    /// Get mutable access to the labels storage - implementors must provide this
    fn labels_mut(&mut self) -> &mut Vec<(String, String)>;

    /// Add labels to this registry and return a new instance with the labels.
    ///   This allows for method chaining like: runtime.namespace(...).add_labels(...)?
    /// Fails if:
    /// - Provided `labels` contains duplicate keys, or
    /// - Any provided key already exists in the registry's stored labels.
    fn add_labels(mut self, labels: &[(&str, &str)]) -> anyhow::Result<Self>
    where
        Self: Sized,
    {
        validate_no_duplicate_label_keys(labels)?;

        // 2) Validate no overlap with existing stored labels
        let existing: std::collections::HashSet<&str> =
            self.stored_labels().into_iter().map(|(k, _)| k).collect();
        if let Some(conflict) = labels
            .iter()
            .map(|(k, _)| *k)
            .find(|k| existing.contains(k))
        {
            return Err(anyhow::anyhow!(
                "Label key '{}' already exists in registry; refusing to overwrite",
                conflict
            ));
        }

        // 3) Safe to append
        let labels_storage = self.labels_mut();
        for (key, value) in labels {
            labels_storage.push((key.to_string(), value.to_string()));
        }
        Ok(self)
    }

454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
    /// Retrieve the complete hierarchy and basename for this registry. Currently, the prefix for drt is an empty string,
    /// so we must account for the leading underscore. The existing code remains unchanged to accommodate any future
    /// scenarios where drt's prefix might be assigned a value.
    fn prefix(&self) -> String {
        [self.parent_hierarchy(), vec![self.basename()]]
            .concat()
            .join("_")
            .trim_start_matches('_')
            .to_string()
    }

    // Get the parent hierarchy for this registry (just the base names, NOT the prefix)
    fn parent_hierarchy(&self) -> Vec<String>;

    // TODO: Add support for additional Prometheus metric types:
    // - Counter: ✅ IMPLEMENTED - create_counter()
    // - CounterVec: ✅ IMPLEMENTED - create_countervec()
    // - Gauge: ✅ IMPLEMENTED - create_gauge()
472
    // - GaugeHistogram: create_gauge_histogram() - for gauge histograms
473
474
475
    // - Histogram: ✅ IMPLEMENTED - create_histogram()
    // - HistogramVec with custom buckets: create_histogram_with_buckets()
    // - Info: create_info() - for info metrics with labels
476
477
478
479
    // - IntCounter: ✅ IMPLEMENTED - create_intcounter()
    // - IntCounterVec: ✅ IMPLEMENTED - create_intcountervec()
    // - IntGauge: ✅ IMPLEMENTED - create_intgauge()
    // - IntGaugeVec: ✅ IMPLEMENTED - create_intgaugevec()
480
    // - Stateset: create_stateset() - for state-based metrics
481
482
483
    // - Summary: create_summary() - for quantiles and sum/count metrics
    // - SummaryVec: create_summary_vec() - for labeled summaries
    // - Untyped: create_untyped() - for untyped metrics
484
485
486
487
488
489
490

    /// Create a Counter metric
    fn create_counter(
        &self,
        name: &str,
        description: &str,
        labels: &[(&str, &str)],
491
    ) -> anyhow::Result<prometheus::Counter> {
492
493
494
        create_metric(self, name, description, labels, None, None)
    }

495
496
    /// Create a CounterVec metric with label names (for dynamic labels)
    fn create_countervec(
497
498
499
        &self,
        name: &str,
        description: &str,
500
501
        const_labels: &[&str],
        const_label_values: &[(&str, &str)],
502
    ) -> anyhow::Result<prometheus::CounterVec> {
503
504
505
506
507
508
509
510
        create_metric(
            self,
            name,
            description,
            const_label_values,
            None,
            Some(const_labels),
        )
511
512
    }

513
514
    /// Create a Gauge metric
    fn create_gauge(
515
516
517
518
        &self,
        name: &str,
        description: &str,
        labels: &[(&str, &str)],
519
    ) -> anyhow::Result<prometheus::Gauge> {
520
521
522
523
524
525
526
527
528
529
        create_metric(self, name, description, labels, None, None)
    }

    /// Create a Histogram metric with custom buckets
    fn create_histogram(
        &self,
        name: &str,
        description: &str,
        labels: &[(&str, &str)],
        buckets: Option<Vec<f64>>,
530
    ) -> anyhow::Result<prometheus::Histogram> {
531
532
533
        create_metric(self, name, description, labels, buckets, None)
    }

534
535
536
537
538
539
    /// Create an IntCounter metric
    fn create_intcounter(
        &self,
        name: &str,
        description: &str,
        labels: &[(&str, &str)],
540
    ) -> anyhow::Result<prometheus::IntCounter> {
541
542
543
544
545
        create_metric(self, name, description, labels, None, None)
    }

    /// Create an IntCounterVec metric with label names (for dynamic labels)
    fn create_intcountervec(
546
547
548
549
550
        &self,
        name: &str,
        description: &str,
        const_labels: &[&str],
        const_label_values: &[(&str, &str)],
551
    ) -> anyhow::Result<prometheus::IntCounterVec> {
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
        create_metric(
            self,
            name,
            description,
            const_label_values,
            None,
            Some(const_labels),
        )
    }

    /// Create an IntGauge metric
    fn create_intgauge(
        &self,
        name: &str,
        description: &str,
        labels: &[(&str, &str)],
568
    ) -> anyhow::Result<prometheus::IntGauge> {
569
570
571
572
573
574
575
576
577
578
        create_metric(self, name, description, labels, None, None)
    }

    /// Create an IntGaugeVec metric with label names (for dynamic labels)
    fn create_intgaugevec(
        &self,
        name: &str,
        description: &str,
        const_labels: &[&str],
        const_label_values: &[(&str, &str)],
579
    ) -> anyhow::Result<prometheus::IntGaugeVec> {
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
        create_metric(
            self,
            name,
            description,
            const_label_values,
            None,
            Some(const_labels),
        )
    }

    /// Get metrics in Prometheus text format
    fn prometheus_metrics_fmt(&self) -> anyhow::Result<String> {
        let prometheus_registry = {
            let mut registry = self.drt().prometheus_registries_by_prefix.lock().unwrap();
            registry.entry(self.prefix()).or_default().clone()
        };
        let metric_families = prometheus_registry.gather();
        let encoder = prometheus::TextEncoder::new();
        let mut buffer = Vec::new();
        encoder.encode(&metric_families, &mut buffer)?;
        Ok(String::from_utf8(buffer)?)
    }
}

#[cfg(test)]
/// Helper function to create a DRT instance for testing
/// Uses the test-friendly constructor without discovery
pub fn create_test_drt() -> crate::DistributedRuntime {
    let rt = crate::Runtime::single_threaded().unwrap();
    tokio::runtime::Runtime::new().unwrap().block_on(async {
        crate::DistributedRuntime::from_settings_without_discovery(rt.clone())
            .await
            .unwrap()
    })
}

616
617
618
619
620
621
#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn test_build_metric_name_with_prefix() {
622
623
624
        // Test that build_metric_name correctly prepends the dynamo_component prefix
        let result = build_metric_name("requests");
        assert_eq!(result, "dynamo_component_requests");
625

626
627
        let result = build_metric_name("counter");
        assert_eq!(result, "dynamo_component_counter");
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
    }

    #[test]
    fn test_lint_prometheus_name() {
        // Test that valid components remain unchanged
        assert_eq!(
            lint_prometheus_name("testnamespace").unwrap(),
            "testnamespace"
        );
        assert_eq!(
            lint_prometheus_name("test_namespace").unwrap(),
            "test_namespace"
        );
        assert_eq!(lint_prometheus_name("test123").unwrap(), "test123");
        assert_eq!(
            lint_prometheus_name("test:namespace").unwrap(),
            "test:namespace"
        );
        assert_eq!(
            lint_prometheus_name("_testnamespace").unwrap(),
            "_testnamespace"
        );
        assert_eq!(
            lint_prometheus_name("testnamespace_123").unwrap(),
            "testnamespace_123"
        );

        // Test that invalid characters are stripped
        assert_eq!(lint_prometheus_name("").unwrap(), ""); // Empty
        assert_eq!(
            lint_prometheus_name("test namespace").unwrap(),
            "testnamespace"
        ); // Space removed
        assert_eq!(
            lint_prometheus_name("test.namespace").unwrap(),
            "testnamespace"
        ); // Dot removed
        assert_eq!(
            lint_prometheus_name("test@namespace").unwrap(),
            "testnamespace"
        ); // @ removed
        assert_eq!(
            lint_prometheus_name("test#namespace").unwrap(),
            "testnamespace"
        ); // # removed
        assert_eq!(
            lint_prometheus_name("test$namespace").unwrap(),
            "testnamespace"
        ); // $ removed
        assert_eq!(
            lint_prometheus_name("test!@#$%^&*()namespace").unwrap(),
            "testnamespace"
        ); // Multiple special chars removed
        assert_eq!(
            lint_prometheus_name("testnamespace_123!").unwrap(),
            "testnamespace_123"
        ); // Trailing special char removed

        // Test that hyphens are stripped (not allowed in Prometheus names)
        assert_eq!(
            lint_prometheus_name("test-namespace").unwrap(),
            "testnamespace"
        ); // Hyphen removed
        assert_eq!(
            lint_prometheus_name("test-namespace_123").unwrap(),
            "testnamespace_123"
        ); // Hyphen removed

        // Test validation errors for invalid patterns
        assert!(lint_prometheus_name("123test").is_err()); // Starts with digit
        assert!(lint_prometheus_name("").is_ok()); // Empty is allowed
    }
}

702
703
704
705
706
#[cfg(feature = "integration")]
#[cfg(test)]
mod test_prefixes {
    use super::create_test_drt;
    use super::*;
707
    use prometheus::core::Collector;
708
709
710
711
712
713
714
715

    #[test]
    fn test_hierarchical_prefixes_and_parent_hierarchies() {
        println!("=== Testing Names, Prefixes, and Parent Hierarchies ===");

        // Create a distributed runtime for testing
        let drt = create_test_drt();

716
717
        // Use a simple constant namespace name
        let namespace_name = "testnamespace";
718
719

        // Create namespace
720
        let namespace = drt.namespace(namespace_name).unwrap();
721
722

        // Create component
723
        let component = namespace.component("testcomponent").unwrap();
724
725

        // Create endpoint
726
        let endpoint = component.endpoint("testendpoint");
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771

        // Test DistributedRuntime hierarchy
        println!("\n=== DistributedRuntime ===");
        println!("basename: '{}'", drt.basename());
        println!("parent_hierarchy: {:?}", drt.parent_hierarchy());
        println!("prefix: '{}'", drt.prefix());

        assert_eq!(drt.basename(), "", "DRT basename should be empty");
        assert_eq!(
            drt.parent_hierarchy(),
            Vec::<String>::new(),
            "DRT parent hierarchy should be empty"
        );
        assert_eq!(drt.prefix(), "", "DRT prefix should be empty");

        // Test Namespace hierarchy
        println!("\n=== Namespace ===");
        println!("basename: '{}'", namespace.basename());
        println!("parent_hierarchy: {:?}", namespace.parent_hierarchy());
        println!("prefix: '{}'", namespace.prefix());

        assert_eq!(
            namespace.basename(),
            namespace_name,
            "Namespace basename should match the generated name"
        );
        assert_eq!(
            namespace.parent_hierarchy(),
            vec![""],
            "Namespace parent hierarchy should be [\"\"]"
        );
        assert_eq!(
            namespace.prefix(),
            namespace_name,
            "Namespace prefix should match the generated name, because drt's prefix is empty"
        );

        // Test Component hierarchy
        println!("\n=== Component ===");
        println!("basename: '{}'", component.basename());
        println!("parent_hierarchy: {:?}", component.parent_hierarchy());
        println!("prefix: '{}'", component.prefix());

        assert_eq!(
            component.basename(),
772
773
            "testcomponent",
            "Component basename should be 'testcomponent'"
774
775
776
777
778
779
780
781
        );
        assert_eq!(
            component.parent_hierarchy(),
            vec!["", &namespace_name],
            "Component parent hierarchy should contain the generated namespace name"
        );
        assert_eq!(
            component.prefix(),
782
783
            format!("{}_testcomponent", namespace),
            "Component prefix should be 'namespace_testcomponent'"
784
785
786
787
788
789
790
791
792
793
        );

        // Test Endpoint hierarchy
        println!("\n=== Endpoint ===");
        println!("basename: '{}'", endpoint.basename());
        println!("parent_hierarchy: {:?}", endpoint.parent_hierarchy());
        println!("prefix: '{}'", endpoint.prefix());

        assert_eq!(
            endpoint.basename(),
794
795
            "testendpoint",
            "Endpoint basename should be 'testendpoint'"
796
797
798
        );
        assert_eq!(
            endpoint.parent_hierarchy(),
799
            vec!["", &namespace_name, "testcomponent"],
800
801
802
803
            "Endpoint parent hierarchy should contain the generated namespace name"
        );
        assert_eq!(
            endpoint.prefix(),
804
805
            format!("{}_testcomponent_testendpoint", namespace),
            "Endpoint prefix should be 'namespace_testcomponent_testendpoint'"
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
        );

        // Test hierarchy relationships
        println!("\n=== Hierarchy Relationships ===");
        assert!(
            namespace.parent_hierarchy().contains(&drt.basename()),
            "Namespace should have DRT prefix in parent hierarchy"
        );
        assert!(
            component.parent_hierarchy().contains(&namespace.basename()),
            "Component should have Namespace prefix in parent hierarchy"
        );
        assert!(
            endpoint.parent_hierarchy().contains(&component.basename()),
            "Endpoint should have Component prefix in parent hierarchy"
        );
        println!("✓ All parent-child relationships verified");

        // Test hierarchy depth
        println!("\n=== Hierarchy Depth ===");
        assert_eq!(
            drt.parent_hierarchy().len(),
            0,
            "DRT should have 0 parent hierarchy levels"
        );
        assert_eq!(
            namespace.parent_hierarchy().len(),
            1,
            "Namespace should have 1 parent hierarchy level"
        );
        assert_eq!(
            component.parent_hierarchy().len(),
            2,
            "Component should have 2 parent hierarchy levels"
        );
        assert_eq!(
            endpoint.parent_hierarchy().len(),
            3,
            "Endpoint should have 3 parent hierarchy levels"
        );
        println!("✓ All hierarchy depths verified");

        // Summary
        println!("\n=== Summary ===");
        println!("DRT prefix: '{}'", drt.prefix());
        println!("Namespace prefix: '{}'", namespace.prefix());
        println!("Component prefix: '{}'", component.prefix());
        println!("Endpoint prefix: '{}'", endpoint.prefix());
        println!("All hierarchy assertions passed!");
855
856
857
858
859

        // Test invalid namespace behavior
        println!("\n=== Testing Invalid Namespace Behavior ===");

        // Create a namespace with invalid name (contains hyphen)
860
        let invalid_namespace = drt.namespace("@@123").unwrap();
861
862
863
864
865
866
867
868
869
870
871
872

        // Debug: Let's see what the hierarchy looks like
        println!(
            "Invalid namespace basename: '{}'",
            invalid_namespace.basename()
        );
        println!(
            "Invalid namespace parent_hierarchy: {:?}",
            invalid_namespace.parent_hierarchy()
        );
        println!("Invalid namespace prefix: '{}'", invalid_namespace.prefix());

873
        // Try to create a metric - this should succeed because the namespace name will be sanitized
874
        let result = invalid_namespace.create_counter("test_counter", "A test counter", &[]);
875
        println!("Result with invalid namespace '@@123':");
876
877
        println!("{:?}", result);

878
        // The result should be an error because '@@123' gets sanitized to '123' which is invalid
879
880
        assert!(
            result.is_err(),
881
            "Creating metric with namespace '@@123' should fail because it gets sanitized to '123' which is invalid"
882
883
        );

884
885
886
887
888
889
890
891
892
893
        // Verify the error message indicates the sanitized name is still invalid
        if let Err(e) = &result {
            let error_msg = e.to_string();
            assert!(
                error_msg.contains("123"),
                "Error message should mention the sanitized name '123', got: {}",
                error_msg
            );
        }

894
895
896
897
898
899
900
901
902
903
904
        // For comparison, show a valid namespace works
        let valid_namespace = drt.namespace("test_namespace").unwrap();
        let valid_result = valid_namespace.create_counter("test_counter", "A test counter", &[]);
        println!("Result with valid namespace 'test_namespace':");
        println!("{:?}", valid_result);
        assert!(
            valid_result.is_ok(),
            "Creating metric with valid namespace should succeed"
        );

        println!("✓ Invalid namespace behavior verified!");
905
906
907
908
909
910
911
912
913
914
915
    }
}

#[cfg(feature = "integration")]
#[cfg(test)]
mod test_simple_metricsregistry_trait {
    use super::create_test_drt;
    use super::*;
    use prometheus::Counter;
    use std::sync::Arc;

916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
    #[test]
    fn test_component_prometheus_output_contains_custom_label() {
        // Arrange: DRT → namespace → component with a custom label
        let drt = create_test_drt();
        let namespace = drt.namespace("testnamespace").unwrap();
        let component = namespace
            .component("testcomponent")
            .unwrap()
            .add_labels(&[("service", "api")])
            .unwrap();

        // Act: create a simple gauge and render Prometheus text
        let gauge = component
            .create_gauge("with_label", "Gauge with custom label", &[])
            .unwrap();
        gauge.set(1.0);

        let output = component.prometheus_metrics_fmt().unwrap();

        // Assert: custom label is present (don’t rely on label ordering)
        assert!(
            output.contains("dynamo_component_with_label{") && output.contains(r#"service="api""#),
            "Expected custom label service=\"api\" in Prometheus output:\n{}",
            output
        );
    }

943
944
945
946
947
    #[test]
    fn test_factory_methods_via_registry_trait() {
        // Setup real DRT and registry using the test-friendly constructor
        let drt = create_test_drt();

948
949
        // Use a simple constant namespace name
        let namespace_name = "testnamespace";
950

951
952
953
        let namespace = drt.namespace(namespace_name).unwrap();
        let component = namespace.component("testcomponent").unwrap();
        let endpoint = component.endpoint("testendpoint");
954
955
956

        // Test Counter creation
        let counter = endpoint
957
            .create_counter("testcounter", "A test counter", &[])
958
959
960
961
962
963
964
965
966
967
            .unwrap();
        counter.inc_by(123.456789);
        let epsilon = 0.01;
        assert!((counter.get() - 123.456789).abs() < epsilon);

        let endpoint_output = endpoint.prometheus_metrics_fmt().unwrap();
        println!("Endpoint output:");
        println!("{}", endpoint_output);

        let expected_endpoint_output = format!(
968
969
970
            r#"# HELP dynamo_component_testcounter A test counter
# TYPE dynamo_component_testcounter counter
dynamo_component_testcounter{{dynamo_component="testcomponent",dynamo_endpoint="testendpoint",dynamo_namespace="testnamespace"}} 123.456789
971
"#
972
973
974
975
976
977
978
979
980
981
982
983
984
        );

        assert_eq!(
            endpoint_output, expected_endpoint_output,
            "\n=== ENDPOINT COMPARISON FAILED ===\n\
             Expected:\n{}\n\
             Actual:\n{}\n\
             ==============================",
            expected_endpoint_output, endpoint_output
        );

        // Test Gauge creation
        let gauge = component
985
            .create_gauge("testgauge", "A test gauge", &[])
986
987
988
989
990
991
992
993
994
995
            .unwrap();
        gauge.set(50000.0);
        assert_eq!(gauge.get(), 50000.0);

        // Test Prometheus format output for Component (gauge + histogram)
        let component_output = component.prometheus_metrics_fmt().unwrap();
        println!("Component output:");
        println!("{}", component_output);

        let expected_component_output = format!(
996
997
998
999
1000
1001
            r#"# HELP dynamo_component_testcounter A test counter
# TYPE dynamo_component_testcounter counter
dynamo_component_testcounter{{dynamo_component="testcomponent",dynamo_endpoint="testendpoint",dynamo_namespace="testnamespace"}} 123.456789
# HELP dynamo_component_testgauge A test gauge
# TYPE dynamo_component_testgauge gauge
dynamo_component_testgauge{{dynamo_component="testcomponent",dynamo_namespace="testnamespace"}} 50000
1002
"#
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
        );

        assert_eq!(
            component_output, expected_component_output,
            "\n=== COMPONENT COMPARISON FAILED ===\n\
             Expected:\n{}\n\
             Actual:\n{}\n\
             ==============================",
            expected_component_output, component_output
        );

        let intcounter = namespace
1015
            .create_intcounter("testintcounter", "A test int counter", &[])
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
            .unwrap();
        intcounter.inc_by(12345);
        assert_eq!(intcounter.get(), 12345);

        // Test Prometheus format output for Namespace (int_counter + gauge + histogram)
        let namespace_output = namespace.prometheus_metrics_fmt().unwrap();
        println!("Namespace output:");
        println!("{}", namespace_output);

        let expected_namespace_output = format!(
1026
1027
1028
1029
1030
1031
1032
1033
1034
            r#"# HELP dynamo_component_testcounter A test counter
# TYPE dynamo_component_testcounter counter
dynamo_component_testcounter{{dynamo_component="testcomponent",dynamo_endpoint="testendpoint",dynamo_namespace="testnamespace"}} 123.456789
# HELP dynamo_component_testgauge A test gauge
# TYPE dynamo_component_testgauge gauge
dynamo_component_testgauge{{dynamo_component="testcomponent",dynamo_namespace="testnamespace"}} 50000
# HELP dynamo_component_testintcounter A test int counter
# TYPE dynamo_component_testintcounter counter
dynamo_component_testintcounter{{dynamo_namespace="testnamespace"}} 12345
1035
"#
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
        );

        assert_eq!(
            namespace_output, expected_namespace_output,
            "\n=== NAMESPACE COMPARISON FAILED ===\n\
             Expected:\n{}\n\
             Actual:\n{}\n\
             ==============================",
            expected_namespace_output, namespace_output
        );

        // Create a histogram with specified buckets. The Prometheus format output will
        // lack labels since the DistributedRuntime is unnamed.
        let histogram = drt
            .create_histogram(
1051
                "testhistogram",
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
                "A test histogram",
                &[],
                Some(vec![1.0, 2.5, 5.0, 10.0]),
            )
            .unwrap();
        histogram.observe(1.5);
        histogram.observe(2.5);
        histogram.observe(3.5);

        // Test CounterVec creation
        let countervec = drt
            .create_countervec(
1064
                "testcountervec",
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
                "A test counter vector",
                &["method", "status"],
                &[("service", "api")],
            )
            .unwrap();
        countervec.with_label_values(&["GET", "200"]).inc_by(10.0);
        countervec.with_label_values(&["POST", "201"]).inc_by(5.0);

        // Test IntGauge creation
        let intgauge = drt
1075
            .create_intgauge("testintgauge", "A test int gauge", &[])
1076
1077
1078
1079
1080
1081
1082
            .unwrap();
        intgauge.set(42);
        assert_eq!(intgauge.get(), 42);

        // Test IntGaugeVec creation
        let intgaugevec = drt
            .create_intgaugevec(
1083
                "testintgaugevec",
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
                "A test int gauge vector",
                &["instance", "status"],
                &[("service", "api")],
            )
            .unwrap();
        intgaugevec
            .with_label_values(&["server1", "active"])
            .set(10);
        intgaugevec
            .with_label_values(&["server2", "inactive"])
            .set(0);

        // Test Prometheus format output for DRT (which should contain everything)
        let drt_output = drt.prometheus_metrics_fmt().unwrap();
        println!("DRT output:");
        println!("{}", drt_output);

        let expected_drt_output = format!(
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
            r#"# HELP dynamo_component_testcounter A test counter
# TYPE dynamo_component_testcounter counter
dynamo_component_testcounter{{dynamo_component="testcomponent",dynamo_endpoint="testendpoint",dynamo_namespace="testnamespace"}} 123.456789
# HELP dynamo_component_testcountervec A test counter vector
# TYPE dynamo_component_testcountervec counter
dynamo_component_testcountervec{{method="GET",service="api",status="200"}} 10
dynamo_component_testcountervec{{method="POST",service="api",status="201"}} 5
# HELP dynamo_component_testgauge A test gauge
# TYPE dynamo_component_testgauge gauge
dynamo_component_testgauge{{dynamo_component="testcomponent",dynamo_namespace="testnamespace"}} 50000
# HELP dynamo_component_testhistogram A test histogram
# TYPE dynamo_component_testhistogram histogram
dynamo_component_testhistogram_bucket{{le="1"}} 0
dynamo_component_testhistogram_bucket{{le="2.5"}} 2
dynamo_component_testhistogram_bucket{{le="5"}} 3
dynamo_component_testhistogram_bucket{{le="10"}} 3
dynamo_component_testhistogram_bucket{{le="+Inf"}} 3
dynamo_component_testhistogram_sum 7.5
dynamo_component_testhistogram_count 3
# HELP dynamo_component_testintcounter A test int counter
# TYPE dynamo_component_testintcounter counter
dynamo_component_testintcounter{{dynamo_namespace="testnamespace"}} 12345
# HELP dynamo_component_testintgauge A test int gauge
# TYPE dynamo_component_testintgauge gauge
dynamo_component_testintgauge 42
# HELP dynamo_component_testintgaugevec A test int gauge vector
# TYPE dynamo_component_testintgaugevec gauge
dynamo_component_testintgaugevec{{instance="server1",service="api",status="active"}} 10
dynamo_component_testintgaugevec{{instance="server2",service="api",status="inactive"}} 0
1131
"#
1132
1133
1134
        );

        assert_eq!(
1135
            filtered_drt_output, expected_drt_output,
1136
1137
1138
1139
            "\n=== DRT COMPARISON FAILED ===\n\
             Expected:\n{}\n\
             Actual:\n{}\n\
             ==============================",
1140
            expected_drt_output, filtered_drt_output
1141
1142
1143
1144
1145
        );

        println!("✓ All Prometheus format outputs verified successfully!");
    }
}