metrics.py 24.6 KB
Newer Older
1
2
from typing import TYPE_CHECKING
from typing import Counter as CollectionsCounter
3
from typing import Dict, List, Optional, Type, Union, cast
4
5

import numpy as np
6
import prometheus_client
7

8
9
from vllm.engine.metrics_types import (StatLoggerBase, Stats,
                                       SupportsMetricsInfo)
10
from vllm.executor.ray_utils import ray
11
from vllm.logger import init_logger
12

13
14
15
16
17
if ray is not None:
    from ray.util import metrics as ray_metrics
else:
    ray_metrics = None

18
19
20
if TYPE_CHECKING:
    from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics

21
22
logger = init_logger(__name__)

23
prometheus_client.disable_created_metrics()
24
25
26
27

# The begin-* and end* here are used by the documentation generator
# to extract the metrics definitions.

28

29
# begin-metrics-definitions
30
class Metrics:
31
32
33
34
35
36
    """
    vLLM uses a multiprocessing-based frontend for the OpenAI server.
    This means that we need to run prometheus_client in multiprocessing mode
    See https://prometheus.github.io/client_python/multiprocess/ for more
    details on limitations.
    """
37

38
    labelname_finish_reason = "finished_reason"
39
40
41
    labelname_waiting_lora_adapters = "waiting_lora_adapters"
    labelname_running_lora_adapters = "running_lora_adapters"
    labelname_max_lora = "max_lora"
42
43
44
    _gauge_cls = prometheus_client.Gauge
    _counter_cls = prometheus_client.Counter
    _histogram_cls = prometheus_client.Histogram
45

46
    def __init__(self, labelnames: List[str], max_model_len: int):
47
        # Unregister any existing vLLM collectors (for CI/CD)
48
        self._unregister_vllm_metrics()
49
50

        # System stats
51
        #   Scheduler State
52
        self.gauge_scheduler_running = self._gauge_cls(
53
54
            name="vllm:num_requests_running",
            documentation="Number of requests currently running on GPU.",
55
56
            labelnames=labelnames,
            multiprocess_mode="sum")
57
        self.gauge_scheduler_waiting = self._gauge_cls(
58
59
            name="vllm:num_requests_waiting",
            documentation="Number of requests waiting to be processed.",
60
61
            labelnames=labelnames,
            multiprocess_mode="sum")
62
63
64
65
66
67
68
69
70
71
        self.gauge_lora_info = self._gauge_cls(
            name="vllm:lora_requests_info",
            documentation="Running stats on lora requests.",
            labelnames=[
                self.labelname_running_lora_adapters,
                self.labelname_max_lora,
                self.labelname_waiting_lora_adapters,
            ],
            multiprocess_mode="livemostrecent",
        )
72
        self.gauge_scheduler_swapped = self._gauge_cls(
73
74
            name="vllm:num_requests_swapped",
            documentation="Number of requests swapped to CPU.",
75
76
            labelnames=labelnames,
            multiprocess_mode="sum")
77
        #   KV Cache Usage in %
78
        self.gauge_gpu_cache_usage = self._gauge_cls(
79
80
            name="vllm:gpu_cache_usage_perc",
            documentation="GPU KV-cache usage. 1 means 100 percent usage.",
81
82
            labelnames=labelnames,
            multiprocess_mode="sum")
83
        self.gauge_cpu_cache_usage = self._gauge_cls(
84
85
            name="vllm:cpu_cache_usage_perc",
            documentation="CPU KV-cache usage. 1 means 100 percent usage.",
86
87
            labelnames=labelnames,
            multiprocess_mode="sum")
88
89
90
91
92
93
94
95
96
97
98
        #   Prefix caching block hit rate
        self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls(
            name="vllm:cpu_prefix_cache_hit_rate",
            documentation="CPU prefix cache block hit rate.",
            labelnames=labelnames,
            multiprocess_mode="sum")
        self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls(
            name="vllm:gpu_prefix_cache_hit_rate",
            documentation="GPU prefix cache block hit rate.",
            labelnames=labelnames,
            multiprocess_mode="sum")
99

100
        # Iteration stats
101
        self.counter_num_preemption = self._counter_cls(
102
103
104
            name="vllm:num_preemptions_total",
            documentation="Cumulative number of preemption from the engine.",
            labelnames=labelnames)
105
        self.counter_prompt_tokens = self._counter_cls(
106
107
108
            name="vllm:prompt_tokens_total",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames)
109
        self.counter_generation_tokens = self._counter_cls(
110
111
112
            name="vllm:generation_tokens_total",
            documentation="Number of generation tokens processed.",
            labelnames=labelnames)
113
        self.histogram_time_to_first_token = self._histogram_cls(
114
115
116
117
118
119
120
            name="vllm:time_to_first_token_seconds",
            documentation="Histogram of time to first token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
                0.75, 1.0, 2.5, 5.0, 7.5, 10.0
            ])
121
        self.histogram_time_per_output_token = self._histogram_cls(
122
123
124
125
126
127
128
            name="vllm:time_per_output_token_seconds",
            documentation="Histogram of time per output token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
                1.0, 2.5
            ])
129
130
131

        # Request stats
        #   Latency
132
        self.histogram_e2e_time_request = self._histogram_cls(
133
134
135
136
            name="vllm:e2e_request_latency_seconds",
            documentation="Histogram of end to end request latency in seconds.",
            labelnames=labelnames,
            buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
137
        #   Metadata
138
        self.histogram_num_prompt_tokens_request = self._histogram_cls(
139
140
141
142
143
            name="vllm:request_prompt_tokens",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len),
        )
144
        self.histogram_num_generation_tokens_request = \
145
            self._histogram_cls(
146
147
148
149
150
                name="vllm:request_generation_tokens",
                documentation="Number of generation tokens processed.",
                labelnames=labelnames,
                buckets=build_1_2_5_buckets(max_model_len),
            )
151
        self.histogram_n_request = self._histogram_cls(
152
153
154
155
156
            name="vllm:request_params_n",
            documentation="Histogram of the n request parameter.",
            labelnames=labelnames,
            buckets=[1, 2, 5, 10, 20],
        )
157
        self.counter_request_success = self._counter_cls(
158
            name="vllm:request_success_total",
159
160
            documentation="Count of successfully processed requests.",
            labelnames=labelnames + [Metrics.labelname_finish_reason])
161

162
        # Speculatie decoding stats
163
        self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
164
165
            name="vllm:spec_decode_draft_acceptance_rate",
            documentation="Speulative token acceptance rate.",
166
167
            labelnames=labelnames,
            multiprocess_mode="sum")
168
        self.gauge_spec_decode_efficiency = self._gauge_cls(
169
170
            name="vllm:spec_decode_efficiency",
            documentation="Speculative decoding system efficiency.",
171
172
            labelnames=labelnames,
            multiprocess_mode="sum")
173
174
175
176
177
        self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_accepted_tokens_total",
            documentation="Number of accepted tokens.",
            labelnames=labelnames))
        self.counter_spec_decode_num_draft_tokens = self._counter_cls(
178
179
180
            name="vllm:spec_decode_num_draft_tokens_total",
            documentation="Number of draft tokens.",
            labelnames=labelnames)
181
182
183
184
        self.counter_spec_decode_num_emitted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_emitted_tokens_total",
            documentation="Number of emitted tokens.",
            labelnames=labelnames))
185

186
        # Deprecated in favor of vllm:prompt_tokens_total
187
        self.gauge_avg_prompt_throughput = self._gauge_cls(
188
189
190
            name="vllm:avg_prompt_throughput_toks_per_s",
            documentation="Average prefill throughput in tokens/s.",
            labelnames=labelnames,
191
            multiprocess_mode="sum",
192
        )
193
        # Deprecated in favor of vllm:generation_tokens_total
194
        self.gauge_avg_generation_throughput = self._gauge_cls(
195
196
197
            name="vllm:avg_generation_throughput_toks_per_s",
            documentation="Average generation throughput in tokens/s.",
            labelnames=labelnames,
198
            multiprocess_mode="sum",
199
200
        )

201
202

# end-metrics-definitions
203

204
    def _unregister_vllm_metrics(self) -> None:
205
        for collector in list(prometheus_client.REGISTRY._collector_to_names):
206
            if hasattr(collector, "_name") and "vllm" in collector._name:
207
208
209
210
211
212
213
214
215
216
                prometheus_client.REGISTRY.unregister(collector)


class _RayGaugeWrapper:
    """Wraps around ray.util.metrics.Gauge to provide same API as
    prometheus_client.Gauge"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
217
218
219
                 labelnames: Optional[List[str]] = None,
                 multiprocess_mode: str = ""):
        del multiprocess_mode
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._gauge = ray_metrics.Gauge(name=name,
                                        description=documentation,
                                        tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._gauge.set_default_tags(labels)
        return self

    def set(self, value: Union[int, float]):
        return self._gauge.set(value)


class _RayCounterWrapper:
    """Wraps around ray.util.metrics.Counter to provide same API as
    prometheus_client.Counter"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._counter = ray_metrics.Counter(name=name,
                                            description=documentation,
                                            tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._counter.set_default_tags(labels)
        return self

    def inc(self, value: Union[int, float] = 1.0):
        if value == 0:
            return
        return self._counter.inc(value)


class _RayHistogramWrapper:
    """Wraps around ray.util.metrics.Histogram to provide same API as
    prometheus_client.Histogram"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None,
                 buckets: Optional[List[float]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
266
        boundaries = buckets if buckets else []
267
268
269
        self._histogram = ray_metrics.Histogram(name=name,
                                                description=documentation,
                                                tag_keys=labelnames_tuple,
270
                                                boundaries=boundaries)
271
272
273
274
275
276
277

    def labels(self, **labels):
        self._histogram.set_default_tags(labels)
        return self

    def observe(self, value: Union[int, float]):
        return self._histogram.observe(value)
278
279
280
281
282
283
284


class RayMetrics(Metrics):
    """
    RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
    Provides the same metrics as Metrics but uses Ray's util.metrics library.
    """
285
286
287
288
289
290
    _gauge_cls: Type[prometheus_client.Gauge] = cast(
        Type[prometheus_client.Gauge], _RayGaugeWrapper)
    _counter_cls: Type[prometheus_client.Counter] = cast(
        Type[prometheus_client.Counter], _RayCounterWrapper)
    _histogram_cls: Type[prometheus_client.Histogram] = cast(
        Type[prometheus_client.Histogram], _RayHistogramWrapper)
291
292
293
294
295
296
297
298
299
300

    def __init__(self, labelnames: List[str], max_model_len: int):
        if ray_metrics is None:
            raise ImportError("RayMetrics requires Ray to be installed.")
        super().__init__(labelnames, max_model_len)

    def _unregister_vllm_metrics(self) -> None:
        # No-op on purpose
        pass

301

302
def build_1_2_5_buckets(max_value: int) -> List[int]:
303
304
305
306
307
308
309
310
311
312
    """
    Builds a list of buckets with increasing powers of 10 multiplied by 
    mantissa values (1, 2, 5) until the value exceeds the specified maximum.

    Example:
    >>> build_1_2_5_buckets(100)
    [1, 2, 5, 10, 20, 50, 100]
    """
    mantissa_lst = [1, 2, 5]
    exponent = 0
313
    buckets: List[int] = []
314
315
316
317
318
319
320
321
322
323
    while True:
        for m in mantissa_lst:
            value = m * 10**exponent
            if value <= max_value:
                buckets.append(value)
            else:
                return buckets
        exponent += 1


324
325
326
327
328
329
330
331
332
def local_interval_elapsed(now: float, last_log: float,
                           local_interval: float) -> bool:
    elapsed_time = now - last_log
    return elapsed_time > local_interval


def get_throughput(tracked_stats: List[int], now: float,
                   last_log: float) -> float:
    return float(np.sum(tracked_stats) / (now - last_log))
333
334


335
336
337
338
339
340
341
342
343
344
345
class LoggingStatLogger(StatLoggerBase):
    """LoggingStatLogger is used in LLMEngine to log to Stdout."""

    def log(self, stats: Stats) -> None:
        """Called by LLMEngine.
           Logs to Stdout every self.local_interval seconds."""

        # Save tracked stats for token counters.
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)

346
347
348
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
        # Log locally every local_interval seconds.
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
            # Compute summary metrics for tracked stats (and log them
            # to promethus if applicable).
            prompt_throughput = get_throughput(self.num_prompt_tokens,
                                               now=stats.now,
                                               last_log=self.last_local_log)
            generation_throughput = get_throughput(
                self.num_generation_tokens,
                now=stats.now,
                last_log=self.last_local_log)

            # Log to stdout.
            logger.info(
                "Avg prompt throughput: %.1f tokens/s, "
                "Avg generation throughput: %.1f tokens/s, "
                "Running: %d reqs, Swapped: %d reqs, "
                "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
                "CPU KV cache usage: %.1f%%.",
                prompt_throughput,
                generation_throughput,
                stats.num_running_sys,
                stats.num_swapped_sys,
                stats.num_waiting_sys,
                stats.gpu_cache_usage_sys * 100,
                stats.cpu_cache_usage_sys * 100,
            )
377
378
379
380
381
382
383
            if (stats.cpu_prefix_cache_hit_rate >= 0
                    or stats.gpu_prefix_cache_hit_rate >= 0):
                logger.info(
                    "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
                    stats.gpu_prefix_cache_hit_rate * 100,
                    stats.cpu_prefix_cache_hit_rate * 100,
                )
384
385
386
387
388
            if self.spec_decode_metrics is not None:
                logger.info(
                    self._format_spec_decode_metrics_str(
                        self.spec_decode_metrics))

389
390
391
392
            # Reset tracked stats for next interval.
            self.num_prompt_tokens = []
            self.num_generation_tokens = []
            self.last_local_log = stats.now
393
            self.spec_decode_metrics = None
394
395
396
397
398
399
400
401
402

    def _format_spec_decode_metrics_str(
            self, metrics: "SpecDecodeWorkerMetrics") -> str:

        return ("Speculative metrics: "
                f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
                f"System efficiency: {metrics.system_efficiency:.3f}, "
                f"Number of speculative tokens: {metrics.num_spec_tokens}, "
                f"Number of accepted tokens: {metrics.accepted_tokens}, "
403
404
                f"Number of draft tokens: {metrics.draft_tokens}, "
                f"Number of emitted tokens: {metrics.emitted_tokens}.")
405

406
407
408
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        raise NotImplementedError

409
410
411
412

class PrometheusStatLogger(StatLoggerBase):
    """PrometheusStatLogger is used LLMEngine to log to Promethus."""
    _metrics_cls = Metrics
413
    _gauge_cls = prometheus_client.Gauge
414
415
416
417

    def __init__(self, local_interval: float, labels: Dict[str, str],
                 max_model_len: int) -> None:
        super().__init__(local_interval)
418
419
        # Prometheus metrics
        self.labels = labels
420
421
        self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
                                         max_model_len=max_model_len)
422

423
424
425
    def _log_gauge(self, gauge, data: Union[int, float]) -> None:
        # Convenience function for logging to gauge.
        gauge.labels(**self.labels).set(data)
426

427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
    def _log_counter(self, counter, data: Union[int, float]) -> None:
        # Convenience function for logging to counter.
        counter.labels(**self.labels).inc(data)

    def _log_counter_labels(self, counter, data: CollectionsCounter,
                            label_key: str) -> None:
        # Convenience function for collection counter of labels.
        for label, count in data.items():
            counter.labels(**{**self.labels, label_key: label}).inc(count)

    def _log_histogram(self, histogram, data: Union[List[int],
                                                    List[float]]) -> None:
        # Convenience function for logging list to histogram.
        for datum in data:
            histogram.labels(**self.labels).observe(datum)
442

443
444
445
    def _log_gauge_string(self, gauge, data: Dict[str, str]) -> None:
        gauge.labels(**data).set(1)

446
    def _log_prometheus(self, stats: Stats) -> None:
447
448
449
450
451
452
453
454
455
456
457
        # System state data
        self._log_gauge(self.metrics.gauge_scheduler_running,
                        stats.num_running_sys)
        self._log_gauge(self.metrics.gauge_scheduler_swapped,
                        stats.num_swapped_sys)
        self._log_gauge(self.metrics.gauge_scheduler_waiting,
                        stats.num_waiting_sys)
        self._log_gauge(self.metrics.gauge_gpu_cache_usage,
                        stats.gpu_cache_usage_sys)
        self._log_gauge(self.metrics.gauge_cpu_cache_usage,
                        stats.cpu_cache_usage_sys)
458
459
460
461
        self._log_gauge(self.metrics.gauge_cpu_prefix_cache_hit_rate,
                        stats.cpu_prefix_cache_hit_rate)
        self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate,
                        stats.gpu_prefix_cache_hit_rate)
462
463
464
465
466
467
468
469
470
471
472
        # Including max-lora in metric, in future this property of lora
        # config maybe extended to be dynamic.
        lora_info = {
            self.metrics.labelname_running_lora_adapters:
            ",".join(stats.running_lora_adapters),
            self.metrics.labelname_waiting_lora_adapters:
            ",".join(stats.waiting_lora_adapters),
            self.metrics.labelname_max_lora:
            stats.max_lora,
        }
        self._log_gauge_string(self.metrics.gauge_lora_info, lora_info)
473
        # Iteration level data
474
475
        self._log_counter(self.metrics.counter_num_preemption,
                          stats.num_preemption_iter)
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
        self._log_counter(self.metrics.counter_prompt_tokens,
                          stats.num_prompt_tokens_iter)
        self._log_counter(self.metrics.counter_generation_tokens,
                          stats.num_generation_tokens_iter)
        self._log_histogram(self.metrics.histogram_time_to_first_token,
                            stats.time_to_first_tokens_iter)
        self._log_histogram(self.metrics.histogram_time_per_output_token,
                            stats.time_per_output_tokens_iter)

        # Request level data
        # Latency
        self._log_histogram(self.metrics.histogram_e2e_time_request,
                            stats.time_e2e_requests)
        # Metadata
        finished_reason_counter = CollectionsCounter(
            stats.finished_reason_requests)
        self._log_counter_labels(self.metrics.counter_request_success,
                                 finished_reason_counter,
                                 Metrics.labelname_finish_reason)
        self._log_histogram(self.metrics.histogram_num_prompt_tokens_request,
                            stats.num_prompt_tokens_requests)
        self._log_histogram(
            self.metrics.histogram_num_generation_tokens_request,
            stats.num_generation_tokens_requests)
        self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)

502
503
504
    def _log_prometheus_interval(self, prompt_throughput: float,
                                 generation_throughput: float) -> None:
        # Logs metrics to prometheus that are computed every logging_interval.
505
506
507
508
509
510
        # Support legacy gauge metrics that make throughput calculations on
        # the vLLM side. Moving forward, we should use counters like
        # counter_prompt_tokens, counter_generation_tokens
        # Which log raw data and calculate summaries using rate() on the
        # grafana/prometheus side. See
        # https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666
511
512
513
514
        self.metrics.gauge_avg_prompt_throughput.labels(
            **self.labels).set(prompt_throughput)
        self.metrics.gauge_avg_generation_throughput.labels(
            **self.labels).set(generation_throughput)
515

516
517
    def log(self, stats: Stats):
        """Logs to prometheus and tracked stats every iteration."""
518
519
520
521
        # Log to prometheus.
        self._log_prometheus(stats)

        # Save tracked stats for token counters.
522
523
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)
524

525
526
527
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

528
        # Log locally every local_interval seconds.
529
530
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
531
532
            # Compute summary metrics for tracked stats (and log them
            # to promethus if applicable).
533
534
535
536
537
538
539
540
            prompt_throughput = get_throughput(self.num_prompt_tokens,
                                               now=stats.now,
                                               last_log=self.last_local_log)
            generation_throughput = get_throughput(
                self.num_generation_tokens,
                now=stats.now,
                last_log=self.last_local_log)

541
542
543
            self._log_prometheus_interval(
                prompt_throughput=prompt_throughput,
                generation_throughput=generation_throughput)
544

545
            if self.spec_decode_metrics is not None:
546
547
                self._log_gauge(
                    self.metrics.gauge_spec_decode_draft_acceptance_rate,
548
                    self.spec_decode_metrics.draft_acceptance_rate)
549
                self._log_gauge(self.metrics.gauge_spec_decode_efficiency,
550
                                self.spec_decode_metrics.system_efficiency)
551
552
                self._log_counter(
                    self.metrics.counter_spec_decode_num_accepted_tokens,
553
                    self.spec_decode_metrics.accepted_tokens)
554
555
                self._log_counter(
                    self.metrics.counter_spec_decode_num_draft_tokens,
556
                    self.spec_decode_metrics.draft_tokens)
557
558
                self._log_counter(
                    self.metrics.counter_spec_decode_num_emitted_tokens,
559
560
561
562
563
564
565
                    self.spec_decode_metrics.emitted_tokens)

            # Reset tracked stats for next interval.
            self.num_prompt_tokens = []
            self.num_generation_tokens = []
            self.last_local_log = stats.now
            self.spec_decode_metrics = None
566

567
568
569
570
571
572
573
574
575
576
577
578
579
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        # Info type metrics are syntactic sugar for a gauge permanently set to 1
        # Since prometheus multiprocessing mode does not support Info, emulate
        # info here with a gauge.
        if type == "cache_config":
            metrics_info = obj.metrics_info()
            info_gauge = self._gauge_cls(
                name="vllm:cache_config_info",
                documentation="Information of the LLMEngine CacheConfig",
                labelnames=metrics_info.keys(),
                multiprocess_mode="mostrecent")
            info_gauge.labels(**metrics_info).set(1)

580

581
582
class RayPrometheusStatLogger(PrometheusStatLogger):
    """RayPrometheusStatLogger uses Ray metrics instead."""
583
    _metrics_cls = RayMetrics
584
585
586

    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        return None