metrics.py 26.1 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import time
4
5
from typing import TYPE_CHECKING
from typing import Counter as CollectionsCounter
6
from typing import Dict, List, Optional, Type, Union, cast
7
8

import numpy as np
9
import prometheus_client
10

11
12
from vllm.config import SupportsMetricsInfo, VllmConfig
from vllm.engine.metrics_types import StatLoggerBase, Stats
13
from vllm.executor.ray_utils import ray
14
from vllm.logger import init_logger
15

16
17
18
19
20
if ray is not None:
    from ray.util import metrics as ray_metrics
else:
    ray_metrics = None

21
22
23
if TYPE_CHECKING:
    from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics

24
25
logger = init_logger(__name__)

26
prometheus_client.disable_created_metrics()
27
28
29
30

# The begin-* and end* here are used by the documentation generator
# to extract the metrics definitions.

31

32
# --8<-- [start:metrics-definitions]
33
class Metrics:
34
35
36
37
38
39
    """
    vLLM uses a multiprocessing-based frontend for the OpenAI server.
    This means that we need to run prometheus_client in multiprocessing mode
    See https://prometheus.github.io/client_python/multiprocess/ for more
    details on limitations.
    """
40

41
    labelname_finish_reason = "finished_reason"
42
43
44
    labelname_waiting_lora_adapters = "waiting_lora_adapters"
    labelname_running_lora_adapters = "running_lora_adapters"
    labelname_max_lora = "max_lora"
45
46
47
    _gauge_cls = prometheus_client.Gauge
    _counter_cls = prometheus_client.Counter
    _histogram_cls = prometheus_client.Histogram
48

49
    def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
50
        # Unregister any existing vLLM collectors (for CI/CD)
51
        self._unregister_vllm_metrics()
52

53
54
        max_model_len = vllm_config.model_config.max_model_len

55
56
57
58
59
        # Use this flag to hide metrics that were deprecated in
        # a previous release and which will be removed future
        self.show_hidden_metrics = \
            vllm_config.observability_config.show_hidden_metrics

60
        # System stats
61
        #   Scheduler State
62
        self.gauge_scheduler_running = self._gauge_cls(
63
64
            name="vllm:num_requests_running",
            documentation="Number of requests currently running on GPU.",
65
66
            labelnames=labelnames,
            multiprocess_mode="sum")
67
        self.gauge_scheduler_waiting = self._gauge_cls(
68
69
            name="vllm:num_requests_waiting",
            documentation="Number of requests waiting to be processed.",
70
71
            labelnames=labelnames,
            multiprocess_mode="sum")
72
73
74
75
76
77
78
79
80
81
        self.gauge_lora_info = self._gauge_cls(
            name="vllm:lora_requests_info",
            documentation="Running stats on lora requests.",
            labelnames=[
                self.labelname_running_lora_adapters,
                self.labelname_max_lora,
                self.labelname_waiting_lora_adapters,
            ],
            multiprocess_mode="livemostrecent",
        )
82

83
        #   KV Cache Usage in %
84
        self.gauge_gpu_cache_usage = self._gauge_cls(
85
86
            name="vllm:gpu_cache_usage_perc",
            documentation="GPU KV-cache usage. 1 means 100 percent usage.",
87
88
            labelnames=labelnames,
            multiprocess_mode="sum")
89

90
        # Iteration stats
91
        self.counter_num_preemption = self._counter_cls(
92
93
94
            name="vllm:num_preemptions_total",
            documentation="Cumulative number of preemption from the engine.",
            labelnames=labelnames)
95
        self.counter_prompt_tokens = self._counter_cls(
96
97
98
            name="vllm:prompt_tokens_total",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames)
99
        self.counter_generation_tokens = self._counter_cls(
100
101
102
            name="vllm:generation_tokens_total",
            documentation="Number of generation tokens processed.",
            labelnames=labelnames)
harrywu's avatar
harrywu committed
103
104
105
106
        self.histogram_iteration_tokens = self._histogram_cls(
            name="vllm:iteration_tokens_total",
            documentation="Histogram of number of tokens per engine_step.",
            labelnames=labelnames,
107
            buckets=[
108
                1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192, 16384
109
            ])
110
        self.histogram_time_to_first_token = self._histogram_cls(
111
112
113
114
115
            name="vllm:time_to_first_token_seconds",
            documentation="Histogram of time to first token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
116
117
                0.75, 1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0, 80.0, 160.0, 640.0,
                2560.0
118
            ])
119
        self.histogram_time_per_output_token = self._histogram_cls(
120
121
122
123
124
            name="vllm:time_per_output_token_seconds",
            documentation="Histogram of time per output token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
125
                1.0, 2.5, 5.0, 7.5, 10.0, 20.0, 40.0, 80.0
126
            ])
127
128
129

        # Request stats
        #   Latency
harrywu's avatar
harrywu committed
130
131
        request_latency_buckets = [
            0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
132
            40.0, 50.0, 60.0, 120.0, 240.0, 480.0, 960.0, 1920.0, 7680.0
harrywu's avatar
harrywu committed
133
        ]
134
        self.histogram_e2e_time_request = self._histogram_cls(
135
136
137
            name="vllm:e2e_request_latency_seconds",
            documentation="Histogram of end to end request latency in seconds.",
            labelnames=labelnames,
harrywu's avatar
harrywu committed
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
            buckets=request_latency_buckets)
        self.histogram_queue_time_request = self._histogram_cls(
            name="vllm:request_queue_time_seconds",
            documentation=
            "Histogram of time spent in WAITING phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_inference_time_request = self._histogram_cls(
            name="vllm:request_inference_time_seconds",
            documentation=
            "Histogram of time spent in RUNNING phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_prefill_time_request = self._histogram_cls(
            name="vllm:request_prefill_time_seconds",
            documentation=
            "Histogram of time spent in PREFILL phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_decode_time_request = self._histogram_cls(
            name="vllm:request_decode_time_seconds",
            documentation=
            "Histogram of time spent in DECODE phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
163

164
        #   Metadata
165
        self.histogram_num_prompt_tokens_request = self._histogram_cls(
166
167
168
169
170
            name="vllm:request_prompt_tokens",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len),
        )
171
        self.histogram_num_generation_tokens_request = \
172
            self._histogram_cls(
173
174
175
176
177
                name="vllm:request_generation_tokens",
                documentation="Number of generation tokens processed.",
                labelnames=labelnames,
                buckets=build_1_2_5_buckets(max_model_len),
            )
harrywu's avatar
harrywu committed
178
179
180
181
182
183
        self.histogram_max_num_generation_tokens_request = self._histogram_cls(
            name="vllm:request_max_num_generation_tokens",
            documentation=
            "Histogram of maximum number of requested generation tokens.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len))
184
        self.histogram_n_request = self._histogram_cls(
185
186
187
188
189
            name="vllm:request_params_n",
            documentation="Histogram of the n request parameter.",
            labelnames=labelnames,
            buckets=[1, 2, 5, 10, 20],
        )
190
191
192
193
194
195
        self.histogram_max_tokens_request = self._histogram_cls(
            name="vllm:request_params_max_tokens",
            documentation="Histogram of the max_tokens request parameter.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len),
        )
196
        self.counter_request_success = self._counter_cls(
197
            name="vllm:request_success_total",
198
199
            documentation="Count of successfully processed requests.",
            labelnames=labelnames + [Metrics.labelname_finish_reason])
200

201
        # Speculative decoding stats
202
        self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
203
204
            name="vllm:spec_decode_draft_acceptance_rate",
            documentation="Speulative token acceptance rate.",
205
206
            labelnames=labelnames,
            multiprocess_mode="sum")
207
        self.gauge_spec_decode_efficiency = self._gauge_cls(
208
209
            name="vllm:spec_decode_efficiency",
            documentation="Speculative decoding system efficiency.",
210
211
            labelnames=labelnames,
            multiprocess_mode="sum")
212
213
214
215
216
        self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_accepted_tokens_total",
            documentation="Number of accepted tokens.",
            labelnames=labelnames))
        self.counter_spec_decode_num_draft_tokens = self._counter_cls(
217
218
219
            name="vllm:spec_decode_num_draft_tokens_total",
            documentation="Number of draft tokens.",
            labelnames=labelnames)
220
221
222
223
        self.counter_spec_decode_num_emitted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_emitted_tokens_total",
            documentation="Number of emitted tokens.",
            labelnames=labelnames))
224

225

226
# --8<-- [end:metrics-definitions]
227

228
    def _unregister_vllm_metrics(self) -> None:
229
        for collector in list(prometheus_client.REGISTRY._collector_to_names):
230
            if hasattr(collector, "_name") and "vllm" in collector._name:
231
232
233
234
235
236
237
238
239
240
                prometheus_client.REGISTRY.unregister(collector)


class _RayGaugeWrapper:
    """Wraps around ray.util.metrics.Gauge to provide same API as
    prometheus_client.Gauge"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
241
242
243
                 labelnames: Optional[List[str]] = None,
                 multiprocess_mode: str = ""):
        del multiprocess_mode
244
245
246
247
248
249
250
251
252
253
254
255
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._gauge = ray_metrics.Gauge(name=name,
                                        description=documentation,
                                        tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._gauge.set_default_tags(labels)
        return self

    def set(self, value: Union[int, float]):
        return self._gauge.set(value)

256
257
258
259
    def set_to_current_time(self):
        # ray metrics doesn't have set_to_current time, https://docs.ray.io/en/latest/_modules/ray/util/metrics.html
        return self._gauge.set(time.time())

260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293

class _RayCounterWrapper:
    """Wraps around ray.util.metrics.Counter to provide same API as
    prometheus_client.Counter"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._counter = ray_metrics.Counter(name=name,
                                            description=documentation,
                                            tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._counter.set_default_tags(labels)
        return self

    def inc(self, value: Union[int, float] = 1.0):
        if value == 0:
            return
        return self._counter.inc(value)


class _RayHistogramWrapper:
    """Wraps around ray.util.metrics.Histogram to provide same API as
    prometheus_client.Histogram"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None,
                 buckets: Optional[List[float]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
294
        boundaries = buckets if buckets else []
295
296
297
        self._histogram = ray_metrics.Histogram(name=name,
                                                description=documentation,
                                                tag_keys=labelnames_tuple,
298
                                                boundaries=boundaries)
299
300
301
302
303
304
305

    def labels(self, **labels):
        self._histogram.set_default_tags(labels)
        return self

    def observe(self, value: Union[int, float]):
        return self._histogram.observe(value)
306
307
308
309
310
311
312


class RayMetrics(Metrics):
    """
    RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
    Provides the same metrics as Metrics but uses Ray's util.metrics library.
    """
313
314
315
316
317
318
    _gauge_cls: Type[prometheus_client.Gauge] = cast(
        Type[prometheus_client.Gauge], _RayGaugeWrapper)
    _counter_cls: Type[prometheus_client.Counter] = cast(
        Type[prometheus_client.Counter], _RayCounterWrapper)
    _histogram_cls: Type[prometheus_client.Histogram] = cast(
        Type[prometheus_client.Histogram], _RayHistogramWrapper)
319

320
    def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
321
322
        if ray_metrics is None:
            raise ImportError("RayMetrics requires Ray to be installed.")
323
        super().__init__(labelnames, vllm_config)
324
325
326
327
328

    def _unregister_vllm_metrics(self) -> None:
        # No-op on purpose
        pass

329

330
def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:
331
    """
332
333
    Builds a list of buckets with increasing powers of 10 multiplied by
    mantissa values until the value exceeds the specified maximum.
334
335
336

    """
    exponent = 0
337
    buckets: List[int] = []
338
339
340
341
342
343
344
345
346
347
    while True:
        for m in mantissa_lst:
            value = m * 10**exponent
            if value <= max_value:
                buckets.append(value)
            else:
                return buckets
        exponent += 1


348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
def build_1_2_5_buckets(max_value: int) -> List[int]:
    """
    Example:
    >>> build_1_2_5_buckets(100)
    [1, 2, 5, 10, 20, 50, 100]
    """
    return build_buckets([1, 2, 5], max_value)


def build_1_2_3_5_8_buckets(max_value: int) -> List[int]:
    """
    Example:
    >>> build_1_2_3_5_8_buckets(100)
    [1, 2, 3, 5, 8, 10, 20, 30, 50, 80, 100]
    """
    return build_buckets([1, 2, 3, 5, 8], max_value)


366
367
368
369
370
371
372
373
374
def local_interval_elapsed(now: float, last_log: float,
                           local_interval: float) -> bool:
    elapsed_time = now - last_log
    return elapsed_time > local_interval


def get_throughput(tracked_stats: List[int], now: float,
                   last_log: float) -> float:
    return float(np.sum(tracked_stats) / (now - last_log))
375
376


377
378
379
class LoggingStatLogger(StatLoggerBase):
    """LoggingStatLogger is used in LLMEngine to log to Stdout."""

380
381
    def __init__(self, local_interval: float, vllm_config: VllmConfig) -> None:
        super().__init__(local_interval, vllm_config)
382
383
384
        self.last_prompt_throughput: Optional[float] = None
        self.last_generation_throughput: Optional[float] = None

385
386
387
388
389
390
391
392
    def log(self, stats: Stats) -> None:
        """Called by LLMEngine.
           Logs to Stdout every self.local_interval seconds."""

        # Save tracked stats for token counters.
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)

393
394
395
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

396
397
398
399
400
401
402
403
404
405
406
407
408
        # Log locally every local_interval seconds.
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
            # Compute summary metrics for tracked stats (and log them
            # to promethus if applicable).
            prompt_throughput = get_throughput(self.num_prompt_tokens,
                                               now=stats.now,
                                               last_log=self.last_local_log)
            generation_throughput = get_throughput(
                self.num_generation_tokens,
                now=stats.now,
                last_log=self.last_local_log)

409
410
411
412
413
414
415
416
            log_fn = logger.info
            if not any((prompt_throughput, generation_throughput,
                        self.last_prompt_throughput,
                        self.last_generation_throughput)):
                # Avoid log noise on an idle production system
                log_fn = logger.debug

            log_fn(
417
418
419
420
421
422
423
424
425
426
427
428
429
                "Avg prompt throughput: %.1f tokens/s, "
                "Avg generation throughput: %.1f tokens/s, "
                "Running: %d reqs, Swapped: %d reqs, "
                "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
                "CPU KV cache usage: %.1f%%.",
                prompt_throughput,
                generation_throughput,
                stats.num_running_sys,
                stats.num_swapped_sys,
                stats.num_waiting_sys,
                stats.gpu_cache_usage_sys * 100,
                stats.cpu_cache_usage_sys * 100,
            )
430
431
            if (stats.cpu_prefix_cache_hit_rate >= 0
                    or stats.gpu_prefix_cache_hit_rate >= 0):
432
                log_fn(
433
434
435
436
                    "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
                    stats.gpu_prefix_cache_hit_rate * 100,
                    stats.cpu_prefix_cache_hit_rate * 100,
                )
437
            if self.spec_decode_metrics is not None:
438
                log_fn(
439
440
441
                    self._format_spec_decode_metrics_str(
                        self.spec_decode_metrics))

442
443
444
445
446
447
448
449
450
451
            self._reset(stats, prompt_throughput, generation_throughput)

    def _reset(self, stats, prompt_throughput, generation_throughput) -> None:
        # Reset tracked stats for next interval.
        self.num_prompt_tokens = []
        self.num_generation_tokens = []
        self.last_local_log = stats.now
        self.spec_decode_metrics = None
        self.last_prompt_throughput = prompt_throughput
        self.last_generation_throughput = generation_throughput
452
453
454
455
456
457
458
459
460

    def _format_spec_decode_metrics_str(
            self, metrics: "SpecDecodeWorkerMetrics") -> str:

        return ("Speculative metrics: "
                f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
                f"System efficiency: {metrics.system_efficiency:.3f}, "
                f"Number of speculative tokens: {metrics.num_spec_tokens}, "
                f"Number of accepted tokens: {metrics.accepted_tokens}, "
461
462
                f"Number of draft tokens: {metrics.draft_tokens}, "
                f"Number of emitted tokens: {metrics.emitted_tokens}.")
463

464
465
466
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        raise NotImplementedError

467
468
469
470

class PrometheusStatLogger(StatLoggerBase):
    """PrometheusStatLogger is used LLMEngine to log to Promethus."""
    _metrics_cls = Metrics
471
    _gauge_cls = prometheus_client.Gauge
472
473

    def __init__(self, local_interval: float, labels: Dict[str, str],
474
475
                 vllm_config: VllmConfig) -> None:
        super().__init__(local_interval, vllm_config)
476
477
        # Prometheus metrics
        self.labels = labels
478
        self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
479
                                         vllm_config=vllm_config)
480

481
482
483
    def _log_gauge(self, gauge, data: Union[int, float]) -> None:
        # Convenience function for logging to gauge.
        gauge.labels(**self.labels).set(data)
484

485
486
    def _log_counter(self, counter, data: Union[int, float]) -> None:
        # Convenience function for logging to counter.
487
488
489
490
491
        # Prevent ValueError from negative increment
        if data < 0:
            logger.warning("Skipping negative increment of %g to %s", data,
                           counter)
            return
492
493
494
495
496
497
498
499
500
501
502
503
504
        counter.labels(**self.labels).inc(data)

    def _log_counter_labels(self, counter, data: CollectionsCounter,
                            label_key: str) -> None:
        # Convenience function for collection counter of labels.
        for label, count in data.items():
            counter.labels(**{**self.labels, label_key: label}).inc(count)

    def _log_histogram(self, histogram, data: Union[List[int],
                                                    List[float]]) -> None:
        # Convenience function for logging list to histogram.
        for datum in data:
            histogram.labels(**self.labels).observe(datum)
505

506
    def _log_gauge_string(self, gauge, data: Dict[str, str]) -> None:
507
        gauge.labels(**data).set_to_current_time()
508

509
    def _log_prometheus(self, stats: Stats) -> None:
510
511
512
513
514
515
516
        # System state data
        self._log_gauge(self.metrics.gauge_scheduler_running,
                        stats.num_running_sys)
        self._log_gauge(self.metrics.gauge_scheduler_waiting,
                        stats.num_waiting_sys)
        self._log_gauge(self.metrics.gauge_gpu_cache_usage,
                        stats.gpu_cache_usage_sys)
517
518
519
520
521
522
523
524
525
526
527
        # Including max-lora in metric, in future this property of lora
        # config maybe extended to be dynamic.
        lora_info = {
            self.metrics.labelname_running_lora_adapters:
            ",".join(stats.running_lora_adapters),
            self.metrics.labelname_waiting_lora_adapters:
            ",".join(stats.waiting_lora_adapters),
            self.metrics.labelname_max_lora:
            stats.max_lora,
        }
        self._log_gauge_string(self.metrics.gauge_lora_info, lora_info)
528
        # Iteration level data
529
530
        self._log_counter(self.metrics.counter_num_preemption,
                          stats.num_preemption_iter)
531
532
533
534
        self._log_counter(self.metrics.counter_prompt_tokens,
                          stats.num_prompt_tokens_iter)
        self._log_counter(self.metrics.counter_generation_tokens,
                          stats.num_generation_tokens_iter)
harrywu's avatar
harrywu committed
535
536
        self._log_histogram(self.metrics.histogram_iteration_tokens,
                            [stats.num_tokens_iter])
537
538
539
540
541
542
543
544
545
        self._log_histogram(self.metrics.histogram_time_to_first_token,
                            stats.time_to_first_tokens_iter)
        self._log_histogram(self.metrics.histogram_time_per_output_token,
                            stats.time_per_output_tokens_iter)

        # Request level data
        # Latency
        self._log_histogram(self.metrics.histogram_e2e_time_request,
                            stats.time_e2e_requests)
harrywu's avatar
harrywu committed
546
547
548
549
550
        self._log_histogram(self.metrics.histogram_queue_time_request,
                            stats.time_queue_requests)
        self._log_histogram(self.metrics.histogram_inference_time_request,
                            stats.time_inference_requests)
        self._log_histogram(self.metrics.histogram_prefill_time_request,
551
552
                            stats.time_prefill_requests)
        self._log_histogram(self.metrics.histogram_decode_time_request,
harrywu's avatar
harrywu committed
553
                            stats.time_decode_requests)
554
555
556
557
558
559
560
561
562
563
564
565
        # Metadata
        finished_reason_counter = CollectionsCounter(
            stats.finished_reason_requests)
        self._log_counter_labels(self.metrics.counter_request_success,
                                 finished_reason_counter,
                                 Metrics.labelname_finish_reason)
        self._log_histogram(self.metrics.histogram_num_prompt_tokens_request,
                            stats.num_prompt_tokens_requests)
        self._log_histogram(
            self.metrics.histogram_num_generation_tokens_request,
            stats.num_generation_tokens_requests)
        self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
harrywu's avatar
harrywu committed
566
567
568
        self._log_histogram(
            self.metrics.histogram_max_num_generation_tokens_request,
            stats.max_num_generation_tokens_requests)
569
570
        self._log_histogram(self.metrics.histogram_max_tokens_request,
                            stats.max_tokens_requests)
571

572
573
    def log(self, stats: Stats):
        """Logs to prometheus and tracked stats every iteration."""
574
575
576
577
        # Log to prometheus.
        self._log_prometheus(stats)

        # Save tracked stats for token counters.
578
579
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)
580

581
582
583
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

584
        # Log locally every local_interval seconds.
585
586
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
587
            if self.spec_decode_metrics is not None:
588
589
                self._log_gauge(
                    self.metrics.gauge_spec_decode_draft_acceptance_rate,
590
                    self.spec_decode_metrics.draft_acceptance_rate)
591
                self._log_gauge(self.metrics.gauge_spec_decode_efficiency,
592
                                self.spec_decode_metrics.system_efficiency)
593
594
                self._log_counter(
                    self.metrics.counter_spec_decode_num_accepted_tokens,
595
                    self.spec_decode_metrics.accepted_tokens)
596
597
                self._log_counter(
                    self.metrics.counter_spec_decode_num_draft_tokens,
598
                    self.spec_decode_metrics.draft_tokens)
599
600
                self._log_counter(
                    self.metrics.counter_spec_decode_num_emitted_tokens,
601
602
603
604
605
606
607
                    self.spec_decode_metrics.emitted_tokens)

            # Reset tracked stats for next interval.
            self.num_prompt_tokens = []
            self.num_generation_tokens = []
            self.last_local_log = stats.now
            self.spec_decode_metrics = None
608

609
610
611
612
613
614
615
616
617
618
619
620
621
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        # Info type metrics are syntactic sugar for a gauge permanently set to 1
        # Since prometheus multiprocessing mode does not support Info, emulate
        # info here with a gauge.
        if type == "cache_config":
            metrics_info = obj.metrics_info()
            info_gauge = self._gauge_cls(
                name="vllm:cache_config_info",
                documentation="Information of the LLMEngine CacheConfig",
                labelnames=metrics_info.keys(),
                multiprocess_mode="mostrecent")
            info_gauge.labels(**metrics_info).set(1)

622

623
624
class RayPrometheusStatLogger(PrometheusStatLogger):
    """RayPrometheusStatLogger uses Ray metrics instead."""
625
    _metrics_cls = RayMetrics
626
627
628

    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        return None