metrics.py 26.6 KB
Newer Older
1
import time
2
3
from typing import TYPE_CHECKING
from typing import Counter as CollectionsCounter
4
from typing import Dict, List, Optional, Type, Union, cast
5
6

import numpy as np
7
import prometheus_client
8

9
10
from vllm.engine.metrics_types import (StatLoggerBase, Stats,
                                       SupportsMetricsInfo)
11
from vllm.executor.ray_utils import ray
12
from vllm.logger import init_logger
13

14
15
16
17
18
if ray is not None:
    from ray.util import metrics as ray_metrics
else:
    ray_metrics = None

19
20
21
if TYPE_CHECKING:
    from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics

22
23
logger = init_logger(__name__)

24
prometheus_client.disable_created_metrics()
25
26
27
28

# The begin-* and end* here are used by the documentation generator
# to extract the metrics definitions.

29

30
# begin-metrics-definitions
31
class Metrics:
32
33
34
35
36
37
    """
    vLLM uses a multiprocessing-based frontend for the OpenAI server.
    This means that we need to run prometheus_client in multiprocessing mode
    See https://prometheus.github.io/client_python/multiprocess/ for more
    details on limitations.
    """
38

39
    labelname_finish_reason = "finished_reason"
40
41
42
    labelname_waiting_lora_adapters = "waiting_lora_adapters"
    labelname_running_lora_adapters = "running_lora_adapters"
    labelname_max_lora = "max_lora"
43
44
45
    _gauge_cls = prometheus_client.Gauge
    _counter_cls = prometheus_client.Counter
    _histogram_cls = prometheus_client.Histogram
46

47
    def __init__(self, labelnames: List[str], max_model_len: int):
48
        # Unregister any existing vLLM collectors (for CI/CD)
49
        self._unregister_vllm_metrics()
50
51

        # System stats
52
        #   Scheduler State
53
        self.gauge_scheduler_running = self._gauge_cls(
54
55
            name="vllm:num_requests_running",
            documentation="Number of requests currently running on GPU.",
56
57
            labelnames=labelnames,
            multiprocess_mode="sum")
58
        self.gauge_scheduler_waiting = self._gauge_cls(
59
60
            name="vllm:num_requests_waiting",
            documentation="Number of requests waiting to be processed.",
61
62
            labelnames=labelnames,
            multiprocess_mode="sum")
63
64
65
66
67
68
69
70
71
72
        self.gauge_lora_info = self._gauge_cls(
            name="vllm:lora_requests_info",
            documentation="Running stats on lora requests.",
            labelnames=[
                self.labelname_running_lora_adapters,
                self.labelname_max_lora,
                self.labelname_waiting_lora_adapters,
            ],
            multiprocess_mode="livemostrecent",
        )
73
        self.gauge_scheduler_swapped = self._gauge_cls(
74
75
            name="vllm:num_requests_swapped",
            documentation="Number of requests swapped to CPU.",
76
77
            labelnames=labelnames,
            multiprocess_mode="sum")
78
        #   KV Cache Usage in %
79
        self.gauge_gpu_cache_usage = self._gauge_cls(
80
81
            name="vllm:gpu_cache_usage_perc",
            documentation="GPU KV-cache usage. 1 means 100 percent usage.",
82
83
            labelnames=labelnames,
            multiprocess_mode="sum")
84
        self.gauge_cpu_cache_usage = self._gauge_cls(
85
86
            name="vllm:cpu_cache_usage_perc",
            documentation="CPU KV-cache usage. 1 means 100 percent usage.",
87
88
            labelnames=labelnames,
            multiprocess_mode="sum")
89
90
91
92
93
94
95
96
97
98
99
        #   Prefix caching block hit rate
        self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls(
            name="vllm:cpu_prefix_cache_hit_rate",
            documentation="CPU prefix cache block hit rate.",
            labelnames=labelnames,
            multiprocess_mode="sum")
        self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls(
            name="vllm:gpu_prefix_cache_hit_rate",
            documentation="GPU prefix cache block hit rate.",
            labelnames=labelnames,
            multiprocess_mode="sum")
100

101
        # Iteration stats
102
        self.counter_num_preemption = self._counter_cls(
103
104
105
            name="vllm:num_preemptions_total",
            documentation="Cumulative number of preemption from the engine.",
            labelnames=labelnames)
106
        self.counter_prompt_tokens = self._counter_cls(
107
108
109
            name="vllm:prompt_tokens_total",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames)
110
        self.counter_generation_tokens = self._counter_cls(
111
112
113
            name="vllm:generation_tokens_total",
            documentation="Number of generation tokens processed.",
            labelnames=labelnames)
114
        self.histogram_time_to_first_token = self._histogram_cls(
115
116
117
118
119
120
121
            name="vllm:time_to_first_token_seconds",
            documentation="Histogram of time to first token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
                0.75, 1.0, 2.5, 5.0, 7.5, 10.0
            ])
122
        self.histogram_time_per_output_token = self._histogram_cls(
123
124
125
126
127
128
129
            name="vllm:time_per_output_token_seconds",
            documentation="Histogram of time per output token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
                1.0, 2.5
            ])
130
131
132

        # Request stats
        #   Latency
133
        self.histogram_e2e_time_request = self._histogram_cls(
134
135
136
            name="vllm:e2e_request_latency_seconds",
            documentation="Histogram of end to end request latency in seconds.",
            labelnames=labelnames,
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
            buckets=[
                0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
                40.0, 50.0, 60.0
            ])
        self.histogram_time_in_queue_request = self._histogram_cls(
            name="vllm:time_in_queue_requests",
            documentation=
            "Histogram of time the request spent in the queue in seconds.",
            labelnames=labelnames,
            buckets=[
                0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
                40.0, 50.0, 60.0
            ])
        self.histogram_model_forward_time_request = self._histogram_cls(
            name="vllm:model_forward_time_milliseconds",
            documentation=
            "Histogram of time spent in the model forward pass in ms.",
            labelnames=labelnames,
            buckets=build_1_2_3_5_8_buckets(3000))
        self.histogram_model_execute_time_request = self._histogram_cls(
            name="vllm:model_execute_time_milliseconds",
            documentation=
            "Histogram of time spent in the model execute function in ms.",
            labelnames=labelnames,
            buckets=build_1_2_3_5_8_buckets(3000))
162
        #   Metadata
163
        self.histogram_num_prompt_tokens_request = self._histogram_cls(
164
165
166
167
168
            name="vllm:request_prompt_tokens",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len),
        )
169
        self.histogram_num_generation_tokens_request = \
170
            self._histogram_cls(
171
172
173
174
175
                name="vllm:request_generation_tokens",
                documentation="Number of generation tokens processed.",
                labelnames=labelnames,
                buckets=build_1_2_5_buckets(max_model_len),
            )
176
        self.histogram_n_request = self._histogram_cls(
177
178
179
180
181
            name="vllm:request_params_n",
            documentation="Histogram of the n request parameter.",
            labelnames=labelnames,
            buckets=[1, 2, 5, 10, 20],
        )
182
        self.counter_request_success = self._counter_cls(
183
            name="vllm:request_success_total",
184
185
            documentation="Count of successfully processed requests.",
            labelnames=labelnames + [Metrics.labelname_finish_reason])
186

187
        # Speculatie decoding stats
188
        self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
189
190
            name="vllm:spec_decode_draft_acceptance_rate",
            documentation="Speulative token acceptance rate.",
191
192
            labelnames=labelnames,
            multiprocess_mode="sum")
193
        self.gauge_spec_decode_efficiency = self._gauge_cls(
194
195
            name="vllm:spec_decode_efficiency",
            documentation="Speculative decoding system efficiency.",
196
197
            labelnames=labelnames,
            multiprocess_mode="sum")
198
199
200
201
202
        self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_accepted_tokens_total",
            documentation="Number of accepted tokens.",
            labelnames=labelnames))
        self.counter_spec_decode_num_draft_tokens = self._counter_cls(
203
204
205
            name="vllm:spec_decode_num_draft_tokens_total",
            documentation="Number of draft tokens.",
            labelnames=labelnames)
206
207
208
209
        self.counter_spec_decode_num_emitted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_emitted_tokens_total",
            documentation="Number of emitted tokens.",
            labelnames=labelnames))
210

211
        # Deprecated in favor of vllm:prompt_tokens_total
212
        self.gauge_avg_prompt_throughput = self._gauge_cls(
213
214
215
            name="vllm:avg_prompt_throughput_toks_per_s",
            documentation="Average prefill throughput in tokens/s.",
            labelnames=labelnames,
216
            multiprocess_mode="sum",
217
        )
218
        # Deprecated in favor of vllm:generation_tokens_total
219
        self.gauge_avg_generation_throughput = self._gauge_cls(
220
221
222
            name="vllm:avg_generation_throughput_toks_per_s",
            documentation="Average generation throughput in tokens/s.",
            labelnames=labelnames,
223
            multiprocess_mode="sum",
224
225
        )

226
227

# end-metrics-definitions
228

229
    def _unregister_vllm_metrics(self) -> None:
230
        for collector in list(prometheus_client.REGISTRY._collector_to_names):
231
            if hasattr(collector, "_name") and "vllm" in collector._name:
232
233
234
235
236
237
238
239
240
241
                prometheus_client.REGISTRY.unregister(collector)


class _RayGaugeWrapper:
    """Wraps around ray.util.metrics.Gauge to provide same API as
    prometheus_client.Gauge"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
242
243
244
                 labelnames: Optional[List[str]] = None,
                 multiprocess_mode: str = ""):
        del multiprocess_mode
245
246
247
248
249
250
251
252
253
254
255
256
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._gauge = ray_metrics.Gauge(name=name,
                                        description=documentation,
                                        tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._gauge.set_default_tags(labels)
        return self

    def set(self, value: Union[int, float]):
        return self._gauge.set(value)

257
258
259
260
    def set_to_current_time(self):
        # ray metrics doesn't have set_to_current time, https://docs.ray.io/en/latest/_modules/ray/util/metrics.html
        return self._gauge.set(time.time())

261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294

class _RayCounterWrapper:
    """Wraps around ray.util.metrics.Counter to provide same API as
    prometheus_client.Counter"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._counter = ray_metrics.Counter(name=name,
                                            description=documentation,
                                            tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._counter.set_default_tags(labels)
        return self

    def inc(self, value: Union[int, float] = 1.0):
        if value == 0:
            return
        return self._counter.inc(value)


class _RayHistogramWrapper:
    """Wraps around ray.util.metrics.Histogram to provide same API as
    prometheus_client.Histogram"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None,
                 buckets: Optional[List[float]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
295
        boundaries = buckets if buckets else []
296
297
298
        self._histogram = ray_metrics.Histogram(name=name,
                                                description=documentation,
                                                tag_keys=labelnames_tuple,
299
                                                boundaries=boundaries)
300
301
302
303
304
305
306

    def labels(self, **labels):
        self._histogram.set_default_tags(labels)
        return self

    def observe(self, value: Union[int, float]):
        return self._histogram.observe(value)
307
308
309
310
311
312
313


class RayMetrics(Metrics):
    """
    RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
    Provides the same metrics as Metrics but uses Ray's util.metrics library.
    """
314
315
316
317
318
319
    _gauge_cls: Type[prometheus_client.Gauge] = cast(
        Type[prometheus_client.Gauge], _RayGaugeWrapper)
    _counter_cls: Type[prometheus_client.Counter] = cast(
        Type[prometheus_client.Counter], _RayCounterWrapper)
    _histogram_cls: Type[prometheus_client.Histogram] = cast(
        Type[prometheus_client.Histogram], _RayHistogramWrapper)
320
321
322
323
324
325
326
327
328
329

    def __init__(self, labelnames: List[str], max_model_len: int):
        if ray_metrics is None:
            raise ImportError("RayMetrics requires Ray to be installed.")
        super().__init__(labelnames, max_model_len)

    def _unregister_vllm_metrics(self) -> None:
        # No-op on purpose
        pass

330

331
def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:
332
    """
333
334
    Builds a list of buckets with increasing powers of 10 multiplied by
    mantissa values until the value exceeds the specified maximum.
335
336
337

    """
    exponent = 0
338
    buckets: List[int] = []
339
340
341
342
343
344
345
346
347
348
    while True:
        for m in mantissa_lst:
            value = m * 10**exponent
            if value <= max_value:
                buckets.append(value)
            else:
                return buckets
        exponent += 1


349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
def build_1_2_5_buckets(max_value: int) -> List[int]:
    """
    Example:
    >>> build_1_2_5_buckets(100)
    [1, 2, 5, 10, 20, 50, 100]
    """
    return build_buckets([1, 2, 5], max_value)


def build_1_2_3_5_8_buckets(max_value: int) -> List[int]:
    """
    Example:
    >>> build_1_2_3_5_8_buckets(100)
    [1, 2, 3, 5, 8, 10, 20, 30, 50, 80, 100]
    """
    return build_buckets([1, 2, 3, 5, 8], max_value)


367
368
369
370
371
372
373
374
375
def local_interval_elapsed(now: float, last_log: float,
                           local_interval: float) -> bool:
    elapsed_time = now - last_log
    return elapsed_time > local_interval


def get_throughput(tracked_stats: List[int], now: float,
                   last_log: float) -> float:
    return float(np.sum(tracked_stats) / (now - last_log))
376
377


378
379
380
381
382
383
384
385
386
387
388
class LoggingStatLogger(StatLoggerBase):
    """LoggingStatLogger is used in LLMEngine to log to Stdout."""

    def log(self, stats: Stats) -> None:
        """Called by LLMEngine.
           Logs to Stdout every self.local_interval seconds."""

        # Save tracked stats for token counters.
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)

389
390
391
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
        # Log locally every local_interval seconds.
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
            # Compute summary metrics for tracked stats (and log them
            # to promethus if applicable).
            prompt_throughput = get_throughput(self.num_prompt_tokens,
                                               now=stats.now,
                                               last_log=self.last_local_log)
            generation_throughput = get_throughput(
                self.num_generation_tokens,
                now=stats.now,
                last_log=self.last_local_log)

            # Log to stdout.
            logger.info(
                "Avg prompt throughput: %.1f tokens/s, "
                "Avg generation throughput: %.1f tokens/s, "
                "Running: %d reqs, Swapped: %d reqs, "
                "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
                "CPU KV cache usage: %.1f%%.",
                prompt_throughput,
                generation_throughput,
                stats.num_running_sys,
                stats.num_swapped_sys,
                stats.num_waiting_sys,
                stats.gpu_cache_usage_sys * 100,
                stats.cpu_cache_usage_sys * 100,
            )
420
421
422
423
424
425
426
            if (stats.cpu_prefix_cache_hit_rate >= 0
                    or stats.gpu_prefix_cache_hit_rate >= 0):
                logger.info(
                    "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
                    stats.gpu_prefix_cache_hit_rate * 100,
                    stats.cpu_prefix_cache_hit_rate * 100,
                )
427
428
429
430
431
            if self.spec_decode_metrics is not None:
                logger.info(
                    self._format_spec_decode_metrics_str(
                        self.spec_decode_metrics))

432
433
434
435
            # Reset tracked stats for next interval.
            self.num_prompt_tokens = []
            self.num_generation_tokens = []
            self.last_local_log = stats.now
436
            self.spec_decode_metrics = None
437
438
439
440
441
442
443
444
445

    def _format_spec_decode_metrics_str(
            self, metrics: "SpecDecodeWorkerMetrics") -> str:

        return ("Speculative metrics: "
                f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
                f"System efficiency: {metrics.system_efficiency:.3f}, "
                f"Number of speculative tokens: {metrics.num_spec_tokens}, "
                f"Number of accepted tokens: {metrics.accepted_tokens}, "
446
447
                f"Number of draft tokens: {metrics.draft_tokens}, "
                f"Number of emitted tokens: {metrics.emitted_tokens}.")
448

449
450
451
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        raise NotImplementedError

452
453
454
455

class PrometheusStatLogger(StatLoggerBase):
    """PrometheusStatLogger is used LLMEngine to log to Promethus."""
    _metrics_cls = Metrics
456
    _gauge_cls = prometheus_client.Gauge
457
458
459
460

    def __init__(self, local_interval: float, labels: Dict[str, str],
                 max_model_len: int) -> None:
        super().__init__(local_interval)
461
462
        # Prometheus metrics
        self.labels = labels
463
464
        self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
                                         max_model_len=max_model_len)
465

466
467
468
    def _log_gauge(self, gauge, data: Union[int, float]) -> None:
        # Convenience function for logging to gauge.
        gauge.labels(**self.labels).set(data)
469

470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
    def _log_counter(self, counter, data: Union[int, float]) -> None:
        # Convenience function for logging to counter.
        counter.labels(**self.labels).inc(data)

    def _log_counter_labels(self, counter, data: CollectionsCounter,
                            label_key: str) -> None:
        # Convenience function for collection counter of labels.
        for label, count in data.items():
            counter.labels(**{**self.labels, label_key: label}).inc(count)

    def _log_histogram(self, histogram, data: Union[List[int],
                                                    List[float]]) -> None:
        # Convenience function for logging list to histogram.
        for datum in data:
            histogram.labels(**self.labels).observe(datum)
485

486
    def _log_gauge_string(self, gauge, data: Dict[str, str]) -> None:
487
        gauge.labels(**data).set_to_current_time()
488

489
    def _log_prometheus(self, stats: Stats) -> None:
490
491
492
493
494
495
496
497
498
499
500
        # System state data
        self._log_gauge(self.metrics.gauge_scheduler_running,
                        stats.num_running_sys)
        self._log_gauge(self.metrics.gauge_scheduler_swapped,
                        stats.num_swapped_sys)
        self._log_gauge(self.metrics.gauge_scheduler_waiting,
                        stats.num_waiting_sys)
        self._log_gauge(self.metrics.gauge_gpu_cache_usage,
                        stats.gpu_cache_usage_sys)
        self._log_gauge(self.metrics.gauge_cpu_cache_usage,
                        stats.cpu_cache_usage_sys)
501
502
503
504
        self._log_gauge(self.metrics.gauge_cpu_prefix_cache_hit_rate,
                        stats.cpu_prefix_cache_hit_rate)
        self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate,
                        stats.gpu_prefix_cache_hit_rate)
505
506
507
508
509
510
511
512
513
514
515
        # Including max-lora in metric, in future this property of lora
        # config maybe extended to be dynamic.
        lora_info = {
            self.metrics.labelname_running_lora_adapters:
            ",".join(stats.running_lora_adapters),
            self.metrics.labelname_waiting_lora_adapters:
            ",".join(stats.waiting_lora_adapters),
            self.metrics.labelname_max_lora:
            stats.max_lora,
        }
        self._log_gauge_string(self.metrics.gauge_lora_info, lora_info)
516
        # Iteration level data
517
518
        self._log_counter(self.metrics.counter_num_preemption,
                          stats.num_preemption_iter)
519
520
521
522
523
524
525
526
527
528
529
530
531
        self._log_counter(self.metrics.counter_prompt_tokens,
                          stats.num_prompt_tokens_iter)
        self._log_counter(self.metrics.counter_generation_tokens,
                          stats.num_generation_tokens_iter)
        self._log_histogram(self.metrics.histogram_time_to_first_token,
                            stats.time_to_first_tokens_iter)
        self._log_histogram(self.metrics.histogram_time_per_output_token,
                            stats.time_per_output_tokens_iter)

        # Request level data
        # Latency
        self._log_histogram(self.metrics.histogram_e2e_time_request,
                            stats.time_e2e_requests)
532
533
534
535
536
537
        self._log_histogram(self.metrics.histogram_time_in_queue_request,
                            stats.time_in_queue_requests)
        self._log_histogram(self.metrics.histogram_model_forward_time_request,
                            stats.model_forward_time_requests)
        self._log_histogram(self.metrics.histogram_model_execute_time_request,
                            stats.model_execute_time_requests)
538
539
540
541
542
543
544
545
546
547
548
549
550
        # Metadata
        finished_reason_counter = CollectionsCounter(
            stats.finished_reason_requests)
        self._log_counter_labels(self.metrics.counter_request_success,
                                 finished_reason_counter,
                                 Metrics.labelname_finish_reason)
        self._log_histogram(self.metrics.histogram_num_prompt_tokens_request,
                            stats.num_prompt_tokens_requests)
        self._log_histogram(
            self.metrics.histogram_num_generation_tokens_request,
            stats.num_generation_tokens_requests)
        self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)

551
552
553
    def _log_prometheus_interval(self, prompt_throughput: float,
                                 generation_throughput: float) -> None:
        # Logs metrics to prometheus that are computed every logging_interval.
554
555
556
557
558
559
        # Support legacy gauge metrics that make throughput calculations on
        # the vLLM side. Moving forward, we should use counters like
        # counter_prompt_tokens, counter_generation_tokens
        # Which log raw data and calculate summaries using rate() on the
        # grafana/prometheus side. See
        # https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666
560
561
562
563
        self.metrics.gauge_avg_prompt_throughput.labels(
            **self.labels).set(prompt_throughput)
        self.metrics.gauge_avg_generation_throughput.labels(
            **self.labels).set(generation_throughput)
564

565
566
    def log(self, stats: Stats):
        """Logs to prometheus and tracked stats every iteration."""
567
568
569
570
        # Log to prometheus.
        self._log_prometheus(stats)

        # Save tracked stats for token counters.
571
572
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)
573

574
575
576
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

577
        # Log locally every local_interval seconds.
578
579
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
580
581
            # Compute summary metrics for tracked stats (and log them
            # to promethus if applicable).
582
583
584
585
586
587
588
589
            prompt_throughput = get_throughput(self.num_prompt_tokens,
                                               now=stats.now,
                                               last_log=self.last_local_log)
            generation_throughput = get_throughput(
                self.num_generation_tokens,
                now=stats.now,
                last_log=self.last_local_log)

590
591
592
            self._log_prometheus_interval(
                prompt_throughput=prompt_throughput,
                generation_throughput=generation_throughput)
593

594
            if self.spec_decode_metrics is not None:
595
596
                self._log_gauge(
                    self.metrics.gauge_spec_decode_draft_acceptance_rate,
597
                    self.spec_decode_metrics.draft_acceptance_rate)
598
                self._log_gauge(self.metrics.gauge_spec_decode_efficiency,
599
                                self.spec_decode_metrics.system_efficiency)
600
601
                self._log_counter(
                    self.metrics.counter_spec_decode_num_accepted_tokens,
602
                    self.spec_decode_metrics.accepted_tokens)
603
604
                self._log_counter(
                    self.metrics.counter_spec_decode_num_draft_tokens,
605
                    self.spec_decode_metrics.draft_tokens)
606
607
                self._log_counter(
                    self.metrics.counter_spec_decode_num_emitted_tokens,
608
609
610
611
612
613
614
                    self.spec_decode_metrics.emitted_tokens)

            # Reset tracked stats for next interval.
            self.num_prompt_tokens = []
            self.num_generation_tokens = []
            self.last_local_log = stats.now
            self.spec_decode_metrics = None
615

616
617
618
619
620
621
622
623
624
625
626
627
628
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        # Info type metrics are syntactic sugar for a gauge permanently set to 1
        # Since prometheus multiprocessing mode does not support Info, emulate
        # info here with a gauge.
        if type == "cache_config":
            metrics_info = obj.metrics_info()
            info_gauge = self._gauge_cls(
                name="vllm:cache_config_info",
                documentation="Information of the LLMEngine CacheConfig",
                labelnames=metrics_info.keys(),
                multiprocess_mode="mostrecent")
            info_gauge.labels(**metrics_info).set(1)

629

630
631
class RayPrometheusStatLogger(PrometheusStatLogger):
    """RayPrometheusStatLogger uses Ray metrics instead."""
632
    _metrics_cls = RayMetrics
633
634
635

    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        return None