metrics.py 31 KB
Newer Older
1
import time
2
3
from typing import TYPE_CHECKING
from typing import Counter as CollectionsCounter
4
from typing import Dict, List, Optional, Type, Union, cast
5
6

import numpy as np
7
import prometheus_client
8

9
from vllm.config import VllmConfig
10
11
from vllm.engine.metrics_types import (StatLoggerBase, Stats,
                                       SupportsMetricsInfo)
12
from vllm.executor.ray_utils import ray
13
from vllm.logger import init_logger
14

15
16
17
18
19
if ray is not None:
    from ray.util import metrics as ray_metrics
else:
    ray_metrics = None

20
21
22
if TYPE_CHECKING:
    from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics

23
24
logger = init_logger(__name__)

25
prometheus_client.disable_created_metrics()
26
27
28
29

# The begin-* and end* here are used by the documentation generator
# to extract the metrics definitions.

30

31
# begin-metrics-definitions
32
class Metrics:
33
34
35
36
37
38
    """
    vLLM uses a multiprocessing-based frontend for the OpenAI server.
    This means that we need to run prometheus_client in multiprocessing mode
    See https://prometheus.github.io/client_python/multiprocess/ for more
    details on limitations.
    """
39

40
    labelname_finish_reason = "finished_reason"
41
42
43
    labelname_waiting_lora_adapters = "waiting_lora_adapters"
    labelname_running_lora_adapters = "running_lora_adapters"
    labelname_max_lora = "max_lora"
44
45
46
    _gauge_cls = prometheus_client.Gauge
    _counter_cls = prometheus_client.Counter
    _histogram_cls = prometheus_client.Histogram
47

48
    def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
49
        # Unregister any existing vLLM collectors (for CI/CD)
50
        self._unregister_vllm_metrics()
51

52
53
        max_model_len = vllm_config.model_config.max_model_len

54
        # System stats
55
        #   Scheduler State
56
        self.gauge_scheduler_running = self._gauge_cls(
57
58
            name="vllm:num_requests_running",
            documentation="Number of requests currently running on GPU.",
59
60
            labelnames=labelnames,
            multiprocess_mode="sum")
61
        self.gauge_scheduler_waiting = self._gauge_cls(
62
63
            name="vllm:num_requests_waiting",
            documentation="Number of requests waiting to be processed.",
64
65
            labelnames=labelnames,
            multiprocess_mode="sum")
66
67
68
69
70
71
72
73
74
75
        self.gauge_lora_info = self._gauge_cls(
            name="vllm:lora_requests_info",
            documentation="Running stats on lora requests.",
            labelnames=[
                self.labelname_running_lora_adapters,
                self.labelname_max_lora,
                self.labelname_waiting_lora_adapters,
            ],
            multiprocess_mode="livemostrecent",
        )
76
        self.gauge_scheduler_swapped = self._gauge_cls(
77
78
            name="vllm:num_requests_swapped",
            documentation="Number of requests swapped to CPU.",
79
80
            labelnames=labelnames,
            multiprocess_mode="sum")
81
        #   KV Cache Usage in %
82
        self.gauge_gpu_cache_usage = self._gauge_cls(
83
84
            name="vllm:gpu_cache_usage_perc",
            documentation="GPU KV-cache usage. 1 means 100 percent usage.",
85
86
            labelnames=labelnames,
            multiprocess_mode="sum")
87
        self.gauge_cpu_cache_usage = self._gauge_cls(
88
89
            name="vllm:cpu_cache_usage_perc",
            documentation="CPU KV-cache usage. 1 means 100 percent usage.",
90
91
            labelnames=labelnames,
            multiprocess_mode="sum")
92
93
94
95
96
97
98
99
100
101
102
        #   Prefix caching block hit rate
        self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls(
            name="vllm:cpu_prefix_cache_hit_rate",
            documentation="CPU prefix cache block hit rate.",
            labelnames=labelnames,
            multiprocess_mode="sum")
        self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls(
            name="vllm:gpu_prefix_cache_hit_rate",
            documentation="GPU prefix cache block hit rate.",
            labelnames=labelnames,
            multiprocess_mode="sum")
103

104
        # Iteration stats
105
        self.counter_num_preemption = self._counter_cls(
106
107
108
            name="vllm:num_preemptions_total",
            documentation="Cumulative number of preemption from the engine.",
            labelnames=labelnames)
109
        self.counter_prompt_tokens = self._counter_cls(
110
111
112
            name="vllm:prompt_tokens_total",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames)
113
        self.counter_generation_tokens = self._counter_cls(
114
115
116
            name="vllm:generation_tokens_total",
            documentation="Number of generation tokens processed.",
            labelnames=labelnames)
harrywu's avatar
harrywu committed
117
118
119
120
        self.counter_tokens = self._counter_cls(
            name="vllm:tokens_total",
            documentation="Number of prefill plus generation tokens processed.",
            labelnames=labelnames)
121
122
        buckets = [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096]
        if not vllm_config.model_config.enforce_eager:
123
124
            buckets = vllm_config.compilation_config.\
                cudagraph_capture_sizes.copy()
125
            buckets.sort()
harrywu's avatar
harrywu committed
126
127
128
129
        self.histogram_iteration_tokens = self._histogram_cls(
            name="vllm:iteration_tokens_total",
            documentation="Histogram of number of tokens per engine_step.",
            labelnames=labelnames,
130
            buckets=buckets)
131
        self.histogram_time_to_first_token = self._histogram_cls(
132
133
134
135
136
137
138
            name="vllm:time_to_first_token_seconds",
            documentation="Histogram of time to first token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
                0.75, 1.0, 2.5, 5.0, 7.5, 10.0
            ])
139
        self.histogram_time_per_output_token = self._histogram_cls(
140
141
142
143
144
145
146
            name="vllm:time_per_output_token_seconds",
            documentation="Histogram of time per output token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
                1.0, 2.5
            ])
147
148
149

        # Request stats
        #   Latency
harrywu's avatar
harrywu committed
150
151
152
153
        request_latency_buckets = [
            0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
            40.0, 50.0, 60.0
        ]
154
        self.histogram_e2e_time_request = self._histogram_cls(
155
156
157
            name="vllm:e2e_request_latency_seconds",
            documentation="Histogram of end to end request latency in seconds.",
            labelnames=labelnames,
harrywu's avatar
harrywu committed
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
            buckets=request_latency_buckets)
        self.histogram_queue_time_request = self._histogram_cls(
            name="vllm:request_queue_time_seconds",
            documentation=
            "Histogram of time spent in WAITING phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_inference_time_request = self._histogram_cls(
            name="vllm:request_inference_time_seconds",
            documentation=
            "Histogram of time spent in RUNNING phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_prefill_time_request = self._histogram_cls(
            name="vllm:request_prefill_time_seconds",
            documentation=
            "Histogram of time spent in PREFILL phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_decode_time_request = self._histogram_cls(
            name="vllm:request_decode_time_seconds",
            documentation=
            "Histogram of time spent in DECODE phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
183
184
185
186
187
        self.histogram_time_in_queue_request = self._histogram_cls(
            name="vllm:time_in_queue_requests",
            documentation=
            "Histogram of time the request spent in the queue in seconds.",
            labelnames=labelnames,
harrywu's avatar
harrywu committed
188
            buckets=request_latency_buckets)
189
190
191
192
193
194
195
196
197
198
199
200
        self.histogram_model_forward_time_request = self._histogram_cls(
            name="vllm:model_forward_time_milliseconds",
            documentation=
            "Histogram of time spent in the model forward pass in ms.",
            labelnames=labelnames,
            buckets=build_1_2_3_5_8_buckets(3000))
        self.histogram_model_execute_time_request = self._histogram_cls(
            name="vllm:model_execute_time_milliseconds",
            documentation=
            "Histogram of time spent in the model execute function in ms.",
            labelnames=labelnames,
            buckets=build_1_2_3_5_8_buckets(3000))
201
        #   Metadata
202
        self.histogram_num_prompt_tokens_request = self._histogram_cls(
203
204
205
206
207
            name="vllm:request_prompt_tokens",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len),
        )
208
        self.histogram_num_generation_tokens_request = \
209
            self._histogram_cls(
210
211
212
213
214
                name="vllm:request_generation_tokens",
                documentation="Number of generation tokens processed.",
                labelnames=labelnames,
                buckets=build_1_2_5_buckets(max_model_len),
            )
harrywu's avatar
harrywu committed
215
216
217
218
219
220
        self.histogram_max_num_generation_tokens_request = self._histogram_cls(
            name="vllm:request_max_num_generation_tokens",
            documentation=
            "Histogram of maximum number of requested generation tokens.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len))
221
        self.histogram_n_request = self._histogram_cls(
222
223
224
225
226
            name="vllm:request_params_n",
            documentation="Histogram of the n request parameter.",
            labelnames=labelnames,
            buckets=[1, 2, 5, 10, 20],
        )
227
228
229
230
231
232
        self.histogram_max_tokens_request = self._histogram_cls(
            name="vllm:request_params_max_tokens",
            documentation="Histogram of the max_tokens request parameter.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len),
        )
233
        self.counter_request_success = self._counter_cls(
234
            name="vllm:request_success_total",
235
236
            documentation="Count of successfully processed requests.",
            labelnames=labelnames + [Metrics.labelname_finish_reason])
237

238
        # Speculatie decoding stats
239
        self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
240
241
            name="vllm:spec_decode_draft_acceptance_rate",
            documentation="Speulative token acceptance rate.",
242
243
            labelnames=labelnames,
            multiprocess_mode="sum")
244
        self.gauge_spec_decode_efficiency = self._gauge_cls(
245
246
            name="vllm:spec_decode_efficiency",
            documentation="Speculative decoding system efficiency.",
247
248
            labelnames=labelnames,
            multiprocess_mode="sum")
249
250
251
252
253
        self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_accepted_tokens_total",
            documentation="Number of accepted tokens.",
            labelnames=labelnames))
        self.counter_spec_decode_num_draft_tokens = self._counter_cls(
254
255
256
            name="vllm:spec_decode_num_draft_tokens_total",
            documentation="Number of draft tokens.",
            labelnames=labelnames)
257
258
259
260
        self.counter_spec_decode_num_emitted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_emitted_tokens_total",
            documentation="Number of emitted tokens.",
            labelnames=labelnames))
261

262
        # Deprecated in favor of vllm:prompt_tokens_total
263
        self.gauge_avg_prompt_throughput = self._gauge_cls(
264
265
266
            name="vllm:avg_prompt_throughput_toks_per_s",
            documentation="Average prefill throughput in tokens/s.",
            labelnames=labelnames,
267
            multiprocess_mode="sum",
268
        )
269
        # Deprecated in favor of vllm:generation_tokens_total
270
        self.gauge_avg_generation_throughput = self._gauge_cls(
271
272
273
            name="vllm:avg_generation_throughput_toks_per_s",
            documentation="Average generation throughput in tokens/s.",
            labelnames=labelnames,
274
            multiprocess_mode="sum",
275
276
        )

277
278

# end-metrics-definitions
279

280
    def _unregister_vllm_metrics(self) -> None:
281
        for collector in list(prometheus_client.REGISTRY._collector_to_names):
282
            if hasattr(collector, "_name") and "vllm" in collector._name:
283
284
285
286
287
288
289
290
291
292
                prometheus_client.REGISTRY.unregister(collector)


class _RayGaugeWrapper:
    """Wraps around ray.util.metrics.Gauge to provide same API as
    prometheus_client.Gauge"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
293
294
295
                 labelnames: Optional[List[str]] = None,
                 multiprocess_mode: str = ""):
        del multiprocess_mode
296
297
298
299
300
301
302
303
304
305
306
307
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._gauge = ray_metrics.Gauge(name=name,
                                        description=documentation,
                                        tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._gauge.set_default_tags(labels)
        return self

    def set(self, value: Union[int, float]):
        return self._gauge.set(value)

308
309
310
311
    def set_to_current_time(self):
        # ray metrics doesn't have set_to_current time, https://docs.ray.io/en/latest/_modules/ray/util/metrics.html
        return self._gauge.set(time.time())

312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345

class _RayCounterWrapper:
    """Wraps around ray.util.metrics.Counter to provide same API as
    prometheus_client.Counter"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._counter = ray_metrics.Counter(name=name,
                                            description=documentation,
                                            tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._counter.set_default_tags(labels)
        return self

    def inc(self, value: Union[int, float] = 1.0):
        if value == 0:
            return
        return self._counter.inc(value)


class _RayHistogramWrapper:
    """Wraps around ray.util.metrics.Histogram to provide same API as
    prometheus_client.Histogram"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None,
                 buckets: Optional[List[float]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
346
        boundaries = buckets if buckets else []
347
348
349
        self._histogram = ray_metrics.Histogram(name=name,
                                                description=documentation,
                                                tag_keys=labelnames_tuple,
350
                                                boundaries=boundaries)
351
352
353
354
355
356
357

    def labels(self, **labels):
        self._histogram.set_default_tags(labels)
        return self

    def observe(self, value: Union[int, float]):
        return self._histogram.observe(value)
358
359
360
361
362
363
364


class RayMetrics(Metrics):
    """
    RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
    Provides the same metrics as Metrics but uses Ray's util.metrics library.
    """
365
366
367
368
369
370
    _gauge_cls: Type[prometheus_client.Gauge] = cast(
        Type[prometheus_client.Gauge], _RayGaugeWrapper)
    _counter_cls: Type[prometheus_client.Counter] = cast(
        Type[prometheus_client.Counter], _RayCounterWrapper)
    _histogram_cls: Type[prometheus_client.Histogram] = cast(
        Type[prometheus_client.Histogram], _RayHistogramWrapper)
371

372
    def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
373
374
        if ray_metrics is None:
            raise ImportError("RayMetrics requires Ray to be installed.")
375
        super().__init__(labelnames, vllm_config)
376
377
378
379
380

    def _unregister_vllm_metrics(self) -> None:
        # No-op on purpose
        pass

381

382
def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:
383
    """
384
385
    Builds a list of buckets with increasing powers of 10 multiplied by
    mantissa values until the value exceeds the specified maximum.
386
387
388

    """
    exponent = 0
389
    buckets: List[int] = []
390
391
392
393
394
395
396
397
398
399
    while True:
        for m in mantissa_lst:
            value = m * 10**exponent
            if value <= max_value:
                buckets.append(value)
            else:
                return buckets
        exponent += 1


400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
def build_1_2_5_buckets(max_value: int) -> List[int]:
    """
    Example:
    >>> build_1_2_5_buckets(100)
    [1, 2, 5, 10, 20, 50, 100]
    """
    return build_buckets([1, 2, 5], max_value)


def build_1_2_3_5_8_buckets(max_value: int) -> List[int]:
    """
    Example:
    >>> build_1_2_3_5_8_buckets(100)
    [1, 2, 3, 5, 8, 10, 20, 30, 50, 80, 100]
    """
    return build_buckets([1, 2, 3, 5, 8], max_value)


418
419
420
421
422
423
424
425
426
def local_interval_elapsed(now: float, last_log: float,
                           local_interval: float) -> bool:
    elapsed_time = now - last_log
    return elapsed_time > local_interval


def get_throughput(tracked_stats: List[int], now: float,
                   last_log: float) -> float:
    return float(np.sum(tracked_stats) / (now - last_log))
427
428


429
430
431
class LoggingStatLogger(StatLoggerBase):
    """LoggingStatLogger is used in LLMEngine to log to Stdout."""

432
433
    def __init__(self, local_interval: float, vllm_config: VllmConfig) -> None:
        super().__init__(local_interval, vllm_config)
434
435
436
        self.last_prompt_throughput: Optional[float] = None
        self.last_generation_throughput: Optional[float] = None

437
438
439
440
441
442
443
444
    def log(self, stats: Stats) -> None:
        """Called by LLMEngine.
           Logs to Stdout every self.local_interval seconds."""

        # Save tracked stats for token counters.
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)

445
446
447
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

448
449
450
451
452
453
454
455
456
457
458
459
460
        # Log locally every local_interval seconds.
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
            # Compute summary metrics for tracked stats (and log them
            # to promethus if applicable).
            prompt_throughput = get_throughput(self.num_prompt_tokens,
                                               now=stats.now,
                                               last_log=self.last_local_log)
            generation_throughput = get_throughput(
                self.num_generation_tokens,
                now=stats.now,
                last_log=self.last_local_log)

461
462
463
464
465
466
467
468
            log_fn = logger.info
            if not any((prompt_throughput, generation_throughput,
                        self.last_prompt_throughput,
                        self.last_generation_throughput)):
                # Avoid log noise on an idle production system
                log_fn = logger.debug

            log_fn(
469
470
471
472
473
474
475
476
477
478
479
480
481
                "Avg prompt throughput: %.1f tokens/s, "
                "Avg generation throughput: %.1f tokens/s, "
                "Running: %d reqs, Swapped: %d reqs, "
                "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
                "CPU KV cache usage: %.1f%%.",
                prompt_throughput,
                generation_throughput,
                stats.num_running_sys,
                stats.num_swapped_sys,
                stats.num_waiting_sys,
                stats.gpu_cache_usage_sys * 100,
                stats.cpu_cache_usage_sys * 100,
            )
482
483
            if (stats.cpu_prefix_cache_hit_rate >= 0
                    or stats.gpu_prefix_cache_hit_rate >= 0):
484
                log_fn(
485
486
487
488
                    "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
                    stats.gpu_prefix_cache_hit_rate * 100,
                    stats.cpu_prefix_cache_hit_rate * 100,
                )
489
            if self.spec_decode_metrics is not None:
490
                log_fn(
491
492
493
                    self._format_spec_decode_metrics_str(
                        self.spec_decode_metrics))

494
495
496
497
498
499
500
501
502
503
            self._reset(stats, prompt_throughput, generation_throughput)

    def _reset(self, stats, prompt_throughput, generation_throughput) -> None:
        # Reset tracked stats for next interval.
        self.num_prompt_tokens = []
        self.num_generation_tokens = []
        self.last_local_log = stats.now
        self.spec_decode_metrics = None
        self.last_prompt_throughput = prompt_throughput
        self.last_generation_throughput = generation_throughput
504
505
506
507
508
509
510
511
512

    def _format_spec_decode_metrics_str(
            self, metrics: "SpecDecodeWorkerMetrics") -> str:

        return ("Speculative metrics: "
                f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
                f"System efficiency: {metrics.system_efficiency:.3f}, "
                f"Number of speculative tokens: {metrics.num_spec_tokens}, "
                f"Number of accepted tokens: {metrics.accepted_tokens}, "
513
514
                f"Number of draft tokens: {metrics.draft_tokens}, "
                f"Number of emitted tokens: {metrics.emitted_tokens}.")
515

516
517
518
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        raise NotImplementedError

519
520
521
522

class PrometheusStatLogger(StatLoggerBase):
    """PrometheusStatLogger is used LLMEngine to log to Promethus."""
    _metrics_cls = Metrics
523
    _gauge_cls = prometheus_client.Gauge
524
525

    def __init__(self, local_interval: float, labels: Dict[str, str],
526
527
                 vllm_config: VllmConfig) -> None:
        super().__init__(local_interval, vllm_config)
528
529
        # Prometheus metrics
        self.labels = labels
530
        self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
531
                                         vllm_config=vllm_config)
532

533
534
535
    def _log_gauge(self, gauge, data: Union[int, float]) -> None:
        # Convenience function for logging to gauge.
        gauge.labels(**self.labels).set(data)
536

537
538
    def _log_counter(self, counter, data: Union[int, float]) -> None:
        # Convenience function for logging to counter.
539
540
541
542
543
        # Prevent ValueError from negative increment
        if data < 0:
            logger.warning("Skipping negative increment of %g to %s", data,
                           counter)
            return
544
545
546
547
548
549
550
551
552
553
554
555
556
        counter.labels(**self.labels).inc(data)

    def _log_counter_labels(self, counter, data: CollectionsCounter,
                            label_key: str) -> None:
        # Convenience function for collection counter of labels.
        for label, count in data.items():
            counter.labels(**{**self.labels, label_key: label}).inc(count)

    def _log_histogram(self, histogram, data: Union[List[int],
                                                    List[float]]) -> None:
        # Convenience function for logging list to histogram.
        for datum in data:
            histogram.labels(**self.labels).observe(datum)
557

558
    def _log_gauge_string(self, gauge, data: Dict[str, str]) -> None:
559
        gauge.labels(**data).set_to_current_time()
560

561
    def _log_prometheus(self, stats: Stats) -> None:
562
563
564
565
566
567
568
569
570
571
572
        # System state data
        self._log_gauge(self.metrics.gauge_scheduler_running,
                        stats.num_running_sys)
        self._log_gauge(self.metrics.gauge_scheduler_swapped,
                        stats.num_swapped_sys)
        self._log_gauge(self.metrics.gauge_scheduler_waiting,
                        stats.num_waiting_sys)
        self._log_gauge(self.metrics.gauge_gpu_cache_usage,
                        stats.gpu_cache_usage_sys)
        self._log_gauge(self.metrics.gauge_cpu_cache_usage,
                        stats.cpu_cache_usage_sys)
573
574
575
576
        self._log_gauge(self.metrics.gauge_cpu_prefix_cache_hit_rate,
                        stats.cpu_prefix_cache_hit_rate)
        self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate,
                        stats.gpu_prefix_cache_hit_rate)
577
578
579
580
581
582
583
584
585
586
587
        # Including max-lora in metric, in future this property of lora
        # config maybe extended to be dynamic.
        lora_info = {
            self.metrics.labelname_running_lora_adapters:
            ",".join(stats.running_lora_adapters),
            self.metrics.labelname_waiting_lora_adapters:
            ",".join(stats.waiting_lora_adapters),
            self.metrics.labelname_max_lora:
            stats.max_lora,
        }
        self._log_gauge_string(self.metrics.gauge_lora_info, lora_info)
588
        # Iteration level data
589
590
        self._log_counter(self.metrics.counter_num_preemption,
                          stats.num_preemption_iter)
591
592
593
594
        self._log_counter(self.metrics.counter_prompt_tokens,
                          stats.num_prompt_tokens_iter)
        self._log_counter(self.metrics.counter_generation_tokens,
                          stats.num_generation_tokens_iter)
harrywu's avatar
harrywu committed
595
596
        self._log_histogram(self.metrics.histogram_iteration_tokens,
                            [stats.num_tokens_iter])
597
598
599
600
601
602
603
604
605
        self._log_histogram(self.metrics.histogram_time_to_first_token,
                            stats.time_to_first_tokens_iter)
        self._log_histogram(self.metrics.histogram_time_per_output_token,
                            stats.time_per_output_tokens_iter)

        # Request level data
        # Latency
        self._log_histogram(self.metrics.histogram_e2e_time_request,
                            stats.time_e2e_requests)
harrywu's avatar
harrywu committed
606
607
608
609
610
        self._log_histogram(self.metrics.histogram_queue_time_request,
                            stats.time_queue_requests)
        self._log_histogram(self.metrics.histogram_inference_time_request,
                            stats.time_inference_requests)
        self._log_histogram(self.metrics.histogram_prefill_time_request,
611
612
                            stats.time_prefill_requests)
        self._log_histogram(self.metrics.histogram_decode_time_request,
harrywu's avatar
harrywu committed
613
                            stats.time_decode_requests)
614
615
616
617
618
619
        self._log_histogram(self.metrics.histogram_time_in_queue_request,
                            stats.time_in_queue_requests)
        self._log_histogram(self.metrics.histogram_model_forward_time_request,
                            stats.model_forward_time_requests)
        self._log_histogram(self.metrics.histogram_model_execute_time_request,
                            stats.model_execute_time_requests)
620
621
622
623
624
625
626
627
628
629
630
631
        # Metadata
        finished_reason_counter = CollectionsCounter(
            stats.finished_reason_requests)
        self._log_counter_labels(self.metrics.counter_request_success,
                                 finished_reason_counter,
                                 Metrics.labelname_finish_reason)
        self._log_histogram(self.metrics.histogram_num_prompt_tokens_request,
                            stats.num_prompt_tokens_requests)
        self._log_histogram(
            self.metrics.histogram_num_generation_tokens_request,
            stats.num_generation_tokens_requests)
        self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
harrywu's avatar
harrywu committed
632
633
634
        self._log_histogram(
            self.metrics.histogram_max_num_generation_tokens_request,
            stats.max_num_generation_tokens_requests)
635
636
        self._log_histogram(self.metrics.histogram_max_tokens_request,
                            stats.max_tokens_requests)
637

638
639
640
    def _log_prometheus_interval(self, prompt_throughput: float,
                                 generation_throughput: float) -> None:
        # Logs metrics to prometheus that are computed every logging_interval.
641
642
643
644
645
646
        # Support legacy gauge metrics that make throughput calculations on
        # the vLLM side. Moving forward, we should use counters like
        # counter_prompt_tokens, counter_generation_tokens
        # Which log raw data and calculate summaries using rate() on the
        # grafana/prometheus side. See
        # https://github.com/vllm-project/vllm/pull/2316#discussion_r1464204666
647
648
649
650
        self.metrics.gauge_avg_prompt_throughput.labels(
            **self.labels).set(prompt_throughput)
        self.metrics.gauge_avg_generation_throughput.labels(
            **self.labels).set(generation_throughput)
651

652
653
    def log(self, stats: Stats):
        """Logs to prometheus and tracked stats every iteration."""
654
655
656
657
        # Log to prometheus.
        self._log_prometheus(stats)

        # Save tracked stats for token counters.
658
659
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)
660

661
662
663
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

664
        # Log locally every local_interval seconds.
665
666
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
667
668
            # Compute summary metrics for tracked stats (and log them
            # to promethus if applicable).
669
670
671
672
673
674
675
676
            prompt_throughput = get_throughput(self.num_prompt_tokens,
                                               now=stats.now,
                                               last_log=self.last_local_log)
            generation_throughput = get_throughput(
                self.num_generation_tokens,
                now=stats.now,
                last_log=self.last_local_log)

677
678
679
            self._log_prometheus_interval(
                prompt_throughput=prompt_throughput,
                generation_throughput=generation_throughput)
680

681
            if self.spec_decode_metrics is not None:
682
683
                self._log_gauge(
                    self.metrics.gauge_spec_decode_draft_acceptance_rate,
684
                    self.spec_decode_metrics.draft_acceptance_rate)
685
                self._log_gauge(self.metrics.gauge_spec_decode_efficiency,
686
                                self.spec_decode_metrics.system_efficiency)
687
688
                self._log_counter(
                    self.metrics.counter_spec_decode_num_accepted_tokens,
689
                    self.spec_decode_metrics.accepted_tokens)
690
691
                self._log_counter(
                    self.metrics.counter_spec_decode_num_draft_tokens,
692
                    self.spec_decode_metrics.draft_tokens)
693
694
                self._log_counter(
                    self.metrics.counter_spec_decode_num_emitted_tokens,
695
696
697
698
699
700
701
                    self.spec_decode_metrics.emitted_tokens)

            # Reset tracked stats for next interval.
            self.num_prompt_tokens = []
            self.num_generation_tokens = []
            self.last_local_log = stats.now
            self.spec_decode_metrics = None
702

703
704
705
706
707
708
709
710
711
712
713
714
715
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        # Info type metrics are syntactic sugar for a gauge permanently set to 1
        # Since prometheus multiprocessing mode does not support Info, emulate
        # info here with a gauge.
        if type == "cache_config":
            metrics_info = obj.metrics_info()
            info_gauge = self._gauge_cls(
                name="vllm:cache_config_info",
                documentation="Information of the LLMEngine CacheConfig",
                labelnames=metrics_info.keys(),
                multiprocess_mode="mostrecent")
            info_gauge.labels(**metrics_info).set(1)

716

717
718
class RayPrometheusStatLogger(PrometheusStatLogger):
    """RayPrometheusStatLogger uses Ray metrics instead."""
719
    _metrics_cls = RayMetrics
720
721
722

    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        return None