metrics.py 30.7 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import time
4
5
from typing import TYPE_CHECKING
from typing import Counter as CollectionsCounter
6
from typing import Dict, List, Optional, Type, Union, cast
7
8

import numpy as np
9
import prometheus_client
10

11
12
from vllm.config import SupportsMetricsInfo, VllmConfig
from vllm.engine.metrics_types import StatLoggerBase, Stats
13
from vllm.executor.ray_utils import ray
14
from vllm.logger import init_logger
15

16
17
18
19
20
if ray is not None:
    from ray.util import metrics as ray_metrics
else:
    ray_metrics = None

21
22
23
if TYPE_CHECKING:
    from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics

24
25
logger = init_logger(__name__)

26
prometheus_client.disable_created_metrics()
27
28
29
30

# The begin-* and end* here are used by the documentation generator
# to extract the metrics definitions.

31

32
# begin-metrics-definitions
33
class Metrics:
34
35
36
37
38
39
    """
    vLLM uses a multiprocessing-based frontend for the OpenAI server.
    This means that we need to run prometheus_client in multiprocessing mode
    See https://prometheus.github.io/client_python/multiprocess/ for more
    details on limitations.
    """
40

41
    labelname_finish_reason = "finished_reason"
42
43
44
    labelname_waiting_lora_adapters = "waiting_lora_adapters"
    labelname_running_lora_adapters = "running_lora_adapters"
    labelname_max_lora = "max_lora"
45
46
47
    _gauge_cls = prometheus_client.Gauge
    _counter_cls = prometheus_client.Counter
    _histogram_cls = prometheus_client.Histogram
48

49
    def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
50
        # Unregister any existing vLLM collectors (for CI/CD)
51
        self._unregister_vllm_metrics()
52

53
54
        max_model_len = vllm_config.model_config.max_model_len

55
56
57
58
59
        # Use this flag to hide metrics that were deprecated in
        # a previous release and which will be removed future
        self.show_hidden_metrics = \
            vllm_config.observability_config.show_hidden_metrics

60
        # System stats
61
        #   Scheduler State
62
        self.gauge_scheduler_running = self._gauge_cls(
63
64
            name="vllm:num_requests_running",
            documentation="Number of requests currently running on GPU.",
65
66
            labelnames=labelnames,
            multiprocess_mode="sum")
67
        self.gauge_scheduler_waiting = self._gauge_cls(
68
69
            name="vllm:num_requests_waiting",
            documentation="Number of requests waiting to be processed.",
70
71
            labelnames=labelnames,
            multiprocess_mode="sum")
72
73
74
75
76
77
78
79
80
81
        self.gauge_lora_info = self._gauge_cls(
            name="vllm:lora_requests_info",
            documentation="Running stats on lora requests.",
            labelnames=[
                self.labelname_running_lora_adapters,
                self.labelname_max_lora,
                self.labelname_waiting_lora_adapters,
            ],
            multiprocess_mode="livemostrecent",
        )
82
83

        # Deprecated in 0.8 - KV cache offloading is not used in V1
84
85
86
87
88
89
90
91
92
        # Hidden in 0.9, due to be removed in 0.10
        if self.show_hidden_metrics:
            self.gauge_scheduler_swapped = self._gauge_cls(
                name="vllm:num_requests_swapped",
                documentation=(
                    "Number of requests swapped to CPU. "
                    "DEPRECATED: KV cache offloading is not used in V1"),
                labelnames=labelnames,
                multiprocess_mode="sum")
93

94
        #   KV Cache Usage in %
95
        self.gauge_gpu_cache_usage = self._gauge_cls(
96
97
            name="vllm:gpu_cache_usage_perc",
            documentation="GPU KV-cache usage. 1 means 100 percent usage.",
98
99
            labelnames=labelnames,
            multiprocess_mode="sum")
100
101

        # Deprecated in 0.8 - KV cache offloading is not used in V1
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
        # Hidden in 0.9, due to be removed in 0.10
        if self.show_hidden_metrics:
            self.gauge_cpu_cache_usage = self._gauge_cls(
                name="vllm:cpu_cache_usage_perc",
                documentation=(
                    "CPU KV-cache usage. 1 means 100 percent usage. "
                    "DEPRECATED: KV cache offloading is not used in V1"),
                labelnames=labelnames,
                multiprocess_mode="sum")
            self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls(
                name="vllm:cpu_prefix_cache_hit_rate",
                documentation=(
                    "CPU prefix cache block hit rate. "
                    "DEPRECATED: KV cache offloading is not used in V1"),
                labelnames=labelnames,
                multiprocess_mode="sum")
118
119

        # Deprecated in 0.8 - replaced by queries+hits counters in V1
120
121
122
123
124
125
126
127
128
        # Hidden in 0.9, due to be removed in 0.10
        if self.show_hidden_metrics:
            self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls(
                name="vllm:gpu_prefix_cache_hit_rate",
                documentation=("GPU prefix cache block hit rate. "
                               "DEPRECATED: use vllm:gpu_prefix_cache_queries "
                               "and vllm:gpu_prefix_cache_queries in V1"),
                labelnames=labelnames,
                multiprocess_mode="sum")
129

130
        # Iteration stats
131
        self.counter_num_preemption = self._counter_cls(
132
133
134
            name="vllm:num_preemptions_total",
            documentation="Cumulative number of preemption from the engine.",
            labelnames=labelnames)
135
        self.counter_prompt_tokens = self._counter_cls(
136
137
138
            name="vllm:prompt_tokens_total",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames)
139
        self.counter_generation_tokens = self._counter_cls(
140
141
142
            name="vllm:generation_tokens_total",
            documentation="Number of generation tokens processed.",
            labelnames=labelnames)
143
144
        buckets = [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096]
        if not vllm_config.model_config.enforce_eager:
145
146
            buckets = vllm_config.compilation_config.\
                cudagraph_capture_sizes.copy()
147
            buckets.sort()
harrywu's avatar
harrywu committed
148
149
150
151
        self.histogram_iteration_tokens = self._histogram_cls(
            name="vllm:iteration_tokens_total",
            documentation="Histogram of number of tokens per engine_step.",
            labelnames=labelnames,
152
            buckets=buckets)
153
        self.histogram_time_to_first_token = self._histogram_cls(
154
155
156
157
158
159
160
            name="vllm:time_to_first_token_seconds",
            documentation="Histogram of time to first token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
                0.75, 1.0, 2.5, 5.0, 7.5, 10.0
            ])
161
        self.histogram_time_per_output_token = self._histogram_cls(
162
163
164
165
166
167
168
            name="vllm:time_per_output_token_seconds",
            documentation="Histogram of time per output token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
                1.0, 2.5
            ])
169
170
171

        # Request stats
        #   Latency
harrywu's avatar
harrywu committed
172
173
174
175
        request_latency_buckets = [
            0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
            40.0, 50.0, 60.0
        ]
176
        self.histogram_e2e_time_request = self._histogram_cls(
177
178
179
            name="vllm:e2e_request_latency_seconds",
            documentation="Histogram of end to end request latency in seconds.",
            labelnames=labelnames,
harrywu's avatar
harrywu committed
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
            buckets=request_latency_buckets)
        self.histogram_queue_time_request = self._histogram_cls(
            name="vllm:request_queue_time_seconds",
            documentation=
            "Histogram of time spent in WAITING phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_inference_time_request = self._histogram_cls(
            name="vllm:request_inference_time_seconds",
            documentation=
            "Histogram of time spent in RUNNING phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_prefill_time_request = self._histogram_cls(
            name="vllm:request_prefill_time_seconds",
            documentation=
            "Histogram of time spent in PREFILL phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_decode_time_request = self._histogram_cls(
            name="vllm:request_decode_time_seconds",
            documentation=
            "Histogram of time spent in DECODE phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
205
        # Deprecated in 0.8 - duplicates vllm:request_queue_time_seconds:
206
207
208
209
210
211
212
213
214
        # Hidden in 0.9, due to be removed in 0.10
        if self.show_hidden_metrics:
            self.histogram_time_in_queue_request = self._histogram_cls(
                name="vllm:time_in_queue_requests",
                documentation=
                ("Histogram of time the request spent in the queue in seconds. "
                 "DEPRECATED: use vllm:request_queue_time_seconds instead."),
                labelnames=labelnames,
                buckets=request_latency_buckets)
215
216

        # Deprecated in 0.8 - use prefill/decode/inference time metrics
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
        # Hidden in 0.9, due to be removed in 0.10
        if self.show_hidden_metrics:
            self.histogram_model_forward_time_request = self._histogram_cls(
                name="vllm:model_forward_time_milliseconds",
                documentation=
                ("Histogram of time spent in the model forward pass in ms. "
                 "DEPRECATED: use prefill/decode/inference time metrics instead"
                 ),
                labelnames=labelnames,
                buckets=build_1_2_3_5_8_buckets(3000))
            self.histogram_model_execute_time_request = self._histogram_cls(
                name="vllm:model_execute_time_milliseconds",
                documentation=
                ("Histogram of time spent in the model execute function in ms."
                 "DEPRECATED: use prefill/decode/inference time metrics instead"
                 ),
                labelnames=labelnames,
                buckets=build_1_2_3_5_8_buckets(3000))
235

236
        #   Metadata
237
        self.histogram_num_prompt_tokens_request = self._histogram_cls(
238
239
240
241
242
            name="vllm:request_prompt_tokens",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len),
        )
243
        self.histogram_num_generation_tokens_request = \
244
            self._histogram_cls(
245
246
247
248
249
                name="vllm:request_generation_tokens",
                documentation="Number of generation tokens processed.",
                labelnames=labelnames,
                buckets=build_1_2_5_buckets(max_model_len),
            )
harrywu's avatar
harrywu committed
250
251
252
253
254
255
        self.histogram_max_num_generation_tokens_request = self._histogram_cls(
            name="vllm:request_max_num_generation_tokens",
            documentation=
            "Histogram of maximum number of requested generation tokens.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len))
256
        self.histogram_n_request = self._histogram_cls(
257
258
259
260
261
            name="vllm:request_params_n",
            documentation="Histogram of the n request parameter.",
            labelnames=labelnames,
            buckets=[1, 2, 5, 10, 20],
        )
262
263
264
265
266
267
        self.histogram_max_tokens_request = self._histogram_cls(
            name="vllm:request_params_max_tokens",
            documentation="Histogram of the max_tokens request parameter.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len),
        )
268
        self.counter_request_success = self._counter_cls(
269
            name="vllm:request_success_total",
270
271
            documentation="Count of successfully processed requests.",
            labelnames=labelnames + [Metrics.labelname_finish_reason])
272

273
        # Speculative decoding stats
274
        self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
275
276
            name="vllm:spec_decode_draft_acceptance_rate",
            documentation="Speulative token acceptance rate.",
277
278
            labelnames=labelnames,
            multiprocess_mode="sum")
279
        self.gauge_spec_decode_efficiency = self._gauge_cls(
280
281
            name="vllm:spec_decode_efficiency",
            documentation="Speculative decoding system efficiency.",
282
283
            labelnames=labelnames,
            multiprocess_mode="sum")
284
285
286
287
288
        self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_accepted_tokens_total",
            documentation="Number of accepted tokens.",
            labelnames=labelnames))
        self.counter_spec_decode_num_draft_tokens = self._counter_cls(
289
290
291
            name="vllm:spec_decode_num_draft_tokens_total",
            documentation="Number of draft tokens.",
            labelnames=labelnames)
292
293
294
295
        self.counter_spec_decode_num_emitted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_emitted_tokens_total",
            documentation="Number of emitted tokens.",
            labelnames=labelnames))
296

297
298

# end-metrics-definitions
299

300
    def _unregister_vllm_metrics(self) -> None:
301
        for collector in list(prometheus_client.REGISTRY._collector_to_names):
302
            if hasattr(collector, "_name") and "vllm" in collector._name:
303
304
305
306
307
308
309
310
311
312
                prometheus_client.REGISTRY.unregister(collector)


class _RayGaugeWrapper:
    """Wraps around ray.util.metrics.Gauge to provide same API as
    prometheus_client.Gauge"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
313
314
315
                 labelnames: Optional[List[str]] = None,
                 multiprocess_mode: str = ""):
        del multiprocess_mode
316
317
318
319
320
321
322
323
324
325
326
327
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._gauge = ray_metrics.Gauge(name=name,
                                        description=documentation,
                                        tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._gauge.set_default_tags(labels)
        return self

    def set(self, value: Union[int, float]):
        return self._gauge.set(value)

328
329
330
331
    def set_to_current_time(self):
        # ray metrics doesn't have set_to_current time, https://docs.ray.io/en/latest/_modules/ray/util/metrics.html
        return self._gauge.set(time.time())

332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365

class _RayCounterWrapper:
    """Wraps around ray.util.metrics.Counter to provide same API as
    prometheus_client.Counter"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._counter = ray_metrics.Counter(name=name,
                                            description=documentation,
                                            tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._counter.set_default_tags(labels)
        return self

    def inc(self, value: Union[int, float] = 1.0):
        if value == 0:
            return
        return self._counter.inc(value)


class _RayHistogramWrapper:
    """Wraps around ray.util.metrics.Histogram to provide same API as
    prometheus_client.Histogram"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None,
                 buckets: Optional[List[float]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
366
        boundaries = buckets if buckets else []
367
368
369
        self._histogram = ray_metrics.Histogram(name=name,
                                                description=documentation,
                                                tag_keys=labelnames_tuple,
370
                                                boundaries=boundaries)
371
372
373
374
375
376
377

    def labels(self, **labels):
        self._histogram.set_default_tags(labels)
        return self

    def observe(self, value: Union[int, float]):
        return self._histogram.observe(value)
378
379
380
381
382
383
384


class RayMetrics(Metrics):
    """
    RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
    Provides the same metrics as Metrics but uses Ray's util.metrics library.
    """
385
386
387
388
389
390
    _gauge_cls: Type[prometheus_client.Gauge] = cast(
        Type[prometheus_client.Gauge], _RayGaugeWrapper)
    _counter_cls: Type[prometheus_client.Counter] = cast(
        Type[prometheus_client.Counter], _RayCounterWrapper)
    _histogram_cls: Type[prometheus_client.Histogram] = cast(
        Type[prometheus_client.Histogram], _RayHistogramWrapper)
391

392
    def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
393
394
        if ray_metrics is None:
            raise ImportError("RayMetrics requires Ray to be installed.")
395
        super().__init__(labelnames, vllm_config)
396
397
398
399
400

    def _unregister_vllm_metrics(self) -> None:
        # No-op on purpose
        pass

401

402
def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:
403
    """
404
405
    Builds a list of buckets with increasing powers of 10 multiplied by
    mantissa values until the value exceeds the specified maximum.
406
407
408

    """
    exponent = 0
409
    buckets: List[int] = []
410
411
412
413
414
415
416
417
418
419
    while True:
        for m in mantissa_lst:
            value = m * 10**exponent
            if value <= max_value:
                buckets.append(value)
            else:
                return buckets
        exponent += 1


420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
def build_1_2_5_buckets(max_value: int) -> List[int]:
    """
    Example:
    >>> build_1_2_5_buckets(100)
    [1, 2, 5, 10, 20, 50, 100]
    """
    return build_buckets([1, 2, 5], max_value)


def build_1_2_3_5_8_buckets(max_value: int) -> List[int]:
    """
    Example:
    >>> build_1_2_3_5_8_buckets(100)
    [1, 2, 3, 5, 8, 10, 20, 30, 50, 80, 100]
    """
    return build_buckets([1, 2, 3, 5, 8], max_value)


438
439
440
441
442
443
444
445
446
def local_interval_elapsed(now: float, last_log: float,
                           local_interval: float) -> bool:
    elapsed_time = now - last_log
    return elapsed_time > local_interval


def get_throughput(tracked_stats: List[int], now: float,
                   last_log: float) -> float:
    return float(np.sum(tracked_stats) / (now - last_log))
447
448


449
450
451
class LoggingStatLogger(StatLoggerBase):
    """LoggingStatLogger is used in LLMEngine to log to Stdout."""

452
453
    def __init__(self, local_interval: float, vllm_config: VllmConfig) -> None:
        super().__init__(local_interval, vllm_config)
454
455
456
        self.last_prompt_throughput: Optional[float] = None
        self.last_generation_throughput: Optional[float] = None

457
458
459
460
461
462
463
464
    def log(self, stats: Stats) -> None:
        """Called by LLMEngine.
           Logs to Stdout every self.local_interval seconds."""

        # Save tracked stats for token counters.
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)

465
466
467
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

468
469
470
471
472
473
474
475
476
477
478
479
480
        # Log locally every local_interval seconds.
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
            # Compute summary metrics for tracked stats (and log them
            # to promethus if applicable).
            prompt_throughput = get_throughput(self.num_prompt_tokens,
                                               now=stats.now,
                                               last_log=self.last_local_log)
            generation_throughput = get_throughput(
                self.num_generation_tokens,
                now=stats.now,
                last_log=self.last_local_log)

481
482
483
484
485
486
487
488
            log_fn = logger.info
            if not any((prompt_throughput, generation_throughput,
                        self.last_prompt_throughput,
                        self.last_generation_throughput)):
                # Avoid log noise on an idle production system
                log_fn = logger.debug

            log_fn(
489
490
491
492
493
494
495
496
497
498
499
500
501
                "Avg prompt throughput: %.1f tokens/s, "
                "Avg generation throughput: %.1f tokens/s, "
                "Running: %d reqs, Swapped: %d reqs, "
                "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
                "CPU KV cache usage: %.1f%%.",
                prompt_throughput,
                generation_throughput,
                stats.num_running_sys,
                stats.num_swapped_sys,
                stats.num_waiting_sys,
                stats.gpu_cache_usage_sys * 100,
                stats.cpu_cache_usage_sys * 100,
            )
502
503
            if (stats.cpu_prefix_cache_hit_rate >= 0
                    or stats.gpu_prefix_cache_hit_rate >= 0):
504
                log_fn(
505
506
507
508
                    "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
                    stats.gpu_prefix_cache_hit_rate * 100,
                    stats.cpu_prefix_cache_hit_rate * 100,
                )
509
            if self.spec_decode_metrics is not None:
510
                log_fn(
511
512
513
                    self._format_spec_decode_metrics_str(
                        self.spec_decode_metrics))

514
515
516
517
518
519
520
521
522
523
            self._reset(stats, prompt_throughput, generation_throughput)

    def _reset(self, stats, prompt_throughput, generation_throughput) -> None:
        # Reset tracked stats for next interval.
        self.num_prompt_tokens = []
        self.num_generation_tokens = []
        self.last_local_log = stats.now
        self.spec_decode_metrics = None
        self.last_prompt_throughput = prompt_throughput
        self.last_generation_throughput = generation_throughput
524
525
526
527
528
529
530
531
532

    def _format_spec_decode_metrics_str(
            self, metrics: "SpecDecodeWorkerMetrics") -> str:

        return ("Speculative metrics: "
                f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
                f"System efficiency: {metrics.system_efficiency:.3f}, "
                f"Number of speculative tokens: {metrics.num_spec_tokens}, "
                f"Number of accepted tokens: {metrics.accepted_tokens}, "
533
534
                f"Number of draft tokens: {metrics.draft_tokens}, "
                f"Number of emitted tokens: {metrics.emitted_tokens}.")
535

536
537
538
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        raise NotImplementedError

539
540
541
542

class PrometheusStatLogger(StatLoggerBase):
    """PrometheusStatLogger is used LLMEngine to log to Promethus."""
    _metrics_cls = Metrics
543
    _gauge_cls = prometheus_client.Gauge
544
545

    def __init__(self, local_interval: float, labels: Dict[str, str],
546
547
                 vllm_config: VllmConfig) -> None:
        super().__init__(local_interval, vllm_config)
548
549
        # Prometheus metrics
        self.labels = labels
550
        self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
551
                                         vllm_config=vllm_config)
552

553
554
555
    def _log_gauge(self, gauge, data: Union[int, float]) -> None:
        # Convenience function for logging to gauge.
        gauge.labels(**self.labels).set(data)
556

557
558
    def _log_counter(self, counter, data: Union[int, float]) -> None:
        # Convenience function for logging to counter.
559
560
561
562
563
        # Prevent ValueError from negative increment
        if data < 0:
            logger.warning("Skipping negative increment of %g to %s", data,
                           counter)
            return
564
565
566
567
568
569
570
571
572
573
574
575
576
        counter.labels(**self.labels).inc(data)

    def _log_counter_labels(self, counter, data: CollectionsCounter,
                            label_key: str) -> None:
        # Convenience function for collection counter of labels.
        for label, count in data.items():
            counter.labels(**{**self.labels, label_key: label}).inc(count)

    def _log_histogram(self, histogram, data: Union[List[int],
                                                    List[float]]) -> None:
        # Convenience function for logging list to histogram.
        for datum in data:
            histogram.labels(**self.labels).observe(datum)
577

578
    def _log_gauge_string(self, gauge, data: Dict[str, str]) -> None:
579
        gauge.labels(**data).set_to_current_time()
580

581
    def _log_prometheus(self, stats: Stats) -> None:
582
583
584
        # System state data
        self._log_gauge(self.metrics.gauge_scheduler_running,
                        stats.num_running_sys)
585
586
587
        if self.metrics.show_hidden_metrics:
            self._log_gauge(self.metrics.gauge_scheduler_swapped,
                            stats.num_swapped_sys)
588
589
590
591
        self._log_gauge(self.metrics.gauge_scheduler_waiting,
                        stats.num_waiting_sys)
        self._log_gauge(self.metrics.gauge_gpu_cache_usage,
                        stats.gpu_cache_usage_sys)
592
593
594
595
596
597
598
        if self.metrics.show_hidden_metrics:
            self._log_gauge(self.metrics.gauge_cpu_cache_usage,
                            stats.cpu_cache_usage_sys)
            self._log_gauge(self.metrics.gauge_cpu_prefix_cache_hit_rate,
                            stats.cpu_prefix_cache_hit_rate)
            self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate,
                            stats.gpu_prefix_cache_hit_rate)
599
600
601
602
603
604
605
606
607
608
609
        # Including max-lora in metric, in future this property of lora
        # config maybe extended to be dynamic.
        lora_info = {
            self.metrics.labelname_running_lora_adapters:
            ",".join(stats.running_lora_adapters),
            self.metrics.labelname_waiting_lora_adapters:
            ",".join(stats.waiting_lora_adapters),
            self.metrics.labelname_max_lora:
            stats.max_lora,
        }
        self._log_gauge_string(self.metrics.gauge_lora_info, lora_info)
610
        # Iteration level data
611
612
        self._log_counter(self.metrics.counter_num_preemption,
                          stats.num_preemption_iter)
613
614
615
616
        self._log_counter(self.metrics.counter_prompt_tokens,
                          stats.num_prompt_tokens_iter)
        self._log_counter(self.metrics.counter_generation_tokens,
                          stats.num_generation_tokens_iter)
harrywu's avatar
harrywu committed
617
618
        self._log_histogram(self.metrics.histogram_iteration_tokens,
                            [stats.num_tokens_iter])
619
620
621
622
623
624
625
626
627
        self._log_histogram(self.metrics.histogram_time_to_first_token,
                            stats.time_to_first_tokens_iter)
        self._log_histogram(self.metrics.histogram_time_per_output_token,
                            stats.time_per_output_tokens_iter)

        # Request level data
        # Latency
        self._log_histogram(self.metrics.histogram_e2e_time_request,
                            stats.time_e2e_requests)
harrywu's avatar
harrywu committed
628
629
630
631
632
        self._log_histogram(self.metrics.histogram_queue_time_request,
                            stats.time_queue_requests)
        self._log_histogram(self.metrics.histogram_inference_time_request,
                            stats.time_inference_requests)
        self._log_histogram(self.metrics.histogram_prefill_time_request,
633
634
                            stats.time_prefill_requests)
        self._log_histogram(self.metrics.histogram_decode_time_request,
harrywu's avatar
harrywu committed
635
                            stats.time_decode_requests)
636
637
638
639
640
641
642
643
644
        if self.metrics.show_hidden_metrics:
            self._log_histogram(self.metrics.histogram_time_in_queue_request,
                                stats.time_in_queue_requests)
            self._log_histogram(
                self.metrics.histogram_model_forward_time_request,
                stats.model_forward_time_requests)
            self._log_histogram(
                self.metrics.histogram_model_execute_time_request,
                stats.model_execute_time_requests)
645
646
647
648
649
650
651
652
653
654
655
656
        # Metadata
        finished_reason_counter = CollectionsCounter(
            stats.finished_reason_requests)
        self._log_counter_labels(self.metrics.counter_request_success,
                                 finished_reason_counter,
                                 Metrics.labelname_finish_reason)
        self._log_histogram(self.metrics.histogram_num_prompt_tokens_request,
                            stats.num_prompt_tokens_requests)
        self._log_histogram(
            self.metrics.histogram_num_generation_tokens_request,
            stats.num_generation_tokens_requests)
        self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
harrywu's avatar
harrywu committed
657
658
659
        self._log_histogram(
            self.metrics.histogram_max_num_generation_tokens_request,
            stats.max_num_generation_tokens_requests)
660
661
        self._log_histogram(self.metrics.histogram_max_tokens_request,
                            stats.max_tokens_requests)
662

663
664
    def log(self, stats: Stats):
        """Logs to prometheus and tracked stats every iteration."""
665
666
667
668
        # Log to prometheus.
        self._log_prometheus(stats)

        # Save tracked stats for token counters.
669
670
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)
671

672
673
674
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

675
        # Log locally every local_interval seconds.
676
677
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
678
            if self.spec_decode_metrics is not None:
679
680
                self._log_gauge(
                    self.metrics.gauge_spec_decode_draft_acceptance_rate,
681
                    self.spec_decode_metrics.draft_acceptance_rate)
682
                self._log_gauge(self.metrics.gauge_spec_decode_efficiency,
683
                                self.spec_decode_metrics.system_efficiency)
684
685
                self._log_counter(
                    self.metrics.counter_spec_decode_num_accepted_tokens,
686
                    self.spec_decode_metrics.accepted_tokens)
687
688
                self._log_counter(
                    self.metrics.counter_spec_decode_num_draft_tokens,
689
                    self.spec_decode_metrics.draft_tokens)
690
691
                self._log_counter(
                    self.metrics.counter_spec_decode_num_emitted_tokens,
692
693
694
695
696
697
698
                    self.spec_decode_metrics.emitted_tokens)

            # Reset tracked stats for next interval.
            self.num_prompt_tokens = []
            self.num_generation_tokens = []
            self.last_local_log = stats.now
            self.spec_decode_metrics = None
699

700
701
702
703
704
705
706
707
708
709
710
711
712
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        # Info type metrics are syntactic sugar for a gauge permanently set to 1
        # Since prometheus multiprocessing mode does not support Info, emulate
        # info here with a gauge.
        if type == "cache_config":
            metrics_info = obj.metrics_info()
            info_gauge = self._gauge_cls(
                name="vllm:cache_config_info",
                documentation="Information of the LLMEngine CacheConfig",
                labelnames=metrics_info.keys(),
                multiprocess_mode="mostrecent")
            info_gauge.labels(**metrics_info).set(1)

713

714
715
class RayPrometheusStatLogger(PrometheusStatLogger):
    """RayPrometheusStatLogger uses Ray metrics instead."""
716
    _metrics_cls = RayMetrics
717
718
719

    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        return None