metrics.py 30.3 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import time
4
5
from typing import TYPE_CHECKING
from typing import Counter as CollectionsCounter
6
from typing import Dict, List, Optional, Type, Union, cast
7
8

import numpy as np
9
import prometheus_client
10

11
12
from vllm.config import SupportsMetricsInfo, VllmConfig
from vllm.engine.metrics_types import StatLoggerBase, Stats
13
from vllm.executor.ray_utils import ray
14
from vllm.logger import init_logger
15

16
17
18
19
20
if ray is not None:
    from ray.util import metrics as ray_metrics
else:
    ray_metrics = None

21
22
23
if TYPE_CHECKING:
    from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics

24
25
logger = init_logger(__name__)

26
prometheus_client.disable_created_metrics()
27
28
29
30

# The begin-* and end* here are used by the documentation generator
# to extract the metrics definitions.

31

32
# begin-metrics-definitions
33
class Metrics:
34
35
36
37
38
39
    """
    vLLM uses a multiprocessing-based frontend for the OpenAI server.
    This means that we need to run prometheus_client in multiprocessing mode
    See https://prometheus.github.io/client_python/multiprocess/ for more
    details on limitations.
    """
40

41
    labelname_finish_reason = "finished_reason"
42
43
44
    labelname_waiting_lora_adapters = "waiting_lora_adapters"
    labelname_running_lora_adapters = "running_lora_adapters"
    labelname_max_lora = "max_lora"
45
46
47
    _gauge_cls = prometheus_client.Gauge
    _counter_cls = prometheus_client.Counter
    _histogram_cls = prometheus_client.Histogram
48

49
    def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
50
        # Unregister any existing vLLM collectors (for CI/CD)
51
        self._unregister_vllm_metrics()
52

53
54
        max_model_len = vllm_config.model_config.max_model_len

55
        # System stats
56
        #   Scheduler State
57
        self.gauge_scheduler_running = self._gauge_cls(
58
59
            name="vllm:num_requests_running",
            documentation="Number of requests currently running on GPU.",
60
61
            labelnames=labelnames,
            multiprocess_mode="sum")
62
        self.gauge_scheduler_waiting = self._gauge_cls(
63
64
            name="vllm:num_requests_waiting",
            documentation="Number of requests waiting to be processed.",
65
66
            labelnames=labelnames,
            multiprocess_mode="sum")
67
68
69
70
71
72
73
74
75
76
        self.gauge_lora_info = self._gauge_cls(
            name="vllm:lora_requests_info",
            documentation="Running stats on lora requests.",
            labelnames=[
                self.labelname_running_lora_adapters,
                self.labelname_max_lora,
                self.labelname_waiting_lora_adapters,
            ],
            multiprocess_mode="livemostrecent",
        )
77
78
79

        # Deprecated in 0.8 - KV cache offloading is not used in V1
        # TODO: in 0.9, only enable if show_hidden_metrics=True
80
        self.gauge_scheduler_swapped = self._gauge_cls(
81
            name="vllm:num_requests_swapped",
82
83
84
            documentation=(
                "Number of requests swapped to CPU. "
                "DEPRECATED: KV cache offloading is not used in V1"),
85
86
            labelnames=labelnames,
            multiprocess_mode="sum")
87

88
        #   KV Cache Usage in %
89
        self.gauge_gpu_cache_usage = self._gauge_cls(
90
91
            name="vllm:gpu_cache_usage_perc",
            documentation="GPU KV-cache usage. 1 means 100 percent usage.",
92
93
            labelnames=labelnames,
            multiprocess_mode="sum")
94
95
96

        # Deprecated in 0.8 - KV cache offloading is not used in V1
        # TODO: in 0.9, only enable if show_hidden_metrics=True
97
        self.gauge_cpu_cache_usage = self._gauge_cls(
98
            name="vllm:cpu_cache_usage_perc",
99
100
101
            documentation=(
                "CPU KV-cache usage. 1 means 100 percent usage. "
                "DEPRECATED: KV cache offloading is not used in V1"),
102
103
            labelnames=labelnames,
            multiprocess_mode="sum")
104
105
106

        # Deprecated in 0.8 - KV cache offloading is not used in V1
        # TODO: in 0.9, only enable if show_hidden_metrics=True
107
108
        self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls(
            name="vllm:cpu_prefix_cache_hit_rate",
109
110
111
            documentation=(
                "CPU prefix cache block hit rate. "
                "DEPRECATED: KV cache offloading is not used in V1"),
112
113
            labelnames=labelnames,
            multiprocess_mode="sum")
114
115
116

        # Deprecated in 0.8 - replaced by queries+hits counters in V1
        # TODO: in 0.9, only enable if show_hidden_metrics=True
117
118
        self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls(
            name="vllm:gpu_prefix_cache_hit_rate",
119
120
121
            documentation=("GPU prefix cache block hit rate. "
                           "DEPRECATED: use vllm:gpu_prefix_cache_queries and "
                           "vllm:gpu_prefix_cache_queries in V1"),
122
123
            labelnames=labelnames,
            multiprocess_mode="sum")
124

125
        # Iteration stats
126
        self.counter_num_preemption = self._counter_cls(
127
128
129
            name="vllm:num_preemptions_total",
            documentation="Cumulative number of preemption from the engine.",
            labelnames=labelnames)
130
        self.counter_prompt_tokens = self._counter_cls(
131
132
133
            name="vllm:prompt_tokens_total",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames)
134
        self.counter_generation_tokens = self._counter_cls(
135
136
137
            name="vllm:generation_tokens_total",
            documentation="Number of generation tokens processed.",
            labelnames=labelnames)
138
139
        buckets = [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096]
        if not vllm_config.model_config.enforce_eager:
140
141
            buckets = vllm_config.compilation_config.\
                cudagraph_capture_sizes.copy()
142
            buckets.sort()
harrywu's avatar
harrywu committed
143
144
145
146
        self.histogram_iteration_tokens = self._histogram_cls(
            name="vllm:iteration_tokens_total",
            documentation="Histogram of number of tokens per engine_step.",
            labelnames=labelnames,
147
            buckets=buckets)
148
        self.histogram_time_to_first_token = self._histogram_cls(
149
150
151
152
153
154
155
            name="vllm:time_to_first_token_seconds",
            documentation="Histogram of time to first token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
                0.75, 1.0, 2.5, 5.0, 7.5, 10.0
            ])
156
        self.histogram_time_per_output_token = self._histogram_cls(
157
158
159
160
161
162
163
            name="vllm:time_per_output_token_seconds",
            documentation="Histogram of time per output token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
                1.0, 2.5
            ])
164
165
166

        # Request stats
        #   Latency
harrywu's avatar
harrywu committed
167
168
169
170
        request_latency_buckets = [
            0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
            40.0, 50.0, 60.0
        ]
171
        self.histogram_e2e_time_request = self._histogram_cls(
172
173
174
            name="vllm:e2e_request_latency_seconds",
            documentation="Histogram of end to end request latency in seconds.",
            labelnames=labelnames,
harrywu's avatar
harrywu committed
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
            buckets=request_latency_buckets)
        self.histogram_queue_time_request = self._histogram_cls(
            name="vllm:request_queue_time_seconds",
            documentation=
            "Histogram of time spent in WAITING phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_inference_time_request = self._histogram_cls(
            name="vllm:request_inference_time_seconds",
            documentation=
            "Histogram of time spent in RUNNING phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_prefill_time_request = self._histogram_cls(
            name="vllm:request_prefill_time_seconds",
            documentation=
            "Histogram of time spent in PREFILL phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_decode_time_request = self._histogram_cls(
            name="vllm:request_decode_time_seconds",
            documentation=
            "Histogram of time spent in DECODE phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
200
201
        # Deprecated in 0.8 - duplicates vllm:request_queue_time_seconds:
        # TODO: in 0.9, only enable if show_hidden_metrics=True
202
203
        self.histogram_time_in_queue_request = self._histogram_cls(
            name="vllm:time_in_queue_requests",
204
205
206
            documentation=(
                "Histogram of time the request spent in the queue in seconds. "
                "DEPRECATED: use vllm:request_queue_time_seconds instead."),
207
            labelnames=labelnames,
harrywu's avatar
harrywu committed
208
            buckets=request_latency_buckets)
209
210
211

        # Deprecated in 0.8 - use prefill/decode/inference time metrics
        # TODO: in 0.9, only enable if show_hidden_metrics=True
212
213
        self.histogram_model_forward_time_request = self._histogram_cls(
            name="vllm:model_forward_time_milliseconds",
214
215
216
217
            documentation=(
                "Histogram of time spent in the model forward pass in ms. "
                "DEPRECATED: use prefill/decode/inference time metrics instead."
            ),
218
219
220
221
            labelnames=labelnames,
            buckets=build_1_2_3_5_8_buckets(3000))
        self.histogram_model_execute_time_request = self._histogram_cls(
            name="vllm:model_execute_time_milliseconds",
222
223
224
225
            documentation=(
                "Histogram of time spent in the model execute function in ms."
                "DEPRECATED: use prefill/decode/inference time metrics instead."
            ),
226
227
            labelnames=labelnames,
            buckets=build_1_2_3_5_8_buckets(3000))
228

229
        #   Metadata
230
        self.histogram_num_prompt_tokens_request = self._histogram_cls(
231
232
233
234
235
            name="vllm:request_prompt_tokens",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len),
        )
236
        self.histogram_num_generation_tokens_request = \
237
            self._histogram_cls(
238
239
240
241
242
                name="vllm:request_generation_tokens",
                documentation="Number of generation tokens processed.",
                labelnames=labelnames,
                buckets=build_1_2_5_buckets(max_model_len),
            )
harrywu's avatar
harrywu committed
243
244
245
246
247
248
        self.histogram_max_num_generation_tokens_request = self._histogram_cls(
            name="vllm:request_max_num_generation_tokens",
            documentation=
            "Histogram of maximum number of requested generation tokens.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len))
249
        self.histogram_n_request = self._histogram_cls(
250
251
252
253
254
            name="vllm:request_params_n",
            documentation="Histogram of the n request parameter.",
            labelnames=labelnames,
            buckets=[1, 2, 5, 10, 20],
        )
255
256
257
258
259
260
        self.histogram_max_tokens_request = self._histogram_cls(
            name="vllm:request_params_max_tokens",
            documentation="Histogram of the max_tokens request parameter.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len),
        )
261
        self.counter_request_success = self._counter_cls(
262
            name="vllm:request_success_total",
263
264
            documentation="Count of successfully processed requests.",
            labelnames=labelnames + [Metrics.labelname_finish_reason])
265

266
        # Speculative decoding stats
267
        self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
268
269
            name="vllm:spec_decode_draft_acceptance_rate",
            documentation="Speulative token acceptance rate.",
270
271
            labelnames=labelnames,
            multiprocess_mode="sum")
272
        self.gauge_spec_decode_efficiency = self._gauge_cls(
273
274
            name="vllm:spec_decode_efficiency",
            documentation="Speculative decoding system efficiency.",
275
276
            labelnames=labelnames,
            multiprocess_mode="sum")
277
278
279
280
281
        self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_accepted_tokens_total",
            documentation="Number of accepted tokens.",
            labelnames=labelnames))
        self.counter_spec_decode_num_draft_tokens = self._counter_cls(
282
283
284
            name="vllm:spec_decode_num_draft_tokens_total",
            documentation="Number of draft tokens.",
            labelnames=labelnames)
285
286
287
288
        self.counter_spec_decode_num_emitted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_emitted_tokens_total",
            documentation="Number of emitted tokens.",
            labelnames=labelnames))
289

290
291

# end-metrics-definitions
292

293
    def _unregister_vllm_metrics(self) -> None:
294
        for collector in list(prometheus_client.REGISTRY._collector_to_names):
295
            if hasattr(collector, "_name") and "vllm" in collector._name:
296
297
298
299
300
301
302
303
304
305
                prometheus_client.REGISTRY.unregister(collector)


class _RayGaugeWrapper:
    """Wraps around ray.util.metrics.Gauge to provide same API as
    prometheus_client.Gauge"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
306
307
308
                 labelnames: Optional[List[str]] = None,
                 multiprocess_mode: str = ""):
        del multiprocess_mode
309
310
311
312
313
314
315
316
317
318
319
320
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._gauge = ray_metrics.Gauge(name=name,
                                        description=documentation,
                                        tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._gauge.set_default_tags(labels)
        return self

    def set(self, value: Union[int, float]):
        return self._gauge.set(value)

321
322
323
324
    def set_to_current_time(self):
        # ray metrics doesn't have set_to_current time, https://docs.ray.io/en/latest/_modules/ray/util/metrics.html
        return self._gauge.set(time.time())

325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358

class _RayCounterWrapper:
    """Wraps around ray.util.metrics.Counter to provide same API as
    prometheus_client.Counter"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._counter = ray_metrics.Counter(name=name,
                                            description=documentation,
                                            tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._counter.set_default_tags(labels)
        return self

    def inc(self, value: Union[int, float] = 1.0):
        if value == 0:
            return
        return self._counter.inc(value)


class _RayHistogramWrapper:
    """Wraps around ray.util.metrics.Histogram to provide same API as
    prometheus_client.Histogram"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None,
                 buckets: Optional[List[float]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
359
        boundaries = buckets if buckets else []
360
361
362
        self._histogram = ray_metrics.Histogram(name=name,
                                                description=documentation,
                                                tag_keys=labelnames_tuple,
363
                                                boundaries=boundaries)
364
365
366
367
368
369
370

    def labels(self, **labels):
        self._histogram.set_default_tags(labels)
        return self

    def observe(self, value: Union[int, float]):
        return self._histogram.observe(value)
371
372
373
374
375
376
377


class RayMetrics(Metrics):
    """
    RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
    Provides the same metrics as Metrics but uses Ray's util.metrics library.
    """
378
379
380
381
382
383
    _gauge_cls: Type[prometheus_client.Gauge] = cast(
        Type[prometheus_client.Gauge], _RayGaugeWrapper)
    _counter_cls: Type[prometheus_client.Counter] = cast(
        Type[prometheus_client.Counter], _RayCounterWrapper)
    _histogram_cls: Type[prometheus_client.Histogram] = cast(
        Type[prometheus_client.Histogram], _RayHistogramWrapper)
384

385
    def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
386
387
        if ray_metrics is None:
            raise ImportError("RayMetrics requires Ray to be installed.")
388
        super().__init__(labelnames, vllm_config)
389
390
391
392
393

    def _unregister_vllm_metrics(self) -> None:
        # No-op on purpose
        pass

394

395
def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:
396
    """
397
398
    Builds a list of buckets with increasing powers of 10 multiplied by
    mantissa values until the value exceeds the specified maximum.
399
400
401

    """
    exponent = 0
402
    buckets: List[int] = []
403
404
405
406
407
408
409
410
411
412
    while True:
        for m in mantissa_lst:
            value = m * 10**exponent
            if value <= max_value:
                buckets.append(value)
            else:
                return buckets
        exponent += 1


413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
def build_1_2_5_buckets(max_value: int) -> List[int]:
    """
    Example:
    >>> build_1_2_5_buckets(100)
    [1, 2, 5, 10, 20, 50, 100]
    """
    return build_buckets([1, 2, 5], max_value)


def build_1_2_3_5_8_buckets(max_value: int) -> List[int]:
    """
    Example:
    >>> build_1_2_3_5_8_buckets(100)
    [1, 2, 3, 5, 8, 10, 20, 30, 50, 80, 100]
    """
    return build_buckets([1, 2, 3, 5, 8], max_value)


431
432
433
434
435
436
437
438
439
def local_interval_elapsed(now: float, last_log: float,
                           local_interval: float) -> bool:
    elapsed_time = now - last_log
    return elapsed_time > local_interval


def get_throughput(tracked_stats: List[int], now: float,
                   last_log: float) -> float:
    return float(np.sum(tracked_stats) / (now - last_log))
440
441


442
443
444
class LoggingStatLogger(StatLoggerBase):
    """LoggingStatLogger is used in LLMEngine to log to Stdout."""

445
446
    def __init__(self, local_interval: float, vllm_config: VllmConfig) -> None:
        super().__init__(local_interval, vllm_config)
447
448
449
        self.last_prompt_throughput: Optional[float] = None
        self.last_generation_throughput: Optional[float] = None

450
451
452
453
454
455
456
457
    def log(self, stats: Stats) -> None:
        """Called by LLMEngine.
           Logs to Stdout every self.local_interval seconds."""

        # Save tracked stats for token counters.
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)

458
459
460
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

461
462
463
464
465
466
467
468
469
470
471
472
473
        # Log locally every local_interval seconds.
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
            # Compute summary metrics for tracked stats (and log them
            # to promethus if applicable).
            prompt_throughput = get_throughput(self.num_prompt_tokens,
                                               now=stats.now,
                                               last_log=self.last_local_log)
            generation_throughput = get_throughput(
                self.num_generation_tokens,
                now=stats.now,
                last_log=self.last_local_log)

474
475
476
477
478
479
480
481
            log_fn = logger.info
            if not any((prompt_throughput, generation_throughput,
                        self.last_prompt_throughput,
                        self.last_generation_throughput)):
                # Avoid log noise on an idle production system
                log_fn = logger.debug

            log_fn(
482
483
484
485
486
487
488
489
490
491
492
493
494
                "Avg prompt throughput: %.1f tokens/s, "
                "Avg generation throughput: %.1f tokens/s, "
                "Running: %d reqs, Swapped: %d reqs, "
                "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
                "CPU KV cache usage: %.1f%%.",
                prompt_throughput,
                generation_throughput,
                stats.num_running_sys,
                stats.num_swapped_sys,
                stats.num_waiting_sys,
                stats.gpu_cache_usage_sys * 100,
                stats.cpu_cache_usage_sys * 100,
            )
495
496
            if (stats.cpu_prefix_cache_hit_rate >= 0
                    or stats.gpu_prefix_cache_hit_rate >= 0):
497
                log_fn(
498
499
500
501
                    "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
                    stats.gpu_prefix_cache_hit_rate * 100,
                    stats.cpu_prefix_cache_hit_rate * 100,
                )
502
            if self.spec_decode_metrics is not None:
503
                log_fn(
504
505
506
                    self._format_spec_decode_metrics_str(
                        self.spec_decode_metrics))

507
508
509
510
511
512
513
514
515
516
            self._reset(stats, prompt_throughput, generation_throughput)

    def _reset(self, stats, prompt_throughput, generation_throughput) -> None:
        # Reset tracked stats for next interval.
        self.num_prompt_tokens = []
        self.num_generation_tokens = []
        self.last_local_log = stats.now
        self.spec_decode_metrics = None
        self.last_prompt_throughput = prompt_throughput
        self.last_generation_throughput = generation_throughput
517
518
519
520
521
522
523
524
525

    def _format_spec_decode_metrics_str(
            self, metrics: "SpecDecodeWorkerMetrics") -> str:

        return ("Speculative metrics: "
                f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
                f"System efficiency: {metrics.system_efficiency:.3f}, "
                f"Number of speculative tokens: {metrics.num_spec_tokens}, "
                f"Number of accepted tokens: {metrics.accepted_tokens}, "
526
527
                f"Number of draft tokens: {metrics.draft_tokens}, "
                f"Number of emitted tokens: {metrics.emitted_tokens}.")
528

529
530
531
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        raise NotImplementedError

532
533
534
535

class PrometheusStatLogger(StatLoggerBase):
    """PrometheusStatLogger is used LLMEngine to log to Promethus."""
    _metrics_cls = Metrics
536
    _gauge_cls = prometheus_client.Gauge
537
538

    def __init__(self, local_interval: float, labels: Dict[str, str],
539
540
                 vllm_config: VllmConfig) -> None:
        super().__init__(local_interval, vllm_config)
541
542
        # Prometheus metrics
        self.labels = labels
543
        self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
544
                                         vllm_config=vllm_config)
545

546
547
548
549
550
        # Use this flag to hide metrics that were deprecated in
        # a previous release and which will be removed future
        self.show_hidden_metrics = \
            vllm_config.observability_config.show_hidden_metrics

551
552
553
    def _log_gauge(self, gauge, data: Union[int, float]) -> None:
        # Convenience function for logging to gauge.
        gauge.labels(**self.labels).set(data)
554

555
556
    def _log_counter(self, counter, data: Union[int, float]) -> None:
        # Convenience function for logging to counter.
557
558
559
560
561
        # Prevent ValueError from negative increment
        if data < 0:
            logger.warning("Skipping negative increment of %g to %s", data,
                           counter)
            return
562
563
564
565
566
567
568
569
570
571
572
573
574
        counter.labels(**self.labels).inc(data)

    def _log_counter_labels(self, counter, data: CollectionsCounter,
                            label_key: str) -> None:
        # Convenience function for collection counter of labels.
        for label, count in data.items():
            counter.labels(**{**self.labels, label_key: label}).inc(count)

    def _log_histogram(self, histogram, data: Union[List[int],
                                                    List[float]]) -> None:
        # Convenience function for logging list to histogram.
        for datum in data:
            histogram.labels(**self.labels).observe(datum)
575

576
    def _log_gauge_string(self, gauge, data: Dict[str, str]) -> None:
577
        gauge.labels(**data).set_to_current_time()
578

579
    def _log_prometheus(self, stats: Stats) -> None:
580
581
582
583
584
585
586
587
588
589
590
        # System state data
        self._log_gauge(self.metrics.gauge_scheduler_running,
                        stats.num_running_sys)
        self._log_gauge(self.metrics.gauge_scheduler_swapped,
                        stats.num_swapped_sys)
        self._log_gauge(self.metrics.gauge_scheduler_waiting,
                        stats.num_waiting_sys)
        self._log_gauge(self.metrics.gauge_gpu_cache_usage,
                        stats.gpu_cache_usage_sys)
        self._log_gauge(self.metrics.gauge_cpu_cache_usage,
                        stats.cpu_cache_usage_sys)
591
592
593
594
        self._log_gauge(self.metrics.gauge_cpu_prefix_cache_hit_rate,
                        stats.cpu_prefix_cache_hit_rate)
        self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate,
                        stats.gpu_prefix_cache_hit_rate)
595
596
597
598
599
600
601
602
603
604
605
        # Including max-lora in metric, in future this property of lora
        # config maybe extended to be dynamic.
        lora_info = {
            self.metrics.labelname_running_lora_adapters:
            ",".join(stats.running_lora_adapters),
            self.metrics.labelname_waiting_lora_adapters:
            ",".join(stats.waiting_lora_adapters),
            self.metrics.labelname_max_lora:
            stats.max_lora,
        }
        self._log_gauge_string(self.metrics.gauge_lora_info, lora_info)
606
        # Iteration level data
607
608
        self._log_counter(self.metrics.counter_num_preemption,
                          stats.num_preemption_iter)
609
610
611
612
        self._log_counter(self.metrics.counter_prompt_tokens,
                          stats.num_prompt_tokens_iter)
        self._log_counter(self.metrics.counter_generation_tokens,
                          stats.num_generation_tokens_iter)
harrywu's avatar
harrywu committed
613
614
        self._log_histogram(self.metrics.histogram_iteration_tokens,
                            [stats.num_tokens_iter])
615
616
617
618
619
620
621
622
623
        self._log_histogram(self.metrics.histogram_time_to_first_token,
                            stats.time_to_first_tokens_iter)
        self._log_histogram(self.metrics.histogram_time_per_output_token,
                            stats.time_per_output_tokens_iter)

        # Request level data
        # Latency
        self._log_histogram(self.metrics.histogram_e2e_time_request,
                            stats.time_e2e_requests)
harrywu's avatar
harrywu committed
624
625
626
627
628
        self._log_histogram(self.metrics.histogram_queue_time_request,
                            stats.time_queue_requests)
        self._log_histogram(self.metrics.histogram_inference_time_request,
                            stats.time_inference_requests)
        self._log_histogram(self.metrics.histogram_prefill_time_request,
629
630
                            stats.time_prefill_requests)
        self._log_histogram(self.metrics.histogram_decode_time_request,
harrywu's avatar
harrywu committed
631
                            stats.time_decode_requests)
632
633
634
635
636
637
        self._log_histogram(self.metrics.histogram_time_in_queue_request,
                            stats.time_in_queue_requests)
        self._log_histogram(self.metrics.histogram_model_forward_time_request,
                            stats.model_forward_time_requests)
        self._log_histogram(self.metrics.histogram_model_execute_time_request,
                            stats.model_execute_time_requests)
638
639
640
641
642
643
644
645
646
647
648
649
        # Metadata
        finished_reason_counter = CollectionsCounter(
            stats.finished_reason_requests)
        self._log_counter_labels(self.metrics.counter_request_success,
                                 finished_reason_counter,
                                 Metrics.labelname_finish_reason)
        self._log_histogram(self.metrics.histogram_num_prompt_tokens_request,
                            stats.num_prompt_tokens_requests)
        self._log_histogram(
            self.metrics.histogram_num_generation_tokens_request,
            stats.num_generation_tokens_requests)
        self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
harrywu's avatar
harrywu committed
650
651
652
        self._log_histogram(
            self.metrics.histogram_max_num_generation_tokens_request,
            stats.max_num_generation_tokens_requests)
653
654
        self._log_histogram(self.metrics.histogram_max_tokens_request,
                            stats.max_tokens_requests)
655

656
657
    def log(self, stats: Stats):
        """Logs to prometheus and tracked stats every iteration."""
658
659
660
661
        # Log to prometheus.
        self._log_prometheus(stats)

        # Save tracked stats for token counters.
662
663
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)
664

665
666
667
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

668
        # Log locally every local_interval seconds.
669
670
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
671
            if self.spec_decode_metrics is not None:
672
673
                self._log_gauge(
                    self.metrics.gauge_spec_decode_draft_acceptance_rate,
674
                    self.spec_decode_metrics.draft_acceptance_rate)
675
                self._log_gauge(self.metrics.gauge_spec_decode_efficiency,
676
                                self.spec_decode_metrics.system_efficiency)
677
678
                self._log_counter(
                    self.metrics.counter_spec_decode_num_accepted_tokens,
679
                    self.spec_decode_metrics.accepted_tokens)
680
681
                self._log_counter(
                    self.metrics.counter_spec_decode_num_draft_tokens,
682
                    self.spec_decode_metrics.draft_tokens)
683
684
                self._log_counter(
                    self.metrics.counter_spec_decode_num_emitted_tokens,
685
686
687
688
689
690
691
                    self.spec_decode_metrics.emitted_tokens)

            # Reset tracked stats for next interval.
            self.num_prompt_tokens = []
            self.num_generation_tokens = []
            self.last_local_log = stats.now
            self.spec_decode_metrics = None
692

693
694
695
696
697
698
699
700
701
702
703
704
705
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        # Info type metrics are syntactic sugar for a gauge permanently set to 1
        # Since prometheus multiprocessing mode does not support Info, emulate
        # info here with a gauge.
        if type == "cache_config":
            metrics_info = obj.metrics_info()
            info_gauge = self._gauge_cls(
                name="vllm:cache_config_info",
                documentation="Information of the LLMEngine CacheConfig",
                labelnames=metrics_info.keys(),
                multiprocess_mode="mostrecent")
            info_gauge.labels(**metrics_info).set(1)

706

707
708
class RayPrometheusStatLogger(PrometheusStatLogger):
    """RayPrometheusStatLogger uses Ray metrics instead."""
709
    _metrics_cls = RayMetrics
710
711
712

    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        return None