metrics.py 29.1 KB
Newer Older
1
2
# SPDX-License-Identifier: Apache-2.0

3
import time
4
5
from typing import TYPE_CHECKING
from typing import Counter as CollectionsCounter
6
from typing import Dict, List, Optional, Type, Union, cast
7
8

import numpy as np
9
import prometheus_client
10

11
12
from vllm.config import SupportsMetricsInfo, VllmConfig
from vllm.engine.metrics_types import StatLoggerBase, Stats
13
from vllm.executor.ray_utils import ray
14
from vllm.logger import init_logger
15

16
17
18
19
20
if ray is not None:
    from ray.util import metrics as ray_metrics
else:
    ray_metrics = None

21
22
23
if TYPE_CHECKING:
    from vllm.spec_decode.metrics import SpecDecodeWorkerMetrics

24
25
logger = init_logger(__name__)

26
prometheus_client.disable_created_metrics()
27
28
29
30

# The begin-* and end* here are used by the documentation generator
# to extract the metrics definitions.

31

32
# begin-metrics-definitions
33
class Metrics:
34
35
36
37
38
39
    """
    vLLM uses a multiprocessing-based frontend for the OpenAI server.
    This means that we need to run prometheus_client in multiprocessing mode
    See https://prometheus.github.io/client_python/multiprocess/ for more
    details on limitations.
    """
40

41
    labelname_finish_reason = "finished_reason"
42
43
44
    labelname_waiting_lora_adapters = "waiting_lora_adapters"
    labelname_running_lora_adapters = "running_lora_adapters"
    labelname_max_lora = "max_lora"
45
46
47
    _gauge_cls = prometheus_client.Gauge
    _counter_cls = prometheus_client.Counter
    _histogram_cls = prometheus_client.Histogram
48

49
    def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
50
        # Unregister any existing vLLM collectors (for CI/CD)
51
        self._unregister_vllm_metrics()
52

53
54
        max_model_len = vllm_config.model_config.max_model_len

55
        # System stats
56
        #   Scheduler State
57
        self.gauge_scheduler_running = self._gauge_cls(
58
59
            name="vllm:num_requests_running",
            documentation="Number of requests currently running on GPU.",
60
61
            labelnames=labelnames,
            multiprocess_mode="sum")
62
        self.gauge_scheduler_waiting = self._gauge_cls(
63
64
            name="vllm:num_requests_waiting",
            documentation="Number of requests waiting to be processed.",
65
66
            labelnames=labelnames,
            multiprocess_mode="sum")
67
68
69
70
71
72
73
74
75
76
        self.gauge_lora_info = self._gauge_cls(
            name="vllm:lora_requests_info",
            documentation="Running stats on lora requests.",
            labelnames=[
                self.labelname_running_lora_adapters,
                self.labelname_max_lora,
                self.labelname_waiting_lora_adapters,
            ],
            multiprocess_mode="livemostrecent",
        )
77
        self.gauge_scheduler_swapped = self._gauge_cls(
78
79
            name="vllm:num_requests_swapped",
            documentation="Number of requests swapped to CPU.",
80
81
            labelnames=labelnames,
            multiprocess_mode="sum")
82
        #   KV Cache Usage in %
83
        self.gauge_gpu_cache_usage = self._gauge_cls(
84
85
            name="vllm:gpu_cache_usage_perc",
            documentation="GPU KV-cache usage. 1 means 100 percent usage.",
86
87
            labelnames=labelnames,
            multiprocess_mode="sum")
88
        self.gauge_cpu_cache_usage = self._gauge_cls(
89
90
            name="vllm:cpu_cache_usage_perc",
            documentation="CPU KV-cache usage. 1 means 100 percent usage.",
91
92
            labelnames=labelnames,
            multiprocess_mode="sum")
93
94
95
96
97
98
99
100
101
102
103
        #   Prefix caching block hit rate
        self.gauge_cpu_prefix_cache_hit_rate = self._gauge_cls(
            name="vllm:cpu_prefix_cache_hit_rate",
            documentation="CPU prefix cache block hit rate.",
            labelnames=labelnames,
            multiprocess_mode="sum")
        self.gauge_gpu_prefix_cache_hit_rate = self._gauge_cls(
            name="vllm:gpu_prefix_cache_hit_rate",
            documentation="GPU prefix cache block hit rate.",
            labelnames=labelnames,
            multiprocess_mode="sum")
104

105
        # Iteration stats
106
        self.counter_num_preemption = self._counter_cls(
107
108
109
            name="vllm:num_preemptions_total",
            documentation="Cumulative number of preemption from the engine.",
            labelnames=labelnames)
110
        self.counter_prompt_tokens = self._counter_cls(
111
112
113
            name="vllm:prompt_tokens_total",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames)
114
        self.counter_generation_tokens = self._counter_cls(
115
116
117
            name="vllm:generation_tokens_total",
            documentation="Number of generation tokens processed.",
            labelnames=labelnames)
harrywu's avatar
harrywu committed
118
119
120
121
        self.counter_tokens = self._counter_cls(
            name="vllm:tokens_total",
            documentation="Number of prefill plus generation tokens processed.",
            labelnames=labelnames)
122
123
        buckets = [1, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8096]
        if not vllm_config.model_config.enforce_eager:
124
125
            buckets = vllm_config.compilation_config.\
                cudagraph_capture_sizes.copy()
126
            buckets.sort()
harrywu's avatar
harrywu committed
127
128
129
130
        self.histogram_iteration_tokens = self._histogram_cls(
            name="vllm:iteration_tokens_total",
            documentation="Histogram of number of tokens per engine_step.",
            labelnames=labelnames,
131
            buckets=buckets)
132
        self.histogram_time_to_first_token = self._histogram_cls(
133
134
135
136
137
138
139
            name="vllm:time_to_first_token_seconds",
            documentation="Histogram of time to first token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.001, 0.005, 0.01, 0.02, 0.04, 0.06, 0.08, 0.1, 0.25, 0.5,
                0.75, 1.0, 2.5, 5.0, 7.5, 10.0
            ])
140
        self.histogram_time_per_output_token = self._histogram_cls(
141
142
143
144
145
146
147
            name="vllm:time_per_output_token_seconds",
            documentation="Histogram of time per output token in seconds.",
            labelnames=labelnames,
            buckets=[
                0.01, 0.025, 0.05, 0.075, 0.1, 0.15, 0.2, 0.3, 0.4, 0.5, 0.75,
                1.0, 2.5
            ])
148
149
150

        # Request stats
        #   Latency
harrywu's avatar
harrywu committed
151
152
153
154
        request_latency_buckets = [
            0.3, 0.5, 0.8, 1.0, 1.5, 2.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0,
            40.0, 50.0, 60.0
        ]
155
        self.histogram_e2e_time_request = self._histogram_cls(
156
157
158
            name="vllm:e2e_request_latency_seconds",
            documentation="Histogram of end to end request latency in seconds.",
            labelnames=labelnames,
harrywu's avatar
harrywu committed
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
            buckets=request_latency_buckets)
        self.histogram_queue_time_request = self._histogram_cls(
            name="vllm:request_queue_time_seconds",
            documentation=
            "Histogram of time spent in WAITING phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_inference_time_request = self._histogram_cls(
            name="vllm:request_inference_time_seconds",
            documentation=
            "Histogram of time spent in RUNNING phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_prefill_time_request = self._histogram_cls(
            name="vllm:request_prefill_time_seconds",
            documentation=
            "Histogram of time spent in PREFILL phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
        self.histogram_decode_time_request = self._histogram_cls(
            name="vllm:request_decode_time_seconds",
            documentation=
            "Histogram of time spent in DECODE phase for request.",
            labelnames=labelnames,
            buckets=request_latency_buckets)
184
185
186
187
188
        self.histogram_time_in_queue_request = self._histogram_cls(
            name="vllm:time_in_queue_requests",
            documentation=
            "Histogram of time the request spent in the queue in seconds.",
            labelnames=labelnames,
harrywu's avatar
harrywu committed
189
            buckets=request_latency_buckets)
190
191
192
193
194
195
196
197
198
199
200
201
        self.histogram_model_forward_time_request = self._histogram_cls(
            name="vllm:model_forward_time_milliseconds",
            documentation=
            "Histogram of time spent in the model forward pass in ms.",
            labelnames=labelnames,
            buckets=build_1_2_3_5_8_buckets(3000))
        self.histogram_model_execute_time_request = self._histogram_cls(
            name="vllm:model_execute_time_milliseconds",
            documentation=
            "Histogram of time spent in the model execute function in ms.",
            labelnames=labelnames,
            buckets=build_1_2_3_5_8_buckets(3000))
202
        #   Metadata
203
        self.histogram_num_prompt_tokens_request = self._histogram_cls(
204
205
206
207
208
            name="vllm:request_prompt_tokens",
            documentation="Number of prefill tokens processed.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len),
        )
209
        self.histogram_num_generation_tokens_request = \
210
            self._histogram_cls(
211
212
213
214
215
                name="vllm:request_generation_tokens",
                documentation="Number of generation tokens processed.",
                labelnames=labelnames,
                buckets=build_1_2_5_buckets(max_model_len),
            )
harrywu's avatar
harrywu committed
216
217
218
219
220
221
        self.histogram_max_num_generation_tokens_request = self._histogram_cls(
            name="vllm:request_max_num_generation_tokens",
            documentation=
            "Histogram of maximum number of requested generation tokens.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len))
222
        self.histogram_n_request = self._histogram_cls(
223
224
225
226
227
            name="vllm:request_params_n",
            documentation="Histogram of the n request parameter.",
            labelnames=labelnames,
            buckets=[1, 2, 5, 10, 20],
        )
228
229
230
231
232
233
        self.histogram_max_tokens_request = self._histogram_cls(
            name="vllm:request_params_max_tokens",
            documentation="Histogram of the max_tokens request parameter.",
            labelnames=labelnames,
            buckets=build_1_2_5_buckets(max_model_len),
        )
234
        self.counter_request_success = self._counter_cls(
235
            name="vllm:request_success_total",
236
237
            documentation="Count of successfully processed requests.",
            labelnames=labelnames + [Metrics.labelname_finish_reason])
238

239
        # Speculative decoding stats
240
        self.gauge_spec_decode_draft_acceptance_rate = self._gauge_cls(
241
242
            name="vllm:spec_decode_draft_acceptance_rate",
            documentation="Speulative token acceptance rate.",
243
244
            labelnames=labelnames,
            multiprocess_mode="sum")
245
        self.gauge_spec_decode_efficiency = self._gauge_cls(
246
247
            name="vllm:spec_decode_efficiency",
            documentation="Speculative decoding system efficiency.",
248
249
            labelnames=labelnames,
            multiprocess_mode="sum")
250
251
252
253
254
        self.counter_spec_decode_num_accepted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_accepted_tokens_total",
            documentation="Number of accepted tokens.",
            labelnames=labelnames))
        self.counter_spec_decode_num_draft_tokens = self._counter_cls(
255
256
257
            name="vllm:spec_decode_num_draft_tokens_total",
            documentation="Number of draft tokens.",
            labelnames=labelnames)
258
259
260
261
        self.counter_spec_decode_num_emitted_tokens = (self._counter_cls(
            name="vllm:spec_decode_num_emitted_tokens_total",
            documentation="Number of emitted tokens.",
            labelnames=labelnames))
262

263
264

# end-metrics-definitions
265

266
    def _unregister_vllm_metrics(self) -> None:
267
        for collector in list(prometheus_client.REGISTRY._collector_to_names):
268
            if hasattr(collector, "_name") and "vllm" in collector._name:
269
270
271
272
273
274
275
276
277
278
                prometheus_client.REGISTRY.unregister(collector)


class _RayGaugeWrapper:
    """Wraps around ray.util.metrics.Gauge to provide same API as
    prometheus_client.Gauge"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
279
280
281
                 labelnames: Optional[List[str]] = None,
                 multiprocess_mode: str = ""):
        del multiprocess_mode
282
283
284
285
286
287
288
289
290
291
292
293
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._gauge = ray_metrics.Gauge(name=name,
                                        description=documentation,
                                        tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._gauge.set_default_tags(labels)
        return self

    def set(self, value: Union[int, float]):
        return self._gauge.set(value)

294
295
296
297
    def set_to_current_time(self):
        # ray metrics doesn't have set_to_current time, https://docs.ray.io/en/latest/_modules/ray/util/metrics.html
        return self._gauge.set(time.time())

298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331

class _RayCounterWrapper:
    """Wraps around ray.util.metrics.Counter to provide same API as
    prometheus_client.Counter"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
        self._counter = ray_metrics.Counter(name=name,
                                            description=documentation,
                                            tag_keys=labelnames_tuple)

    def labels(self, **labels):
        self._counter.set_default_tags(labels)
        return self

    def inc(self, value: Union[int, float] = 1.0):
        if value == 0:
            return
        return self._counter.inc(value)


class _RayHistogramWrapper:
    """Wraps around ray.util.metrics.Histogram to provide same API as
    prometheus_client.Histogram"""

    def __init__(self,
                 name: str,
                 documentation: str = "",
                 labelnames: Optional[List[str]] = None,
                 buckets: Optional[List[float]] = None):
        labelnames_tuple = tuple(labelnames) if labelnames else None
332
        boundaries = buckets if buckets else []
333
334
335
        self._histogram = ray_metrics.Histogram(name=name,
                                                description=documentation,
                                                tag_keys=labelnames_tuple,
336
                                                boundaries=boundaries)
337
338
339
340
341
342
343

    def labels(self, **labels):
        self._histogram.set_default_tags(labels)
        return self

    def observe(self, value: Union[int, float]):
        return self._histogram.observe(value)
344
345
346
347
348
349
350


class RayMetrics(Metrics):
    """
    RayMetrics is used by RayPrometheusStatLogger to log to Ray metrics.
    Provides the same metrics as Metrics but uses Ray's util.metrics library.
    """
351
352
353
354
355
356
    _gauge_cls: Type[prometheus_client.Gauge] = cast(
        Type[prometheus_client.Gauge], _RayGaugeWrapper)
    _counter_cls: Type[prometheus_client.Counter] = cast(
        Type[prometheus_client.Counter], _RayCounterWrapper)
    _histogram_cls: Type[prometheus_client.Histogram] = cast(
        Type[prometheus_client.Histogram], _RayHistogramWrapper)
357

358
    def __init__(self, labelnames: List[str], vllm_config: VllmConfig):
359
360
        if ray_metrics is None:
            raise ImportError("RayMetrics requires Ray to be installed.")
361
        super().__init__(labelnames, vllm_config)
362
363
364
365
366

    def _unregister_vllm_metrics(self) -> None:
        # No-op on purpose
        pass

367

368
def build_buckets(mantissa_lst: List[int], max_value: int) -> List[int]:
369
    """
370
371
    Builds a list of buckets with increasing powers of 10 multiplied by
    mantissa values until the value exceeds the specified maximum.
372
373
374

    """
    exponent = 0
375
    buckets: List[int] = []
376
377
378
379
380
381
382
383
384
385
    while True:
        for m in mantissa_lst:
            value = m * 10**exponent
            if value <= max_value:
                buckets.append(value)
            else:
                return buckets
        exponent += 1


386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
def build_1_2_5_buckets(max_value: int) -> List[int]:
    """
    Example:
    >>> build_1_2_5_buckets(100)
    [1, 2, 5, 10, 20, 50, 100]
    """
    return build_buckets([1, 2, 5], max_value)


def build_1_2_3_5_8_buckets(max_value: int) -> List[int]:
    """
    Example:
    >>> build_1_2_3_5_8_buckets(100)
    [1, 2, 3, 5, 8, 10, 20, 30, 50, 80, 100]
    """
    return build_buckets([1, 2, 3, 5, 8], max_value)


404
405
406
407
408
409
410
411
412
def local_interval_elapsed(now: float, last_log: float,
                           local_interval: float) -> bool:
    elapsed_time = now - last_log
    return elapsed_time > local_interval


def get_throughput(tracked_stats: List[int], now: float,
                   last_log: float) -> float:
    return float(np.sum(tracked_stats) / (now - last_log))
413
414


415
416
417
class LoggingStatLogger(StatLoggerBase):
    """LoggingStatLogger is used in LLMEngine to log to Stdout."""

418
419
    def __init__(self, local_interval: float, vllm_config: VllmConfig) -> None:
        super().__init__(local_interval, vllm_config)
420
421
422
        self.last_prompt_throughput: Optional[float] = None
        self.last_generation_throughput: Optional[float] = None

423
424
425
426
427
428
429
430
    def log(self, stats: Stats) -> None:
        """Called by LLMEngine.
           Logs to Stdout every self.local_interval seconds."""

        # Save tracked stats for token counters.
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)

431
432
433
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

434
435
436
437
438
439
440
441
442
443
444
445
446
        # Log locally every local_interval seconds.
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
            # Compute summary metrics for tracked stats (and log them
            # to promethus if applicable).
            prompt_throughput = get_throughput(self.num_prompt_tokens,
                                               now=stats.now,
                                               last_log=self.last_local_log)
            generation_throughput = get_throughput(
                self.num_generation_tokens,
                now=stats.now,
                last_log=self.last_local_log)

447
448
449
450
451
452
453
454
            log_fn = logger.info
            if not any((prompt_throughput, generation_throughput,
                        self.last_prompt_throughput,
                        self.last_generation_throughput)):
                # Avoid log noise on an idle production system
                log_fn = logger.debug

            log_fn(
455
456
457
458
459
460
461
462
463
464
465
466
467
                "Avg prompt throughput: %.1f tokens/s, "
                "Avg generation throughput: %.1f tokens/s, "
                "Running: %d reqs, Swapped: %d reqs, "
                "Pending: %d reqs, GPU KV cache usage: %.1f%%, "
                "CPU KV cache usage: %.1f%%.",
                prompt_throughput,
                generation_throughput,
                stats.num_running_sys,
                stats.num_swapped_sys,
                stats.num_waiting_sys,
                stats.gpu_cache_usage_sys * 100,
                stats.cpu_cache_usage_sys * 100,
            )
468
469
            if (stats.cpu_prefix_cache_hit_rate >= 0
                    or stats.gpu_prefix_cache_hit_rate >= 0):
470
                log_fn(
471
472
473
474
                    "Prefix cache hit rate: GPU: %.2f%%, CPU: %.2f%%",
                    stats.gpu_prefix_cache_hit_rate * 100,
                    stats.cpu_prefix_cache_hit_rate * 100,
                )
475
            if self.spec_decode_metrics is not None:
476
                log_fn(
477
478
479
                    self._format_spec_decode_metrics_str(
                        self.spec_decode_metrics))

480
481
482
483
484
485
486
487
488
489
            self._reset(stats, prompt_throughput, generation_throughput)

    def _reset(self, stats, prompt_throughput, generation_throughput) -> None:
        # Reset tracked stats for next interval.
        self.num_prompt_tokens = []
        self.num_generation_tokens = []
        self.last_local_log = stats.now
        self.spec_decode_metrics = None
        self.last_prompt_throughput = prompt_throughput
        self.last_generation_throughput = generation_throughput
490
491
492
493
494
495
496
497
498

    def _format_spec_decode_metrics_str(
            self, metrics: "SpecDecodeWorkerMetrics") -> str:

        return ("Speculative metrics: "
                f"Draft acceptance rate: {metrics.draft_acceptance_rate:.3f}, "
                f"System efficiency: {metrics.system_efficiency:.3f}, "
                f"Number of speculative tokens: {metrics.num_spec_tokens}, "
                f"Number of accepted tokens: {metrics.accepted_tokens}, "
499
500
                f"Number of draft tokens: {metrics.draft_tokens}, "
                f"Number of emitted tokens: {metrics.emitted_tokens}.")
501

502
503
504
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        raise NotImplementedError

505
506
507
508

class PrometheusStatLogger(StatLoggerBase):
    """PrometheusStatLogger is used LLMEngine to log to Promethus."""
    _metrics_cls = Metrics
509
    _gauge_cls = prometheus_client.Gauge
510
511

    def __init__(self, local_interval: float, labels: Dict[str, str],
512
513
                 vllm_config: VllmConfig) -> None:
        super().__init__(local_interval, vllm_config)
514
515
        # Prometheus metrics
        self.labels = labels
516
        self.metrics = self._metrics_cls(labelnames=list(labels.keys()),
517
                                         vllm_config=vllm_config)
518

519
520
521
522
523
        # Use this flag to hide metrics that were deprecated in
        # a previous release and which will be removed future
        self.show_hidden_metrics = \
            vllm_config.observability_config.show_hidden_metrics

524
525
526
    def _log_gauge(self, gauge, data: Union[int, float]) -> None:
        # Convenience function for logging to gauge.
        gauge.labels(**self.labels).set(data)
527

528
529
    def _log_counter(self, counter, data: Union[int, float]) -> None:
        # Convenience function for logging to counter.
530
531
532
533
534
        # Prevent ValueError from negative increment
        if data < 0:
            logger.warning("Skipping negative increment of %g to %s", data,
                           counter)
            return
535
536
537
538
539
540
541
542
543
544
545
546
547
        counter.labels(**self.labels).inc(data)

    def _log_counter_labels(self, counter, data: CollectionsCounter,
                            label_key: str) -> None:
        # Convenience function for collection counter of labels.
        for label, count in data.items():
            counter.labels(**{**self.labels, label_key: label}).inc(count)

    def _log_histogram(self, histogram, data: Union[List[int],
                                                    List[float]]) -> None:
        # Convenience function for logging list to histogram.
        for datum in data:
            histogram.labels(**self.labels).observe(datum)
548

549
    def _log_gauge_string(self, gauge, data: Dict[str, str]) -> None:
550
        gauge.labels(**data).set_to_current_time()
551

552
    def _log_prometheus(self, stats: Stats) -> None:
553
554
555
556
557
558
559
560
561
562
563
        # System state data
        self._log_gauge(self.metrics.gauge_scheduler_running,
                        stats.num_running_sys)
        self._log_gauge(self.metrics.gauge_scheduler_swapped,
                        stats.num_swapped_sys)
        self._log_gauge(self.metrics.gauge_scheduler_waiting,
                        stats.num_waiting_sys)
        self._log_gauge(self.metrics.gauge_gpu_cache_usage,
                        stats.gpu_cache_usage_sys)
        self._log_gauge(self.metrics.gauge_cpu_cache_usage,
                        stats.cpu_cache_usage_sys)
564
565
566
567
        self._log_gauge(self.metrics.gauge_cpu_prefix_cache_hit_rate,
                        stats.cpu_prefix_cache_hit_rate)
        self._log_gauge(self.metrics.gauge_gpu_prefix_cache_hit_rate,
                        stats.gpu_prefix_cache_hit_rate)
568
569
570
571
572
573
574
575
576
577
578
        # Including max-lora in metric, in future this property of lora
        # config maybe extended to be dynamic.
        lora_info = {
            self.metrics.labelname_running_lora_adapters:
            ",".join(stats.running_lora_adapters),
            self.metrics.labelname_waiting_lora_adapters:
            ",".join(stats.waiting_lora_adapters),
            self.metrics.labelname_max_lora:
            stats.max_lora,
        }
        self._log_gauge_string(self.metrics.gauge_lora_info, lora_info)
579
        # Iteration level data
580
581
        self._log_counter(self.metrics.counter_num_preemption,
                          stats.num_preemption_iter)
582
583
584
585
        self._log_counter(self.metrics.counter_prompt_tokens,
                          stats.num_prompt_tokens_iter)
        self._log_counter(self.metrics.counter_generation_tokens,
                          stats.num_generation_tokens_iter)
harrywu's avatar
harrywu committed
586
587
        self._log_histogram(self.metrics.histogram_iteration_tokens,
                            [stats.num_tokens_iter])
588
589
590
591
592
593
594
595
596
        self._log_histogram(self.metrics.histogram_time_to_first_token,
                            stats.time_to_first_tokens_iter)
        self._log_histogram(self.metrics.histogram_time_per_output_token,
                            stats.time_per_output_tokens_iter)

        # Request level data
        # Latency
        self._log_histogram(self.metrics.histogram_e2e_time_request,
                            stats.time_e2e_requests)
harrywu's avatar
harrywu committed
597
598
599
600
601
        self._log_histogram(self.metrics.histogram_queue_time_request,
                            stats.time_queue_requests)
        self._log_histogram(self.metrics.histogram_inference_time_request,
                            stats.time_inference_requests)
        self._log_histogram(self.metrics.histogram_prefill_time_request,
602
603
                            stats.time_prefill_requests)
        self._log_histogram(self.metrics.histogram_decode_time_request,
harrywu's avatar
harrywu committed
604
                            stats.time_decode_requests)
605
606
607
608
609
610
        self._log_histogram(self.metrics.histogram_time_in_queue_request,
                            stats.time_in_queue_requests)
        self._log_histogram(self.metrics.histogram_model_forward_time_request,
                            stats.model_forward_time_requests)
        self._log_histogram(self.metrics.histogram_model_execute_time_request,
                            stats.model_execute_time_requests)
611
612
613
614
615
616
617
618
619
620
621
622
        # Metadata
        finished_reason_counter = CollectionsCounter(
            stats.finished_reason_requests)
        self._log_counter_labels(self.metrics.counter_request_success,
                                 finished_reason_counter,
                                 Metrics.labelname_finish_reason)
        self._log_histogram(self.metrics.histogram_num_prompt_tokens_request,
                            stats.num_prompt_tokens_requests)
        self._log_histogram(
            self.metrics.histogram_num_generation_tokens_request,
            stats.num_generation_tokens_requests)
        self._log_histogram(self.metrics.histogram_n_request, stats.n_requests)
harrywu's avatar
harrywu committed
623
624
625
        self._log_histogram(
            self.metrics.histogram_max_num_generation_tokens_request,
            stats.max_num_generation_tokens_requests)
626
627
        self._log_histogram(self.metrics.histogram_max_tokens_request,
                            stats.max_tokens_requests)
628

629
630
    def log(self, stats: Stats):
        """Logs to prometheus and tracked stats every iteration."""
631
632
633
634
        # Log to prometheus.
        self._log_prometheus(stats)

        # Save tracked stats for token counters.
635
636
        self.num_prompt_tokens.append(stats.num_prompt_tokens_iter)
        self.num_generation_tokens.append(stats.num_generation_tokens_iter)
637

638
639
640
        # Update spec decode metrics
        self.maybe_update_spec_decode_metrics(stats)

641
        # Log locally every local_interval seconds.
642
643
        if local_interval_elapsed(stats.now, self.last_local_log,
                                  self.local_interval):
644
            if self.spec_decode_metrics is not None:
645
646
                self._log_gauge(
                    self.metrics.gauge_spec_decode_draft_acceptance_rate,
647
                    self.spec_decode_metrics.draft_acceptance_rate)
648
                self._log_gauge(self.metrics.gauge_spec_decode_efficiency,
649
                                self.spec_decode_metrics.system_efficiency)
650
651
                self._log_counter(
                    self.metrics.counter_spec_decode_num_accepted_tokens,
652
                    self.spec_decode_metrics.accepted_tokens)
653
654
                self._log_counter(
                    self.metrics.counter_spec_decode_num_draft_tokens,
655
                    self.spec_decode_metrics.draft_tokens)
656
657
                self._log_counter(
                    self.metrics.counter_spec_decode_num_emitted_tokens,
658
659
660
661
662
663
664
                    self.spec_decode_metrics.emitted_tokens)

            # Reset tracked stats for next interval.
            self.num_prompt_tokens = []
            self.num_generation_tokens = []
            self.last_local_log = stats.now
            self.spec_decode_metrics = None
665

666
667
668
669
670
671
672
673
674
675
676
677
678
    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        # Info type metrics are syntactic sugar for a gauge permanently set to 1
        # Since prometheus multiprocessing mode does not support Info, emulate
        # info here with a gauge.
        if type == "cache_config":
            metrics_info = obj.metrics_info()
            info_gauge = self._gauge_cls(
                name="vllm:cache_config_info",
                documentation="Information of the LLMEngine CacheConfig",
                labelnames=metrics_info.keys(),
                multiprocess_mode="mostrecent")
            info_gauge.labels(**metrics_info).set(1)

679

680
681
class RayPrometheusStatLogger(PrometheusStatLogger):
    """RayPrometheusStatLogger uses Ray metrics instead."""
682
    _metrics_cls = RayMetrics
683
684
685

    def info(self, type: str, obj: SupportsMetricsInfo) -> None:
        return None