vllm.py 80.6 KB
Newer Older
1
2
3
4
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import copy
5
import getpass
6
7
import json
import os
8
9
import tempfile
import threading
10
import time
11
from contextlib import contextmanager
12
from dataclasses import is_dataclass
13
from datetime import datetime
14
from enum import IntEnum
15
from functools import lru_cache
16
from importlib.metadata import version
17
from pathlib import Path
18
from typing import TYPE_CHECKING, Any, Literal, TypeVar, get_args
19
20

import torch
21
from packaging.version import Version
22
from pydantic import ConfigDict, Field, model_validator
23
24

import vllm.envs as envs
25
from vllm.logger import enable_trace_function_call, init_logger
26
27
from vllm.transformers_utils.runai_utils import is_runai_obj_uri
from vllm.utils import random_uuid
28
from vllm.utils.hashing import safe_hash
29

30
from .attention import AttentionConfig
31
from .cache import CacheConfig
32
from .compilation import CompilationConfig, CompilationMode, CUDAGraphMode
33
from .device import DeviceConfig
34
from .ec_transfer import ECTransferConfig
35
from .kernel import KernelConfig
36
37
38
39
40
41
from .kv_events import KVEventsConfig
from .kv_transfer import KVTransferConfig
from .load import LoadConfig
from .lora import LoRAConfig
from .model import ModelConfig
from .observability import ObservabilityConfig
42
from .offload import OffloadConfig
43
from .parallel import ParallelConfig
44
from .profiler import ProfilerConfig
45
from .reasoning import ReasoningConfig
46
from .scheduler import SchedulerConfig
47
from .speculative import EagleModelTypes, NgramGPUTypes, SpeculativeConfig
48
from .structured_outputs import StructuredOutputsConfig
49
from .utils import SupportsHash, config, replace
50
from .weight_transfer import WeightTransferConfig
51
52
53
54

if TYPE_CHECKING:
    from transformers import PretrainedConfig

55
    from vllm.model_executor.layers.quantization.base_config import QuantizationConfig
56
    from vllm.v1.kv_cache_interface import KVCacheConfig
57
58
59
60
61
else:
    PretrainedConfig = Any

    QuantizationConfig = Any

62
63
    KVCacheConfig = Any

64
65
66
logger = init_logger(__name__)


67
68
69
70
71
72
73
class OptimizationLevel(IntEnum):
    """Optimization level enum."""

    O0 = 0
    """O0 : No optimization. no compilation, no cudagraphs, no other
    optimization, just starting up immediately"""
    O1 = 1
74
    """O1: Quick optimizations. Dynamo+Inductor compilation and Piecewise
75
76
77
78
79
80
81
    cudagraphs"""
    O2 = 2
    """O2: Full optimizations. -O1 as well as Full and Piecewise cudagraphs."""
    O3 = 3
    """O3: Currently the same as -O2s."""


82
83
PerformanceMode = Literal["balanced", "interactivity", "throughput"]

84
85
86
87
88
89
90
91
92
93
IS_QUANTIZED = False
IS_DENSE = False
# The optimizations that depend on these properties currently set to False
# in all cases.
# if model_config is not None:
#     IS_QUANTIZED = lambda c: c.model_config.is_quantized()
#     IS_DENSE = lambda c: not c.model_config.is_model_moe()
# See https://github.com/vllm-project/vllm/issues/25689.


94
95
96
97
def enable_norm_fusion(cfg: "VllmConfig") -> bool:
    """Enable if either RMS norm or quant FP8 custom op is active;
    otherwise Inductor handles fusion."""

98
99
100
101
102
    return cfg.compilation_config.is_custom_op_enabled(
        "rms_norm"
    ) or cfg.compilation_config.is_custom_op_enabled("quant_fp8")


103
def enable_act_fusion(cfg: "VllmConfig") -> bool:
104
105
106
107
108
109
110
111
112
113
    """
    Enable if either SiLU+Mul or quant FP8 custom op is active;
    otherwise Inductor handles fusion.
    Also enable for FP4 models as FP4 quant is always custom so Inductor cannot fuse it.
    """
    return (
        cfg.compilation_config.is_custom_op_enabled("silu_and_mul")
        or cfg.compilation_config.is_custom_op_enabled("quant_fp8")
        or (cfg.model_config is not None and cfg.model_config.is_nvfp4_quantized())
    )
114
115


116
def enable_allreduce_rms_fusion(cfg: "VllmConfig") -> bool:
117
    """Enable if TP > 1 and Hopper/Blackwell and flashinfer installed."""
118
119
120
121
122
123
124
    from vllm.platforms import current_platform
    from vllm.utils.flashinfer import has_flashinfer

    return (
        cfg.parallel_config.tensor_parallel_size > 1
        and current_platform.is_cuda()
        and has_flashinfer()
125
        and (
126
            current_platform.is_device_capability_family(100)
127
128
129
130
131
            or current_platform.is_device_capability(90)
        )
        # tp-dp combination broken:
        # https://github.com/vllm-project/vllm/issues/34458
        and cfg.parallel_config.data_parallel_size == 1
132
133
134
        # tp-pp combination broken:
        # https://github.com/vllm-project/vllm/issues/35426
        and cfg.parallel_config.pipeline_parallel_size == 1
135
136
137
    )


138
139
140
141
142
143
144
145
146
def enable_rope_kvcache_fusion(cfg: "VllmConfig") -> bool:
    """Enable if rotary embedding custom op is active and
    use_inductor_graph_partition is enabled.
    """
    from vllm._aiter_ops import rocm_aiter_ops

    return (
        rocm_aiter_ops.is_enabled()
        and cfg.compilation_config.is_custom_op_enabled("rotary_embedding")
147
148
149
150
        and (
            cfg.compilation_config.use_inductor_graph_partition
            or not cfg.compilation_config.splitting_ops_contain_kv_cache_update()
        )
151
152
153
    )


154
def enable_norm_pad_fusion(cfg: "VllmConfig") -> bool:
155
    """Enable if using AITER RMSNorm and hidden size is 2880 i.e. gpt-oss."""
156
    from vllm._aiter_ops import rocm_aiter_ops
157
158

    return (
159
        rocm_aiter_ops.is_rmsnorm_enabled()
160
        and cfg.model_config is not None
161
162
163
164
        and cfg.model_config.get_hidden_size() == 2880
    )


165
166
167
OPTIMIZATION_LEVEL_00 = {
    "compilation_config": {
        "pass_config": {
168
169
170
171
172
173
            "fuse_norm_quant": False,
            "fuse_act_quant": False,
            "fuse_allreduce_rms": False,
            "fuse_attn_quant": False,
            "enable_sp": False,
            "fuse_gemm_comms": False,
174
            "fuse_act_padding": False,
175
            "fuse_rope_kvcache": False,
176
177
178
179
        },
        "cudagraph_mode": CUDAGraphMode.NONE,
        "use_inductor_graph_partition": False,
    },
180
181
182
    "kernel_config": {
        "enable_flashinfer_autotune": False,
    },
183
184
185
186
}
OPTIMIZATION_LEVEL_01 = {
    "compilation_config": {
        "pass_config": {
187
188
189
190
191
192
            "fuse_norm_quant": enable_norm_fusion,
            "fuse_act_quant": enable_act_fusion,
            "fuse_allreduce_rms": False,
            "fuse_attn_quant": False,
            "enable_sp": False,
            "fuse_gemm_comms": False,
193
            "fuse_act_padding": enable_norm_pad_fusion,
194
            "fuse_rope_kvcache": False,
195
196
197
198
        },
        "cudagraph_mode": CUDAGraphMode.PIECEWISE,
        "use_inductor_graph_partition": False,
    },
199
200
201
    "kernel_config": {
        "enable_flashinfer_autotune": True,
    },
202
203
204
205
}
OPTIMIZATION_LEVEL_02 = {
    "compilation_config": {
        "pass_config": {
206
207
            "fuse_norm_quant": enable_norm_fusion,
            "fuse_act_quant": enable_act_fusion,
208
            "fuse_allreduce_rms": enable_allreduce_rms_fusion,
209
210
211
            "fuse_attn_quant": IS_QUANTIZED,
            "enable_sp": IS_DENSE,
            "fuse_gemm_comms": IS_DENSE,
212
            "fuse_act_padding": enable_norm_pad_fusion,
213
            "fuse_rope_kvcache": enable_rope_kvcache_fusion,
214
215
216
217
        },
        "cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
        "use_inductor_graph_partition": False,
    },
218
219
220
    "kernel_config": {
        "enable_flashinfer_autotune": True,
    },
221
222
223
224
}
OPTIMIZATION_LEVEL_03 = {
    "compilation_config": {
        "pass_config": {
225
226
            "fuse_norm_quant": enable_norm_fusion,
            "fuse_act_quant": enable_act_fusion,
227
            "fuse_allreduce_rms": enable_allreduce_rms_fusion,
228
229
230
            "fuse_attn_quant": IS_QUANTIZED,
            "enable_sp": IS_DENSE,
            "fuse_gemm_comms": IS_DENSE,
231
            "fuse_act_padding": enable_norm_pad_fusion,
232
            "fuse_rope_kvcache": enable_rope_kvcache_fusion,
233
234
235
236
        },
        "cudagraph_mode": CUDAGraphMode.FULL_AND_PIECEWISE,
        "use_inductor_graph_partition": False,
    },
237
238
239
    "kernel_config": {
        "enable_flashinfer_autotune": True,
    },
240
241
242
243
244
245
246
247
248
249
}

OPTIMIZATION_LEVEL_TO_CONFIG = {
    OptimizationLevel.O0: OPTIMIZATION_LEVEL_00,
    OptimizationLevel.O1: OPTIMIZATION_LEVEL_01,
    OptimizationLevel.O2: OPTIMIZATION_LEVEL_02,
    OptimizationLevel.O3: OPTIMIZATION_LEVEL_03,
}


250
251
@config(config=ConfigDict(arbitrary_types_allowed=True))
class VllmConfig:
252
253
254
255
256
257
    """Dataclass which contains all vllm-related configuration. This
    simplifies passing around the distinct configurations in the codebase.
    """

    # TODO: use default_factory once default constructing ModelConfig doesn't
    # try to download a model
258
    model_config: ModelConfig = None  # type: ignore[assignment]
259
    """Model configuration."""
260
    cache_config: CacheConfig = Field(default_factory=CacheConfig)
261
    """Cache configuration."""
262
    parallel_config: ParallelConfig = Field(default_factory=ParallelConfig)
263
    """Parallel configuration."""
264
265
266
    scheduler_config: SchedulerConfig = Field(
        default_factory=SchedulerConfig.default_factory,
    )
267
    """Scheduler configuration."""
268
    device_config: DeviceConfig = Field(default_factory=DeviceConfig)
269
    """Device configuration."""
270
    load_config: LoadConfig = Field(default_factory=LoadConfig)
271
    """Load configuration."""
272
273
    offload_config: OffloadConfig = Field(default_factory=OffloadConfig)
    """Model weight offloading configuration."""
274
275
    attention_config: AttentionConfig = Field(default_factory=AttentionConfig)
    """Attention configuration."""
276
277
    kernel_config: KernelConfig = Field(default_factory=KernelConfig)
    """Kernel configuration."""
278
    lora_config: LoRAConfig | None = None
279
    """LoRA configuration."""
280
    speculative_config: SpeculativeConfig | None = None
281
    """Speculative decoding configuration."""
282
    structured_outputs_config: StructuredOutputsConfig = Field(
283
284
        default_factory=StructuredOutputsConfig
    )
285
    """Structured outputs configuration."""
286
287
288
    observability_config: ObservabilityConfig = Field(
        default_factory=ObservabilityConfig
    )
289
    """Observability configuration."""
290
    quant_config: QuantizationConfig | None = None
291
    """Quantization configuration."""
292
    compilation_config: CompilationConfig = Field(default_factory=CompilationConfig)
293
294
    """`torch.compile` and cudagraph capture configuration for the model.

295
296
    As a shorthand, one can append compilation arguments via
    -cc.parameter=argument such as `-cc.mode=3` (same as `-cc='{"mode":3}'`).
297
298

    You can specify the full compilation config like so:
299
    `{"mode": 3, "cudagraph_capture_sizes": [1, 2, 4, 8]}`
300
    """
301
302
    profiler_config: ProfilerConfig = Field(default_factory=ProfilerConfig)
    """Profiling configuration."""
303
    kv_transfer_config: KVTransferConfig | None = None
304
    """The configurations for distributed KV cache transfer."""
305
    kv_events_config: KVEventsConfig | None = None
306
    """The configurations for event publishing."""
307
308
    ec_transfer_config: ECTransferConfig | None = None
    """The configurations for distributed EC cache transfer."""
309
310
    reasoning_config: ReasoningConfig | None = None
    """The configurations for reasoning model."""
311
312
313
    # some opaque config, only used to provide additional information
    # for the hash computation, mainly used for testing, debugging or out of
    # tree config registration.
314
    additional_config: dict | SupportsHash = Field(default_factory=dict)
315
316
317
318
319
    """Additional config for specified platform. Different platforms may
    support different configs. Make sure the configs are valid for the platform
    you are using. Contents must be hashable."""
    instance_id: str = ""
    """The ID of the vLLM instance."""
320
321
322
    optimization_level: OptimizationLevel = OptimizationLevel.O2
    """The optimization level. These levels trade startup time cost for
    performance, with -O0 having the best startup time and -O3 having the best
323
    performance. -O2 is used by default. See OptimizationLevel for full
324
    description."""
325

326
327
328
329
330
331
332
    performance_mode: PerformanceMode = "balanced"
    """Performance mode for runtime behavior, 'balanced' is the default.
    'interactivity' favors low end-to-end per-request latency at small batch
    sizes (fine-grained CUDA graphs, latency-oriented kernels).
    'throughput' favors aggregate tokens/sec at high concurrency (larger CUDA
    graphs, more aggressive batching, throughput-oriented kernels)."""

333
334
335
    weight_transfer_config: WeightTransferConfig | None = None
    """The configurations for weight transfer during RL training."""

336
337
338
339
340
341
    shutdown_timeout: int = Field(default=0, ge=0)
    """Shutdown grace period for in-flight requests. Shutdown will be delayed for
    up to this amount of time to allow already-running requests to complete. Any
    remaining requests are aborted once the timeout is reached.
    """

342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
    def compute_hash(self) -> str:
        """
        WARNING: Whenever a new field is added to this config,
        ensure that it is included in the factors list if
        it affects the computation graph.

        Provide a hash that uniquely identifies all the configs
        that affect the structure of the computation
        graph from input ids/embeddings to the final hidden states,
        excluding anything before input ids/embeddings and after
        the final hidden states.
        """
        factors: list[Any] = []

        # summarize vllm config
        vllm_factors: list[Any] = []
        from vllm import __version__
359

360
361
362
        vllm_factors.append(__version__)
        if self.model_config:
            vllm_factors.append(self.model_config.compute_hash())
363
364
365
366
367
368
            if (
                self.compilation_config
                and getattr(self.compilation_config, "compile_mm_encoder", False)
                and self.model_config.multimodal_config
            ):
                vllm_factors.append(self.model_config.multimodal_config.compute_hash())
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
        else:
            vllm_factors.append("None")
        if self.cache_config:
            vllm_factors.append(self.cache_config.compute_hash())
        else:
            vllm_factors.append("None")
        if self.parallel_config:
            vllm_factors.append(self.parallel_config.compute_hash())
        else:
            vllm_factors.append("None")
        if self.scheduler_config:
            vllm_factors.append(self.scheduler_config.compute_hash())
        else:
            vllm_factors.append("None")
        if self.device_config:
            vllm_factors.append(self.device_config.compute_hash())
        else:
            vllm_factors.append("None")
        if self.load_config:
            vllm_factors.append(self.load_config.compute_hash())
        else:
            vllm_factors.append("None")
391
392
393
394
        if self.offload_config:
            vllm_factors.append(self.offload_config.compute_hash())
        else:
            vllm_factors.append("None")
395
396
397
398
        if self.attention_config:
            vllm_factors.append(self.attention_config.compute_hash())
        else:
            vllm_factors.append("None")
399
400
401
402
403
404
405
406
407
408
        if self.lora_config:
            vllm_factors.append(self.lora_config.compute_hash())
        else:
            vllm_factors.append("None")
        if self.speculative_config:
            vllm_factors.append(self.speculative_config.compute_hash())
        else:
            vllm_factors.append("None")
        if self.structured_outputs_config:
            vllm_factors.append(self.structured_outputs_config.compute_hash())
409
410
        if self.profiler_config:
            vllm_factors.append(self.profiler_config.compute_hash())
411
412
        else:
            vllm_factors.append("None")
413
        vllm_factors.append(self.observability_config.compute_hash())
414
415
416
417
418
419
420
421
422
423
        if self.quant_config:
            pass  # should be captured by model_config.quantization
        if self.compilation_config:
            vllm_factors.append(self.compilation_config.compute_hash())
        else:
            vllm_factors.append("None")
        if self.kv_transfer_config:
            vllm_factors.append(self.kv_transfer_config.compute_hash())
        else:
            vllm_factors.append("None")
424
425
426
427
        if self.ec_transfer_config:
            vllm_factors.append(self.ec_transfer_config.compute_hash())
        else:
            vllm_factors.append("None")
428
429
        if self.additional_config:
            if isinstance(additional_config := self.additional_config, dict):
430
                additional_config_hash = safe_hash(
431
432
433
434
435
436
437
438
439
440
                    json.dumps(additional_config, sort_keys=True).encode(),
                    usedforsecurity=False,
                ).hexdigest()
            else:
                additional_config_hash = additional_config.compute_hash()
            vllm_factors.append(additional_config_hash)
        else:
            vllm_factors.append("None")
        factors.append(vllm_factors)

441
442
443
        hash_str = safe_hash(str(factors).encode(), usedforsecurity=False).hexdigest()[
            :10
        ]
444
445
        return hash_str

446
447
448
449
450
451
452
453
454
    @property
    def num_speculative_tokens(self) -> int:
        if (
            self.speculative_config is not None
            and self.speculative_config.num_speculative_tokens is not None
        ):
            return self.speculative_config.num_speculative_tokens
        return 0

455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
    @property
    def needs_dp_coordinator(self) -> bool:
        """
        Determine if the DPCoordinator process is needed.

        The DPCoordinator is needed in two cases:
        1. For MoE models with DP > 1: to handle wave coordination
           (even in external LB mode, since wave coordination runs in the coordinator)
        2. For non-MoE models in internal/hybrid LB mode: to collect and publish
           queue stats for load balancing across DP ranks

        Returns:
            True if DPCoordinator process is needed, False otherwise.
        """

        # For non-MoE models, only need coordinator in internal/hybrid LB mode
        # (for stats collection).
        return self.parallel_config.data_parallel_size > 1 and (
            self.model_config is None
            or self.model_config.is_moe
            or not self.parallel_config.data_parallel_external_lb
        )

478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
    def enable_trace_function_call_for_thread(self) -> None:
        """
        Set up function tracing for the current thread,
        if enabled via the `VLLM_TRACE_FUNCTION` environment variable.
        """
        if envs.VLLM_TRACE_FUNCTION:
            tmp_dir = tempfile.gettempdir()
            # add username to tmp_dir to avoid permission issues
            tmp_dir = os.path.join(tmp_dir, getpass.getuser())
            filename = (
                f"VLLM_TRACE_FUNCTION_for_process_{os.getpid()}"
                f"_thread_{threading.get_ident()}_at_{datetime.now()}.log"
            ).replace(" ", "_")
            log_path = os.path.join(
                tmp_dir,
                "vllm",
                f"vllm-instance-{self.instance_id}",
                filename,
            )
            os.makedirs(os.path.dirname(log_path), exist_ok=True)
            enable_trace_function_call(log_path)

500
501
    @staticmethod
    def _get_quantization_config(
502
        model_config: ModelConfig, load_config: LoadConfig
503
    ) -> QuantizationConfig | None:
504
505
        """Get the quantization config."""
        from vllm.platforms import current_platform
506

507
        if model_config.quantization is not None:
508
509
            from vllm.model_executor.model_loader.weight_utils import get_quant_config

510
511
512
513
514
515
516
517
518
519
            quant_config = get_quant_config(model_config, load_config)
            capability_tuple = current_platform.get_device_capability()

            if capability_tuple is not None:
                capability = capability_tuple.to_int()
                if capability < quant_config.get_min_capability():
                    raise ValueError(
                        f"The quantization method {model_config.quantization} "
                        "is not supported for the current GPU. Minimum "
                        f"capability: {quant_config.get_min_capability()}. "
520
521
                        f"Current capability: {capability}."
                    )
522
523
524
525
526
            supported_dtypes = quant_config.get_supported_act_dtypes()
            if model_config.dtype not in supported_dtypes:
                raise ValueError(
                    f"{model_config.dtype} is not supported for quantization "
                    f"method {model_config.quantization}. Supported dtypes: "
527
528
                    f"{supported_dtypes}"
                )
529
530
531
532
533
534
            quant_config.maybe_update_config(model_config.model)
            return quant_config
        return None

    @staticmethod
    def get_quantization_config(
535
        model_config: ModelConfig, load_config: LoadConfig
536
    ) -> QuantizationConfig | None:
537
538
539
540
        import copy

        # For some reason, the _ version of this modifies the model_config
        # object, so using deepcopy to avoid this problem.
541
542
543
        return VllmConfig._get_quantization_config(
            copy.deepcopy(model_config), load_config
        )
544
545
546
547

    def with_hf_config(
        self,
        hf_config: PretrainedConfig,
548
        architectures: list[str] | None = None,
549
550
551
552
553
554
    ) -> "VllmConfig":
        if architectures is not None:
            hf_config = copy.deepcopy(hf_config)
            hf_config.architectures = architectures

        model_config = copy.deepcopy(self.model_config)
555

556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
        # In Transformers v5, tie_word_embeddings belongs to the config of the class
        # that can see both layers to be tied. For example:
        #
        # SomeVLModel:
        #   self.language_model = SomeLanguageModel(SomeVLTextConfig)
        #   self.vision_model = SomeVisionModel(SomeVLVisionConfig)
        #
        # SomeVLModelForMultimodalLM:
        #   self.model = SomeVLModel(SomeVLConfig)
        #   self.lm_head = nn.Linear()
        #
        # Therefore, tie_word_embeddings is defined in SomeVLConfig and is not present
        # in SomeVLTextConfig*. In vLLM, the lm_head belongs to the language_model, so
        # we must ensure that tie_word_embeddings is set in the language_model's config.
        #
        # *For some models, SomeVLTextConfig may also have a tie_word_embeddings field.
        # This is only the case if SomeVLTextConfig is also used for a text only version
        # of the same model. For example:
        #
        # SomeVLModelForCausalLM:
        #   self.model = SomeLanguageModel(SomeVLTextConfig)
        #   self.lm_head = nn.Linear()
        #
        # Therefore, the presence of tie_word_embeddings in SomeVLTextConfig cannot
        # be used as a signal for whether tie_word_embeddings should be copied from
        # hf_config to the language_model config.
582
        if (
583
584
            Version(version("transformers")) >= Version("5.0.0")
            and model_config.is_multimodal_model
585
586
587
588
589
            and hasattr(model_config.hf_config, "tie_word_embeddings")
        ):
            tie_word_embeddings = model_config.hf_config.tie_word_embeddings
            hf_config.get_text_config().tie_word_embeddings = tie_word_embeddings

590
        model_config.hf_config = hf_config
591
        model_config.model_arch_config = model_config.get_model_arch_config()
592
593
594

        return replace(self, model_config=model_config)

595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
    def _set_config_default(self, config_obj: Any, key: str, value: Any) -> None:
        """Set config attribute to default if not already set by user.

        Args:
            config_obj: Configuration object to update.
            key: Attribute name.
            value: Default value (static or callable).
        """
        if getattr(config_obj, key) is None:
            # Some config values are known before initialization and are
            # hard coded.
            # Other values depend on the user given configuration, so they are
            # implemented with lambda functions and decided at run time.
            setattr(config_obj, key, value(self) if callable(value) else value)

    def _apply_optimization_level_defaults(self, defaults: dict[str, Any]) -> None:
        """Apply optimization level defaults using self as root.

        Recursively applies values from defaults into nested config objects.
        Only fields present in defaults are overwritten.

        If the user configuration does not specify a value for a default field
        and if the default field is still None after all user selections are
Jiayi Yan's avatar
Jiayi Yan committed
618
        applied, then default values will be applied to the field. User specified
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
        fields will not be overridden by the default.

        Args:
            defaults: Dictionary of default values to apply.
        """

        def apply_recursive(config_obj: Any, config_defaults: dict[str, Any]) -> None:
            """Recursively apply defaults to config_obj, using self as root."""
            for key, value in config_defaults.items():
                if not hasattr(config_obj, key):
                    continue

                current = getattr(config_obj, key)
                if isinstance(value, dict) and is_dataclass(current):
                    apply_recursive(current, value)
                else:
                    self._set_config_default(config_obj, key, value)

        apply_recursive(self, defaults)

639
640
641
642
643
644
    def _post_init_kv_transfer_config(self) -> None:
        """Update KVTransferConfig based on top-level configs in VllmConfig.

        Right now, this function reads the offloading settings from
        CacheConfig and configures the KVTransferConfig accordingly.
        """
645
646
        # KV offloading is only activated when kv_offloading_size is set.
        if (kv_offloading_size := self.cache_config.kv_offloading_size) is None:
647
648
            return

649
650
        kv_offloading_backend = self.cache_config.kv_offloading_backend

651
652
653
654
655
656
657
658
659
        # If no KVTransferConfig is provided, create a default one.
        if self.kv_transfer_config is None:
            self.kv_transfer_config = KVTransferConfig()
        num_kv_ranks = (
            self.parallel_config.tensor_parallel_size
            * self.parallel_config.pipeline_parallel_size
        )

        if kv_offloading_backend == "native":
660
661
662
663
664
            if envs.VLLM_USE_SIMPLE_KV_OFFLOAD:
                config_connector = "SimpleCPUOffloadConnector"
            else:
                config_connector = "OffloadingConnector"
            self.kv_transfer_config.kv_connector = config_connector
665
            self.kv_transfer_config.kv_connector_extra_config.update(
666
                {"cpu_bytes_to_use": kv_offloading_size * (1 << 30)}
667
668
669
670
671
672
673
674
675
676
677
678
            )
        elif kv_offloading_backend == "lmcache":
            self.kv_transfer_config.kv_connector = "LMCacheConnectorV1"
            kv_gb_per_rank = kv_offloading_size / num_kv_ranks
            self.kv_transfer_config.kv_connector_extra_config = {
                "lmcache.local_cpu": True,
                "lmcache.max_local_cpu_size": kv_gb_per_rank,
            }

        # This is the same for all backends
        self.kv_transfer_config.kv_role = "kv_both"

679
    def __post_init__(self):
680
        """Verify configs are valid & consistent with each other."""
681

682
683
684
        # To give each torch profile run a unique instance name.
        self.instance_id = f"{time.time_ns()}"

685
686
687
688
689
        if self.performance_mode != "balanced":
            logger.info_once(
                "Performance mode set to '%s'.", self.performance_mode, scope="local"
            )

690
691
692
693
        self.try_verify_and_update_config()

        if self.model_config is not None:
            self.model_config.verify_with_parallel_config(self.parallel_config)
694
            self.model_config.verify_dual_chunk_attention_config(self.load_config)
695

696
697
            self.parallel_config.is_moe_model = self.model_config.is_moe

698
699
700
701
702
        if self.lora_config is not None:
            self.lora_config.verify_with_model_config(self.model_config)

        if self.quant_config is None and self.model_config is not None:
            self.quant_config = VllmConfig._get_quantization_config(
703
704
                self.model_config, self.load_config
            )
705

706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
        if (
            self.quant_config is not None
            and self.model_config is not None
            and hasattr(self.quant_config, "use_deep_gemm")
            and self.quant_config.use_deep_gemm is None
        ):
            from vllm.utils.deep_gemm import should_auto_disable_deep_gemm

            model_type = getattr(self.model_config.hf_text_config, "model_type", None)
            if should_auto_disable_deep_gemm(model_type):
                self.quant_config.use_deep_gemm = False
                logger.warning_once(
                    "Auto-disabled DeepGemm for model_type=%s on Blackwell. "
                    "DeepGemm E8M0 scale format causes accuracy degradation "
                    "for this architecture. Falling back to CUTLASS. "
                    "To disable DeepGemm globally, set VLLM_USE_DEEP_GEMM=0.",
                    model_type,
                )

725
726
        from vllm.v1.executor.abstract import Executor

727
        executor_backend = self.parallel_config.distributed_executor_backend
728
729
        executor_class = Executor.get_class(self)
        executor_supports_async_sched = executor_class.supports_async_scheduling()
730
731
732

        if self.scheduler_config.async_scheduling:
            # Async scheduling explicitly enabled, hard fail any incompatibilities.
733
734
            # Currently, async scheduling only support eagle speculative
            # decoding.
735
            if self.speculative_config is not None:
736
737
                if (
                    self.speculative_config.method not in get_args(EagleModelTypes)
738
                    and self.speculative_config.method not in get_args(NgramGPUTypes)
739
740
                    and self.speculative_config.method != "draft_model"
                ):
741
742
                    raise ValueError(
                        "Currently, async scheduling is only supported "
743
744
                        "with EAGLE/MTP/Draft Model/NGram GPU kind of "
                        "speculative decoding"
745
746
747
                    )
                if self.speculative_config.disable_padded_drafter_batch:
                    raise ValueError(
748
749
                        "Async scheduling is not compatible with "
                        "disable_padded_drafter_batch=True."
750
                    )
751
752
            if not executor_supports_async_sched:
                raise ValueError(
753
                    f"`{executor_backend}` does not support async scheduling yet."
754
755
756
                )
        elif self.scheduler_config.async_scheduling is None:
            # Enable async scheduling unless there is an incompatible option.
757
            if (
758
759
                self.speculative_config is not None
                and self.speculative_config.method not in get_args(EagleModelTypes)
760
                and self.speculative_config.method not in get_args(NgramGPUTypes)
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
            ):
                logger.warning_once(
                    "Async scheduling not supported with %s-based "
                    "speculative decoding and will be disabled.",
                    self.speculative_config.method,
                    scope="local",
                )
                self.scheduler_config.async_scheduling = False
            elif (
                self.speculative_config is not None
                and self.speculative_config.disable_padded_drafter_batch
            ):
                logger.warning_once(
                    "Async scheduling is not compatible with "
                    "disable_padded_drafter_batch=True and will be disabled.",
                    scope="local",
                )
778
                self.scheduler_config.async_scheduling = False
779
            elif not executor_supports_async_sched:
780
                logger.warning_once(
781
                    "Async scheduling will be disabled because it is not supported "
782
                    "with the `%s` distributed executor backend. ",
783
                    executor_backend,
784
                    scope="local",
785
786
787
788
789
                )
                self.scheduler_config.async_scheduling = False
            else:
                self.scheduler_config.async_scheduling = True

790
791
792
793
794
        logger.info_once(
            "Asynchronous scheduling is %s.",
            "enabled" if self.scheduler_config.async_scheduling else "disabled",
        )

795
796
        if self.parallel_config.disable_nccl_for_dp_synchronization is None:
            if self.scheduler_config.async_scheduling:
797
798
799
800
801
802
803
804
                if self.parallel_config.data_parallel_size > 1 and (
                    self.model_config is None or self.model_config.is_moe
                ):
                    logger.info_once(
                        "Disabling NCCL for DP synchronization "
                        "when using async scheduling.",
                        scope="local",
                    )
805
806
807
808
                self.parallel_config.disable_nccl_for_dp_synchronization = True
            else:
                self.parallel_config.disable_nccl_for_dp_synchronization = False

809
810
811
812
813
814
815
816
817
818
819
820
821
        if (
            self.speculative_config is not None
            and self.scheduler_config.async_scheduling
            and self.model_config is not None
            and not self.model_config.disable_cascade_attn
        ):
            logger.warning_once(
                "Disabling cascade attention (not yet compatible with "
                "async speculative decoding).",
                scope="local",
            )
            self.model_config.disable_cascade_attn = True

822
823
824
825
826
827
828
829
830
831
832
        if (
            self.model_config is not None
            and self.model_config.multimodal_config is not None
            and self.model_config.multimodal_config.mm_tensor_ipc == "torch_shm"
            and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"
        ):
            raise ValueError(
                "torch_shm is known to fail without "
                "VLLM_WORKER_MULTIPROC_METHOD set to spawn"
            )

833
        from vllm.platforms import current_platform
834
835
836

        if (
            self.model_config is not None
837
            and self.scheduler_config.enable_chunked_prefill
838
839
840
            and self.model_config.dtype == torch.float32
            and current_platform.get_device_capability() == (7, 5)
        ):
841
842
843
            logger.warning_once(
                "Turing devices tensor cores do not support float32 matmul. "
                "To workaround this limitation, vLLM will set 'ieee' input "
844
845
                "precision for chunked prefill triton kernels."
            )
846

847
848
849
850
851
852
853
        if self.model_config is not None and self.model_config.enforce_eager:
            logger.warning(
                "Enforce eager set, disabling torch.compile and CUDAGraphs. "
                "This is equivalent to setting -cc.mode=none -cc.cudagraph_mode=none"
            )
            self.compilation_config.mode = CompilationMode.NONE
            self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
854
855
856
857
858
859

        if self.compilation_config.backend == "eager" or (
            self.compilation_config.mode is not None
            and self.compilation_config.mode != CompilationMode.VLLM_COMPILE
        ):
            logger.warning(
860
861
862
                "Inductor compilation was disabled by user settings, "
                "optimizations settings that are only active during "
                "inductor compilation will be ignored."
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
            )

        def has_blocked_weights():
            if self.quant_config is not None:
                if hasattr(self.quant_config, "weight_block_size"):
                    return self.quant_config.weight_block_size is not None
                elif hasattr(self.quant_config, "has_blocked_weights"):
                    return self.quant_config.has_blocked_weights()
            return False

        # Enable quant_fp8 CUDA ops (TODO disable in follow up)
        # On H100 the CUDA kernel is faster than
        # native implementation
        # https://github.com/vllm-project/vllm/issues/25094
        if has_blocked_weights():
            custom_ops = self.compilation_config.custom_ops
            if "-quant_fp8" not in custom_ops:
                custom_ops.append("+quant_fp8")

882
883
        current_platform.apply_config_platform_defaults(self)

884
        if self.compilation_config.mode is None:
885
            if self.optimization_level > OptimizationLevel.O0:
886
                self.compilation_config.mode = CompilationMode.VLLM_COMPILE
887
            else:
888
                self.compilation_config.mode = CompilationMode.NONE
889
890
891
892

        if all(s not in self.compilation_config.custom_ops for s in ("all", "none")):
            if (
                self.compilation_config.backend == "inductor"
893
                and self.compilation_config.mode != CompilationMode.NONE
894
895
896
897
            ):
                self.compilation_config.custom_ops.append("none")
            else:
                self.compilation_config.custom_ops.append("all")
898

899
900
        default_config = OPTIMIZATION_LEVEL_TO_CONFIG[self.optimization_level]
        self._apply_optimization_level_defaults(default_config)
901
902
903
904
905
        if self.kernel_config.enable_flashinfer_autotune is None:
            raise ValueError(
                "KernelConfig.enable_flashinfer_autotune must be set after applying "
                "optimization level defaults."
            )
906

907
        if (
908
            self.compilation_config.cudagraph_mode.requires_piecewise_compilation()
909
910
911
912
913
914
915
916
917
918
            and self.compilation_config.mode != CompilationMode.VLLM_COMPILE
        ):
            logger.info(
                "Cudagraph mode %s is not compatible with compilation mode %s."
                "Overriding to NONE.",
                self.compilation_config.cudagraph_mode,
                self.compilation_config.mode,
            )
            self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE

919
920
        # async tp is built on top of sequence parallelism
        # and requires it to be enabled.
921
922
923
        if self.compilation_config.pass_config.fuse_gemm_comms:
            self.compilation_config.pass_config.enable_sp = True
        if self.compilation_config.pass_config.enable_sp:
924
925
926
927
            if self.parallel_config.tensor_parallel_size == 1:
                logger.warning("Sequence Parallelism requires TP>1, disabling")
                self.compilation_config.pass_config.enable_sp = False
                self.compilation_config.pass_config.fuse_gemm_comms = False
928
929
            else:
                # Compute SP threshold early; disable if None (model too
930
                # small for SP to be beneficial).
931
932
933
934
935
936
937
938
                pass_config = self.compilation_config.pass_config
                if pass_config.sp_min_token_num is None:
                    from vllm.compilation.passes.fusion.sequence_parallelism import (
                        get_sequence_parallelism_threshold,
                    )

                    tp_size = self.parallel_config.tensor_parallel_size
                    hidden_size = self.model_config.get_hidden_size()
939
940
                    assert isinstance(self.model_config.dtype, torch.dtype)
                    element_size = self.model_config.dtype.itemsize
941
942
943
                    pass_config.sp_min_token_num = get_sequence_parallelism_threshold(
                        hidden_size, tp_size, element_size
                    )
944

945
946
947
948
949
950
951
952
953
                if pass_config.sp_min_token_num is None:
                    logger.warning(
                        "Model hidden_size too small for the SP "
                        "threshold heuristic, disabling. To force SP, "
                        "set pass_config.sp_min_token_num manually."
                    )
                    self.compilation_config.pass_config.enable_sp = False
                    self.compilation_config.pass_config.fuse_gemm_comms = False

954
955
956
957
958
959
960
        from vllm.utils.torch_utils import HAS_OPAQUE_TYPE

        if HAS_OPAQUE_TYPE:
            # On torch >= 2.11 the hoisted OpaqueObject approach supersedes
            # fast_moe_cold_start, so force it off.
            self.compilation_config.fast_moe_cold_start = False
        elif self.compilation_config.fast_moe_cold_start is None:
961
962
963
964
965
966
967
            # resolve default behavior: try to be as safe as possible
            # this config is unsafe if any spec decoding draft model has a MOE.
            # We'll conservatively turn it off if we see spec decoding.
            self.compilation_config.fast_moe_cold_start = (
                self.speculative_config is None
            )

968
969
        self._set_max_num_scheduled_tokens()

970
        if current_platform.support_static_graph_mode():
971
            # if cudagraph_mode has full cudagraphs, we need to check support
972
973
974
975
976
            if model_config := self.model_config:
                if (
                    self.compilation_config.cudagraph_mode.has_full_cudagraphs()
                    and model_config.pooler_config is not None
                ):
977
                    logger.warning_once(
978
                        "Pooling models do not support full cudagraphs. "
979
980
981
                        "Overriding cudagraph_mode to PIECEWISE."
                    )
                    self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE
982
983
984
985
986
987
988
989
990
991
992
993
                elif (
                    model_config.is_encoder_decoder
                    and self.compilation_config.cudagraph_mode
                    not in (CUDAGraphMode.NONE, CUDAGraphMode.FULL_DECODE_ONLY)
                ):
                    logger.info_once(
                        "Encoder-decoder models do not support %s. "
                        "Overriding cudagraph_mode to FULL_DECODE_ONLY.",
                        self.compilation_config.cudagraph_mode.name,
                    )
                    self.compilation_config.cudagraph_mode = (
                        CUDAGraphMode.FULL_DECODE_ONLY
994
                    )
995

996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
            # Check if KV connector requires PIECEWISE mode for CUDA graphs
            if (
                self.kv_transfer_config is not None
                and self.kv_transfer_config.is_kv_transfer_instance
                and self.compilation_config.cudagraph_mode.has_full_cudagraphs()
            ):
                # Lazy import to avoid circular dependencies
                from vllm.distributed.kv_transfer.kv_connector.factory import (
                    KVConnectorFactory,
                )

                connector_cls = KVConnectorFactory.get_connector_class(
                    self.kv_transfer_config
                )
                if connector_cls.requires_piecewise_for_cudagraph(
                    self.kv_transfer_config.kv_connector_extra_config
                ):
                    logger.warning_once(
                        "KV connector %s requires PIECEWISE CUDA graph mode "
                        "due to layerwise async operations that cannot be "
                        "captured in CUDA graphs. "
                        "Overriding cudagraph_mode from %s to PIECEWISE.",
                        connector_cls.__name__,
                        self.compilation_config.cudagraph_mode.name,
                    )
                    self.compilation_config.cudagraph_mode = CUDAGraphMode.PIECEWISE

1023
            # disable cudagraph when enforce eager execution
1024
            if self.model_config is not None and self.model_config.enforce_eager:
1025
1026
                logger.info("Cudagraph is disabled under eager mode")
                self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE
1027
1028
1029
                # override related settings when enforce eager
                self.compilation_config.max_cudagraph_capture_size = 0
                self.compilation_config.cudagraph_capture_sizes = []
1030
            else:
1031
1032
1033
1034
1035
1036
1037
                self.compilation_config.cudagraph_num_of_warmups = 1

            self._set_cudagraph_sizes()
        else:
            self.compilation_config.cudagraph_mode = CUDAGraphMode.NONE

        if self.cache_config.kv_sharing_fast_prefill:
1038
1039
1040
1041
            if (
                self.speculative_config is not None
                and self.speculative_config.use_eagle()
            ):
1042
                raise ValueError(
1043
1044
1045
                    "Fast prefill optimization for KV sharing is not "
                    "compatible with EAGLE as EAGLE requires correct logits "
                    "for all tokens while fast prefill gives incorrect logits "
1046
1047
                    "for prompt tokens."
                )
1048
1049
1050

            logger.warning_once(
                "--kv-sharing-fast-prefill requires changes on model side for "
1051
                "correctness and to realize prefill savings."
1052
            )
1053

1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
        if (
            self.model_config
            and self.model_config.architecture == "WhisperForConditionalGeneration"
            and os.environ.get("VLLM_WORKER_MULTIPROC_METHOD") != "spawn"
        ):
            logger.warning(
                "Whisper is known to have issues with "
                "forked workers. If startup is hanging, "
                "try setting 'VLLM_WORKER_MULTIPROC_METHOD' "
                "to 'spawn'."
1064
            )
1065

1066
1067
1068
1069
1070
        if (
            self.kv_events_config is not None
            and self.kv_events_config.enable_kv_cache_events
            and not self.cache_config.enable_prefix_caching
        ):
1071
            logger.warning(
1072
                "KV cache events are on, but prefix caching is not enabled. "
1073
1074
1075
1076
1077
1078
1079
1080
                "Use --enable-prefix-caching to enable."
            )
        if (
            self.kv_events_config is not None
            and self.kv_events_config.publisher != "null"
            and not self.kv_events_config.enable_kv_cache_events
        ):
            logger.warning(
1081
1082
1083
                "KV cache events are disabled, "
                "but the scheduler is configured to publish them. "
                "Modify KVEventsConfig.enable_kv_cache_events "
1084
1085
                "to True to enable."
            )
1086
1087
        current_platform.check_and_update_config(self)

1088
1089
1090
1091
        # Re-compute compile ranges after platform-specific config updates
        # (e.g., XPU may lower max_num_batched_tokens when MLA is enabled)
        self._set_compile_ranges()

1092
        # Do this after all the updates to compilation_config.mode
1093
1094
1095
1096
1097
        effective_dp_size = (
            self.parallel_config.data_parallel_size
            if self.model_config is None or self.model_config.is_moe
            else 1
        )
1098
1099
        self.compilation_config.set_splitting_ops_for_v1(
            all2all_backend=self.parallel_config.all2all_backend,
1100
            data_parallel_size=effective_dp_size,
1101
        )
1102

1103
        if self.compilation_config.pass_config.enable_sp:
1104
1105
1106
1107
1108
            # With pipeline parallelism or dynamo partitioning,
            # native rms norm tracing errors due to incorrect residual shape.
            # Use custom rms norm to unblock. In the future,
            # the pass will operate on higher-level IR to avoid the issue.
            # TODO: https://github.com/vllm-project/vllm/issues/27894
1109
1110
1111
1112
1113
1114
1115
            if self.compilation_config.mode != CompilationMode.VLLM_COMPILE:
                logger.warning(
                    "Sequence parallelism is enabled, but running in wrong "
                    "vllm compile mode: %s.",
                    self.compilation_config.mode,
                )

1116
1117
            is_fullgraph = (
                self.compilation_config.use_inductor_graph_partition
1118
                or len(self.compilation_config.splitting_ops or []) == 0
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
            )
            if self.parallel_config.pipeline_parallel_size > 1 or not is_fullgraph:
                if "-rms_norm" not in self.compilation_config.custom_ops:
                    self.compilation_config.custom_ops.append("+rms_norm")
                else:
                    regime = (
                        "Dynamo partition"
                        if not is_fullgraph
                        else "pipeline parallelism"
                    )
                    logger.warning_once(
1130
                        "Sequence parallelism not supported with "
1131
1132
1133
1134
1135
                        "native rms_norm when using %s, "
                        "this will likely lead to an error.",
                        regime,
                    )

1136
        # final check of cudagraph mode after all possible updates
1137
        if current_platform.is_cuda_alike():
1138
1139
1140
1141
            if (
                self.compilation_config.cudagraph_mode.has_full_cudagraphs()
                and self.model_config is not None
                and not self.model_config.disable_cascade_attn
1142
                and not self.compilation_config.cudagraph_mode.has_piecewise_cudagraphs()  # noqa: E501
1143
            ):
1144
1145
1146
                logger.warning_once(
                    "No piecewise cudagraph for executing cascade attention."
                    " Will fall back to eager execution if a batch runs "
1147
                    "into cascade attentions."
1148
1149
1150
                )

            if self.compilation_config.cudagraph_mode.requires_piecewise_compilation():
1151
1152
                assert self.compilation_config.mode == CompilationMode.VLLM_COMPILE, (
                    "Compilation mode should be CompilationMode.VLLM_COMPILE "
1153
                    "when cudagraph_mode piecewise cudagraphs is used, "
1154
                    f"cudagraph_mode={self.compilation_config.cudagraph_mode}"
1155
                )
1156
1157
        if (
            self.model_config
1158
            and envs.VLLM_BATCH_INVARIANT
1159
1160
1161
1162
1163
1164
1165
            and not self.model_config.disable_cascade_attn
        ):
            self.model_config.disable_cascade_attn = True
            logger.warning_once(
                "Disabling cascade attention when VLLM_BATCH_INVARIANT is enabled.",
                scope="local",
            )
1166

1167
        if self.parallel_config.use_ubatching:
1168
            a2a_backend = self.parallel_config.all2all_backend
1169
1170
1171
1172
            assert a2a_backend in [
                "deepep_low_latency",
                "deepep_high_throughput",
            ], (
1173
1174
                "Microbatching currently only supports the deepep_low_latency and "
                f"deepep_high_throughput all2all backend. {a2a_backend} is not "
1175
1176
1177
                "supported. To fix use --all2all-backend=deepep_low_latency or "
                "--all2all-backend=deepep_high_throughput and install the DeepEP"
                " kernels."
1178
            )
1179
1180
1181

            if not self.model_config.disable_cascade_attn:
                self.model_config.disable_cascade_attn = True
1182
                logger.warning_once("Disabling cascade attention when DBO is enabled.")
1183
1184
1185
1186

        if not self.instance_id:
            self.instance_id = random_uuid()[:5]

1187
1188
1189
        if self.reasoning_config is not None and self.model_config is not None:
            self.reasoning_config.initialize_token_ids(self.model_config)

1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
        # Hybrid KV cache manager (HMA) runtime rules:
        # - Explicit enable (--no-disable-kv-cache-manager): error if runtime
        #   disables it
        # - No preference: auto-disable for unsupported features (e.g. kv connector)
        # - Explicit disable (--disable-kv-cache-manager): always respect it
        need_disable_hybrid_kv_cache_manager = False
        # logger should only print warning message for hybrid models. As we
        # can't know whether the model is hybrid or not now, so we don't log
        # warning message here and will log it later.
        if not current_platform.support_hybrid_kv_cache():
            # Hybrid KV cache manager is not supported on non-GPU platforms.
            need_disable_hybrid_kv_cache_manager = True
        if self.kv_events_config is not None:
            # Hybrid KV cache manager is not compatible with KV events.
            need_disable_hybrid_kv_cache_manager = True
        if (
            self.model_config is not None
            and self.model_config.attention_chunk_size is not None
        ):
            if (
                self.speculative_config is not None
                and self.speculative_config.use_eagle()
            ):
                # Hybrid KV cache manager is not yet supported with chunked
                # local attention + eagle.
                need_disable_hybrid_kv_cache_manager = True
            elif not envs.VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE:
                logger.warning(
                    "There is a latency regression when using chunked local"
                    " attention with the hybrid KV cache manager. Disabling"
                    " it, by default. To enable it, set the environment "
                    "VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE=1."
                )
                # Hybrid KV cache manager is not yet supported with chunked
                # local attention.
                need_disable_hybrid_kv_cache_manager = True

        if self.scheduler_config.disable_hybrid_kv_cache_manager is None:
            # Default to disable HMA, but only if the user didn't express a preference.
1229
            if self.kv_transfer_config is not None:
1230
1231
                # NOTE(Kuntai): turn HMA off for connector unless specifically enabled.
                need_disable_hybrid_kv_cache_manager = True
1232
1233
1234
1235
1236
1237
1238
                logger.warning(
                    "Turning off hybrid kv cache manager because "
                    "`--kv-transfer-config` is set. This will reduce the "
                    "performance of vLLM on LLMs with sliding window attention "
                    "or Mamba attention. If you are a developer of kv connector"
                    ", please consider supporting hybrid kv cache manager for "
                    "your connector by making sure your connector is a subclass"
1239
1240
                    " of `SupportsHMA` defined in kv_connector/v1/base.py and"
                    " use --no-disable-hybrid-kv-cache-manager to start vLLM."
1241
                )
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
            self.scheduler_config.disable_hybrid_kv_cache_manager = (
                need_disable_hybrid_kv_cache_manager
            )
        elif (
            self.scheduler_config.disable_hybrid_kv_cache_manager is False
            and need_disable_hybrid_kv_cache_manager
        ):
            raise ValueError(
                "Hybrid KV cache manager was explicitly enabled but is not "
                "supported in this configuration. Consider omitting the "
                "--no-disable-hybrid-kv-cache-manager flag to let vLLM decide"
                " automatically."
            )

        if self.scheduler_config.disable_hybrid_kv_cache_manager is None:
            # Default to enable HMA if not explicitly disabled by user or logic above.
            self.scheduler_config.disable_hybrid_kv_cache_manager = False
1259
1260

        if self.compilation_config.debug_dump_path:
1261
            self.compilation_config.debug_dump_path = (
1262
                self.compilation_config.debug_dump_path.absolute().expanduser()
1263
            )
1264
1265
1266
1267
1268
        if envs.VLLM_DEBUG_DUMP_PATH is not None:
            env_path = Path(envs.VLLM_DEBUG_DUMP_PATH).absolute().expanduser()
            if self.compilation_config.debug_dump_path:
                logger.warning(
                    "Config-specified debug dump path is overridden"
1269
1270
1271
                    " by VLLM_DEBUG_DUMP_PATH to %s",
                    env_path,
                )
1272
1273
            self.compilation_config.debug_dump_path = env_path

1274
1275
1276
1277
1278
1279
        # Enable quant_fp8 CUDA ops (TODO disable in follow up)
        # On H100 the CUDA kernel is faster than
        # native implementation
        # https://github.com/vllm-project/vllm/issues/25094
        if has_blocked_weights():
            custom_ops = self.compilation_config.custom_ops
1280
            if "-quant_fp8" not in custom_ops:
1281
1282
                custom_ops.append("+quant_fp8")

1283
1284
1285
        # Handle the KV connector configs
        self._post_init_kv_transfer_config()

1286
1287
1288
        # Log the custom passes that are enabled
        self.compilation_config.pass_config.log_enabled_passes()

1289
    def update_sizes_for_sequence_parallelism(self, possible_sizes: list) -> list:
1290
1291
1292
        # remove the sizes that not multiple of tp_size when
        # enable sequence parallelism
        removed_sizes = [
1293
1294
            size
            for size in possible_sizes
1295
1296
1297
1298
1299
1300
            if size % self.parallel_config.tensor_parallel_size != 0
        ]
        if removed_sizes:
            logger.warning(
                "Batch sizes %s are removed because they are not "
                "multiple of tp_size %d when "
1301
1302
1303
1304
                "sequence parallelism is enabled",
                removed_sizes,
                self.parallel_config.tensor_parallel_size,
            )
1305
1306

        return [
1307
1308
            size
            for size in possible_sizes
1309
1310
1311
            if size % self.parallel_config.tensor_parallel_size == 0
        ]

1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
    def _set_max_num_scheduled_tokens(self):
        """
        In most cases, the scheduler may schedule a batch with as many tokens as the
        worker is configured to handle. However for some speculative decoding methods,
        the drafter model may insert additional slots into the batch when drafting.
        To account for this, we need to decrease the max_num_scheduled_tokens by an
        upper bound on the number of slots that can be added.
        """
        if self.speculative_config is not None:
            scheduled_token_delta = (
                self.speculative_config.max_num_new_slots_for_drafting
                * self.scheduler_config.max_num_seqs
            )
            max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
            if self.scheduler_config.max_num_scheduled_tokens is None:
                self.scheduler_config.max_num_scheduled_tokens = (
                    max_num_batched_tokens - scheduled_token_delta
                )

            max_num_scheduled_tokens = self.scheduler_config.max_num_scheduled_tokens
            if max_num_batched_tokens < max_num_scheduled_tokens + (
                self.speculative_config.max_num_new_slots_for_drafting
                * self.scheduler_config.max_num_seqs
            ):
                raise ValueError(
                    f"VllmConfig received max_num_scheduled_tokens but it does not have"
                    " enough slots to support the speculative decoding settings."
                    f" It should be greater by at least {scheduled_token_delta}, but"
                    f" got {max_num_batched_tokens=} and {max_num_scheduled_tokens=}."
                )

1343
1344
1345
1346
1347
1348
1349
    def _set_cudagraph_sizes(self):
        """
        vLLM defines the default candidate list of batch sizes for CUDA graph
        capture as:

        ```python
        max_graph_size = min(max_num_seqs * 2, 512)
1350
1351
        # 1, 2, 4, then multiples of 8 up to 256 and then multiples of 16
        # up to max_graph_size
1352
        cudagraph_capture_sizes = [1, 2, 4] + list(range(8, 256, 8)) + list(
1353
            range(256, max_graph_size + 1, 16))
1354
1355

        In the end, `vllm_config.compilation_config.cudagraph_capture_sizes`
1356
        will be the final sizes to capture cudagraph (in ascending order).
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382

        These sizes are used to capture and reuse CUDA graphs for
        performance-critical paths (e.g., decoding). Capturing enables
        significantly faster kernel dispatch by avoiding Python overhead. The
        list is then filtered based on `max_num_batched_tokens` (e.g., 8192 on
        most GPUs), which controls the total allowed number of tokens in a
        batch. Since each sequence may have a variable number of tokens, the
        maximum usable batch size will depend on actual sequence lengths.

        Example:
            With `max_num_batched_tokens = 8192`, and typical sequences
            averaging ~32 tokens, most practical batch sizes fall below 256.
            However, the system will still allow capture sizes up to 512 if
            shape and memory permit.

        Note:
            If users explicitly specify cudagraph capture sizes in the
            compilation config, those will override this default logic.
            At runtime:

            - If batch size <= one of the `cudagraph_capture_sizes`, the closest
            padded CUDA graph will be used.
            - If batch size > largest `cudagraph_capture_sizes`, cudagraph will
            not be used.
        """

1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
        if (
            self.model_config is not None
            and not self.model_config.enforce_eager
            and self.compilation_config.cudagraph_mode != CUDAGraphMode.NONE
        ):
            # determine the initial max_cudagraph_capture_size
            max_cudagraph_capture_size = (
                self.compilation_config.max_cudagraph_capture_size
            )
            if max_cudagraph_capture_size is None:
1393
1394
1395
1396
1397
1398
                decode_query_len = 1
                if (
                    self.speculative_config
                    and self.speculative_config.num_speculative_tokens
                ):
                    decode_query_len += self.speculative_config.num_speculative_tokens
1399
                max_cudagraph_capture_size = min(
1400
                    self.scheduler_config.max_num_seqs * decode_query_len * 2, 512
1401
                )
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
            max_num_tokens = self.scheduler_config.max_num_batched_tokens
            max_cudagraph_capture_size = min(max_num_tokens, max_cudagraph_capture_size)

            assert max_cudagraph_capture_size >= 1, (
                "Maximum cudagraph size should be greater than or equal to 1 "
                "when using cuda graph."
            )

            # determine the cudagraph_capture_sizes
            if self.compilation_config.cudagraph_capture_sizes is not None:
                assert len(self.compilation_config.cudagraph_capture_sizes) > 0, (
                    "cudagraph_capture_sizes should contain at least one element "
                    "when using cuda graph."
                )
                # de-duplicate the sizes provided by the config
                dedup_sizes = list(set(self.compilation_config.cudagraph_capture_sizes))
1418
1419
1420
                cudagraph_capture_sizes = [
                    i for i in dedup_sizes if i <= max_num_tokens
                ]
1421
1422
                # sort to make sure the sizes are in ascending order
                cudagraph_capture_sizes.sort()
1423
            else:
1424
1425
1426
1427
1428
1429
1430
1431
1432
                if self.performance_mode == "interactivity":
                    # Fine-grained CUDA graphs at small batch sizes
                    # for minimal padding overhead
                    interactivity_max = min(max_cudagraph_capture_size, 32)
                    cudagraph_capture_sizes = list(range(1, interactivity_max + 1))
                else:
                    cudagraph_capture_sizes = [
                        i for i in [1, 2, 4] if i <= max_cudagraph_capture_size
                    ]
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
                if max_cudagraph_capture_size >= 8:
                    # Step size 8 for small batch sizes, up to 256(not included)
                    cudagraph_capture_sizes += list(
                        range(8, min(max_cudagraph_capture_size + 1, 256), 8)
                    )
                if max_cudagraph_capture_size >= 256:
                    # Step size 16 for larger batch sizes
                    cudagraph_capture_sizes += list(
                        range(256, max_cudagraph_capture_size + 1, 16)
                    )
1443
1444
                # de-duplicate and sort the sizes
                cudagraph_capture_sizes = sorted(set(cudagraph_capture_sizes))
1445

1446
1447
            if (
                self.parallel_config.tensor_parallel_size > 1
1448
                and self.compilation_config.pass_config.enable_sp
1449
            ):
1450
1451
                cudagraph_capture_sizes = self.update_sizes_for_sequence_parallelism(
                    cudagraph_capture_sizes
1452
                )
1453

1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
            # user-specific compilation_config.max_cudagraph_capture_size get
            # truncated to valid_max_size when they are inconsistent.
            valid_max_size = (
                cudagraph_capture_sizes[-1] if cudagraph_capture_sizes else 0
            )
            if (
                self.compilation_config.max_cudagraph_capture_size is not None
                and self.compilation_config.max_cudagraph_capture_size != valid_max_size
            ):
                # raise error only when both two flags are user-specified
                # and they are inconsistent with each other
                if self.compilation_config.cudagraph_capture_sizes is not None:
                    raise ValueError(
                        "customized max_cudagraph_capture_size"
                        f"(={self.compilation_config.max_cudagraph_capture_size}) "
                        "should be consistent with the max value of "
                        f"cudagraph_capture_sizes(={valid_max_size})"
                    )

                logger.warning(
                    "Truncating max_cudagraph_capture_size to %d",
                    valid_max_size,
                )
            # always set the final max_cudagraph_capture_size
            self.compilation_config.max_cudagraph_capture_size = valid_max_size

            if self.compilation_config.cudagraph_capture_sizes is not None and len(
                cudagraph_capture_sizes
            ) < len(self.compilation_config.cudagraph_capture_sizes):
                # If users have specified capture sizes, we only need to
                # compare the lens before and after modification since the modified
                # list is only the subset of the original list.
                logger.warning(
                    (
                        "cudagraph_capture_sizes specified in compilation_config"
                        " %s is overridden by config %s"
                    ),
                    self.compilation_config.cudagraph_capture_sizes,
                    cudagraph_capture_sizes,
                )
            # always write back the final sizes
            self.compilation_config.cudagraph_capture_sizes = cudagraph_capture_sizes

        else:
            # no cudagraph in use
            self.compilation_config.max_cudagraph_capture_size = 0
            self.compilation_config.cudagraph_capture_sizes = []

        # complete the remaining process.
        self.compilation_config.post_init_cudagraph_sizes()
1504

1505
1506
1507
1508
1509
    def _set_compile_ranges(self):
        """
        Set the compile ranges for the compilation config.
        """
        compilation_config = self.compilation_config
1510
        computed_compile_ranges_endpoints = []
1511

1512
1513
1514
        # The upper bound of the compile ranges is the max_num_batched_tokens.
        compile_range_end = self.scheduler_config.max_num_batched_tokens
        if compile_range_end is not None:
1515
            computed_compile_ranges_endpoints.append(compile_range_end)
1516
1517
1518
1519
1520
1521

        # Add the compile ranges for flashinfer
        if compilation_config.pass_config.fuse_allreduce_rms:
            tp_size = self.parallel_config.tensor_parallel_size
            max_size = compilation_config.pass_config.flashinfer_max_size(tp_size)
            if max_size is not None:
1522
                assert isinstance(self.model_config.dtype, torch.dtype)
1523
1524
                max_token_num = max_size // (
                    self.model_config.get_hidden_size()
1525
                    * self.model_config.dtype.itemsize
1526
                )
1527
                if compile_range_end is not None and max_token_num < compile_range_end:
1528
                    computed_compile_ranges_endpoints.append(max_token_num)
1529
1530
1531
1532
1533
1534
                else:
                    logger.debug(
                        "Max num batched tokens below allreduce-rms fusion threshold, "
                        "allreduce-rms fusion will be enabled for all num_tokens."
                    )

1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
        # Add the compile ranges for sequence parallelism
        if compilation_config.pass_config.enable_sp:
            pass_config = compilation_config.pass_config

            # Calculate min_token_num if not explicitly provided
            # User override works regardless of hidden_size
            if pass_config.sp_min_token_num is None:
                from vllm.compilation.passes.fusion.sequence_parallelism import (
                    get_sequence_parallelism_threshold,
                )

                tp_size = self.parallel_config.tensor_parallel_size
                hidden_size = self.model_config.get_hidden_size()
1548
1549
                assert isinstance(self.model_config.dtype, torch.dtype)
                element_size = self.model_config.dtype.itemsize
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
                pass_config.sp_min_token_num = get_sequence_parallelism_threshold(
                    hidden_size, tp_size, element_size
                )

            min_token_num = pass_config.sp_min_token_num
            max_num_batched_tokens = self.scheduler_config.max_num_batched_tokens
            if min_token_num is not None and (
                max_num_batched_tokens is not None
                and min_token_num < max_num_batched_tokens
                and min_token_num > 1
            ):
1561
                # Add endpoint at min_token_num - 1 to ensure SP applies
1562
1563
                # starting from min_token_num
                # This creates ranges: [1, min-1] (no SP), [min, max] (SP applies)
1564
                computed_compile_ranges_endpoints.append(min_token_num - 1)
1565

1566
1567
1568
1569
1570
1571
        if compilation_config.pass_config.fuse_rope_kvcache:
            max_token_num = (
                compilation_config.pass_config.rope_kvcache_fusion_max_token_num
            )
            if max_token_num is not None:
                if compile_range_end is not None and max_token_num < compile_range_end:
1572
                    computed_compile_ranges_endpoints.append(max_token_num)
1573
1574
1575
1576
1577
1578
1579
                else:
                    logger.debug(
                        "Max num batched tokens below rope+kvcache fusion threshold, "
                        "rope+kvcache fusion enabled for num_tokens <= %d.",
                        compile_range_end,
                    )

1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
        if compilation_config.pass_config.fuse_minimax_qk_norm:
            from vllm.compilation.passes.fusion.minimax_qk_norm_fusion import (
                MAX_TOKEN_NUM,
            )

            max_token_num = min(
                MAX_TOKEN_NUM, self.scheduler_config.max_num_batched_tokens
            )
            if compile_range_end is not None and max_token_num < compile_range_end:
                computed_compile_ranges_endpoints.append(max_token_num)
            else:
                logger.debug(
                    "Max num batched tokens below MiniMax QK norm fusion threshold, "
                    "MiniMax QK norm fusion enabled for all num_tokens."
                )

1596
1597
        if compilation_config.compile_ranges_endpoints is not None:
            for x in compilation_config.compile_ranges_endpoints:
1598
                assert isinstance(x, int)
1599
                assert x > 0, f"Invalid compile range endpoint: {x}"
1600
                if compile_range_end is not None and x < compile_range_end and x > 1:
1601
1602
1603
                    computed_compile_ranges_endpoints.append(x)
        compilation_config.compile_ranges_endpoints = sorted(
            computed_compile_ranges_endpoints
1604
1605
        )

1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
    def try_verify_and_update_config(self):
        if self.model_config is None:
            return

        # Avoid running try_verify_and_update_config multiple times
        if getattr(self.model_config, "config_updated", False):
            return
        self.model_config.config_updated = True

        architecture = self.model_config.architecture
        if architecture is None:
            return

        from vllm.model_executor.models.config import (
1620
1621
1622
1623
            MODELS_CONFIG_MAP,
            HybridAttentionMambaModelConfig,
        )

1624
1625
1626
1627
1628
1629
1630
1631
1632
        cls = MODELS_CONFIG_MAP.get(architecture, None)
        if cls is not None:
            cls.verify_and_update_config(self)

        if self.model_config.is_hybrid:
            HybridAttentionMambaModelConfig.verify_and_update_config(self)

        if self.model_config.convert_type == "classify":
            # Maybe convert ForCausalLM into ForSequenceClassification model.
1633
1634
            from vllm.model_executor.models.adapters import SequenceClassificationConfig

1635
1636
1637
            SequenceClassificationConfig.verify_and_update_config(self)

        if hasattr(self.model_config, "model_weights") and is_runai_obj_uri(
1638
1639
            self.model_config.model_weights
        ):
1640
            if self.load_config.load_format == "auto":
1641
1642
1643
1644
                logger.info(
                    "Detected Run:ai model config. "
                    "Overriding `load_format` to 'runai_streamer'"
                )
1645
                self.load_config.load_format = "runai_streamer"
1646
1647
1648
1649
            elif self.load_config.load_format not in (
                "runai_streamer",
                "runai_streamer_sharded",
            ):
1650
                raise ValueError(
1651
1652
1653
                    f"To load a model from object storage (S3/GCS/Azure), "
                    f"'load_format' must be 'runai_streamer' or "
                    f"'runai_streamer_sharded', "
1654
1655
1656
                    f"but got '{self.load_config.load_format}'. "
                    f"Model: {self.model_config.model}"
                )
1657

1658
    def compile_debug_dump_path(self) -> Path | None:
1659
        """Returns a rank-aware path for dumping
1660
1661
1662
1663
1664
        torch.compile debug information.
        """
        if self.compilation_config.debug_dump_path is None:
            return None
        tp_rank = self.parallel_config.rank
1665
1666
        dp_rank = self.parallel_config.data_parallel_index
        append_path = f"rank_{tp_rank}_dp_{dp_rank}"
1667
1668
1669
1670
1671
1672
1673
        path = self.compilation_config.debug_dump_path / append_path
        return path

    def __str__(self):
        return (
            f"model={self.model_config.model!r}, "
            f"speculative_config={self.speculative_config!r}, "
1674
1675
1676
            f"tokenizer={self.model_config.tokenizer!r}, "
            f"skip_tokenizer_init={self.model_config.skip_tokenizer_init}, "
            f"tokenizer_mode={self.model_config.tokenizer_mode}, "
1677
            f"revision={self.model_config.revision}, "
1678
            f"tokenizer_revision={self.model_config.tokenizer_revision}, "
1679
1680
1681
1682
1683
1684
1685
1686
            f"trust_remote_code={self.model_config.trust_remote_code}, "
            f"dtype={self.model_config.dtype}, "
            f"max_seq_len={self.model_config.max_model_len}, "
            f"download_dir={self.load_config.download_dir!r}, "
            f"load_format={self.load_config.load_format}, "
            f"tensor_parallel_size={self.parallel_config.tensor_parallel_size}, "  # noqa
            f"pipeline_parallel_size={self.parallel_config.pipeline_parallel_size}, "  # noqa
            f"data_parallel_size={self.parallel_config.data_parallel_size}, "  # noqa
1687
1688
            f"decode_context_parallel_size={self.parallel_config.decode_context_parallel_size}, "  # noqa
            f"dcp_comm_backend={self.parallel_config.dcp_comm_backend}, "  # noqa
1689
1690
1691
            f"disable_custom_all_reduce={self.parallel_config.disable_custom_all_reduce}, "  # noqa
            f"quantization={self.model_config.quantization}, "
            f"enforce_eager={self.model_config.enforce_eager}, "
1692
            f"enable_return_routed_experts={self.model_config.enable_return_routed_experts}, "  # noqa
1693
1694
1695
1696
1697
1698
1699
            f"kv_cache_dtype={self.cache_config.cache_dtype}, "
            f"device_config={self.device_config.device}, "
            f"structured_outputs_config={self.structured_outputs_config!r}, "
            f"observability_config={self.observability_config!r}, "
            f"seed={self.model_config.seed}, "
            f"served_model_name={self.model_config.served_model_name}, "
            f"enable_prefix_caching={self.cache_config.enable_prefix_caching}, "
1700
            f"enable_chunked_prefill={self.scheduler_config.enable_chunked_prefill}, "  # noqa
1701
            f"pooler_config={self.model_config.pooler_config!r}, "
1702
1703
            f"compilation_config={self.compilation_config!r}"
        )
1704

1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
    def validate_block_size(self) -> None:
        """Validate block_size against DCP and mamba constraints.

        Called after Platform.update_block_size_for_backend() has
        finalised block_size.
        """
        block_size = self.cache_config.block_size

        # DCP interleave-size compatibility
        if self.parallel_config.decode_context_parallel_size > 1:
            if self.parallel_config.dcp_kv_cache_interleave_size > 1 and (
                self.parallel_config.cp_kv_cache_interleave_size
                != self.parallel_config.dcp_kv_cache_interleave_size
            ):
                self.parallel_config.cp_kv_cache_interleave_size = (
                    self.parallel_config.dcp_kv_cache_interleave_size
                )
                logger.warning_once(
                    "cp_kv_cache_interleave_size is overridden by dcp_kv_cache"
                    "_interleave_size. And dcp-kv-cache-interleave-size will be "
                    "deprecated when PCP is fully supported."
                )
            assert (
                self.parallel_config.cp_kv_cache_interleave_size <= block_size
                and block_size % self.parallel_config.cp_kv_cache_interleave_size == 0
            ), (
                f"Block_size({block_size}) should be greater "
                "than or equal to and divisible by cp_kv_cache_interleave_size "
                f"({self.parallel_config.cp_kv_cache_interleave_size})."
            )

        # Mamba cache align-mode constraints
        if self.cache_config.mamba_cache_mode == "align":
            assert block_size <= self.scheduler_config.max_num_batched_tokens, (
                "In Mamba cache align mode, block_size "
                f"({block_size}) must be <= "
                "max_num_batched_tokens "
                f"({self.scheduler_config.max_num_batched_tokens})."
            )
            if self.scheduler_config.long_prefill_token_threshold > 0:
                assert self.scheduler_config.long_prefill_token_threshold >= block_size
            assert not self.scheduler_config.disable_chunked_mm_input, (
                "Chunked MM input is required because we need the flexibility "
                "to schedule a multiple of block_size tokens even if they are "
                "in the middle of a mm input"
            )

1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
    @model_validator(mode="after")
    def validate_mamba_block_size(self) -> "VllmConfig":
        if self.model_config is None:
            return self
        mamba_block_size_is_set = (
            self.cache_config.mamba_block_size is not None
            and self.cache_config.mamba_block_size != self.model_config.max_model_len
        )
        if mamba_block_size_is_set and not self.cache_config.enable_prefix_caching:
            raise ValueError(
                "--mamba-block-size can only be set with --enable-prefix-caching"
            )
        return self

1766

1767
1768
_current_vllm_config: VllmConfig | None = None
_current_prefix: str | None = None
1769
1770
1771


@contextmanager
1772
def set_current_vllm_config(
1773
    vllm_config: VllmConfig, check_compile=False, prefix: str | None = None
1774
):
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
    """
    Temporarily set the current vLLM config.
    Used during model initialization.
    We save the current vLLM config in a global variable,
    so that all modules can access it, e.g. custom ops
    can access the vLLM config to determine how to dispatch.
    """
    global _current_vllm_config, _current_prefix
    old_vllm_config = _current_vllm_config
    old_prefix = _current_prefix
    from vllm.compilation.counter import compilation_counter
1786

1787
1788
    num_models_seen = compilation_counter.num_models_seen
    try:
1789
1790
1791
1792
1793
        # Clear the compilation config cache when context changes.
        # This is needed since the old config may have been accessed
        # and cached before the new config is set.
        get_cached_compilation_config.cache_clear()

1794
1795
1796
1797
1798
1799
1800
1801
1802
        _current_vllm_config = vllm_config
        _current_prefix = prefix
        yield
    except Exception:
        raise
    else:
        if check_compile:
            vllm_config.compilation_config.custom_op_log_check()

1803
1804
        if (
            check_compile
1805
            and vllm_config.compilation_config.mode == CompilationMode.VLLM_COMPILE
1806
1807
            and compilation_counter.num_models_seen == num_models_seen
        ):
1808
1809
1810
1811
1812
1813
1814
1815
1816
            # If the model supports compilation,
            # compilation_counter.num_models_seen should be increased
            # by at least 1.
            # If it is not increased, it means the model does not support
            # compilation (does not have @support_torch_compile decorator).
            logger.warning(
                "`torch.compile` is turned on, but the model %s"
                " does not support it. Please open an issue on GitHub"
                " if you want it to be supported.",
1817
1818
                vllm_config.model_config.model,
            )
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
    finally:
        _current_vllm_config = old_vllm_config
        _current_prefix = old_prefix
        # Clear the compilation config cache when context changes
        get_cached_compilation_config.cache_clear()


@lru_cache(maxsize=1)
def get_cached_compilation_config():
    """Cache config to avoid repeated calls to get_current_vllm_config()"""
    return get_current_vllm_config().compilation_config


def get_current_vllm_config() -> VllmConfig:
    if _current_vllm_config is None:
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
        raise AssertionError(
            "Current vLLM config is not set. This typically means "
            "get_current_vllm_config() was called outside of a "
            "set_current_vllm_config() context, or a CustomOp was instantiated "
            "at module import time or model forward time when config is not set. "
            "For tests that directly test custom ops/modules, use the "
            "'default_vllm_config' pytest fixture from tests/conftest.py."
        )
    return _current_vllm_config


def get_current_vllm_config_or_none() -> VllmConfig | None:
1846
1847
1848
1849
1850
1851
1852
    return _current_vllm_config


T = TypeVar("T")


def get_layers_from_vllm_config(
1853
1854
    vllm_config: VllmConfig,
    layer_type: type[T],
1855
    layer_names: list[str] | None = None,
1856
) -> dict[str, T]:
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
    """
    Get layers from the vLLM config.

    Args:
        vllm_config: The vLLM config.
        layer_type: The type of the layer to get.
        layer_names: The names of the layers to get. If None, return all layers.
    """

    if layer_names is None:
1867
        layer_names = list(vllm_config.compilation_config.static_forward_context.keys())
1868
1869
1870
1871
1872
1873

    forward_context = vllm_config.compilation_config.static_forward_context

    return {
        layer_name: forward_context[layer_name]
        for layer_name in layer_names
1874
1875
        if layer_name in forward_context
        and isinstance(forward_context[layer_name], layer_type)
1876
    }